From ab42cdb86290f8466a440e07ce252f785382a48a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 06:40:11 -0600 Subject: [PATCH 01/74] Remove prebuild and capgen --- CMakeLists.txt | 122 - LICENSE | 2 +- README.md | 2 + cmake/ccpp_capgen.cmake | 137 - pytest.ini | 3 - requirements.txt | 3 - run_codee_tmp.sh | 17 - schema/suite_v1_0.xsd | 128 - schema/suite_v2_0.xsd | 125 - scripts/ccpp_capgen.py | 768 ----- scripts/ccpp_database_obj.py | 87 - scripts/ccpp_datafile.py | 1214 -------- scripts/ccpp_fortran_to_metadata.py | 258 -- scripts/ccpp_prebuild.py | 856 ------ scripts/ccpp_state_machine.py | 28 - scripts/ccpp_suite.py | 1270 -------- scripts/ccpp_track_variables.py | 218 -- scripts/code_block.py | 119 - scripts/common.py | 233 -- scripts/constituents.py | 798 ----- scripts/conversion_tools/__init__.py | 54 - scripts/conversion_tools/unit_conversion.py | 206 -- scripts/ddt_library.py | 358 --- scripts/file_utils.py | 303 -- scripts/fortran_tools/__init__.py | 18 - scripts/fortran_tools/fortran_write.py | 436 --- .../offline_check_fortran_vs_metadata.py | 91 - scripts/fortran_tools/parse_fortran.py | 910 ------ scripts/fortran_tools/parse_fortran_file.py | 1095 ------- scripts/framework_env.py | 476 --- scripts/host_cap.py | 821 ----- scripts/host_model.py | 344 --- scripts/metadata2html.py | 131 - scripts/metadata_parser.py | 790 ----- scripts/metadata_table.py | 1504 ---------- scripts/metavar.py | 2139 ------------- scripts/mkcap.py | 831 ------ scripts/mkdoc.py | 177 -- scripts/mkstatic.py | 2145 -------------- scripts/parse_tools/__init__.py | 83 - scripts/parse_tools/fortran_conditional.py | 13 - scripts/parse_tools/parse_checkers.py | 1120 ------- scripts/parse_tools/parse_log.py | 53 - scripts/parse_tools/parse_object.py | 166 -- scripts/parse_tools/parse_source.py | 411 --- scripts/parse_tools/preprocess.py | 425 --- scripts/parse_tools/xml_tools.py | 572 ---- scripts/state_machine.py | 192 -- scripts/suite_objects.py | 2024 ------------- scripts/var_props.py | 1565 ---------- src/CMakeLists.txt | 45 - src/ccpp_constituent_prop_mod.F90 | 2636 ----------------- src/ccpp_constituent_prop_mod.meta | 64 - src/ccpp_hash_table.F90 | 520 ---- src/ccpp_hashable.F90 | 98 - src/ccpp_scheme_utils.F90 | 122 - src/ccpp_types.F90 | 92 - src/ccpp_types.meta | 103 - test/.pylintrc | 466 --- test/CMakeLists.txt | 17 - test/README.md | 75 - test/advection_test/CMakeLists.txt | 55 - test/advection_test/README.md | 21 - test/advection_test/advection_test_reports.py | 127 - .../apply_constituent_tendencies.F90 | 39 - .../apply_constituent_tendencies.meta | 36 - test/advection_test/cld_ice.F90 | 127 - test/advection_test/cld_ice.meta | 143 - test/advection_test/cld_liq.F90 | 102 - test/advection_test/cld_liq.meta | 135 - test/advection_test/cld_suite.xml | 11 - test/advection_test/cld_suite_error.xml | 9 - test/advection_test/const_indices.F90 | 95 - test/advection_test/const_indices.meta | 108 - test/advection_test/dlc_liq.F90 | 41 - test/advection_test/dlc_liq.meta | 29 - .../test_advection_host_integration.F90 | 80 - test/advection_test/test_host.F90 | 1114 ------- test/advection_test/test_host.meta | 38 - test/advection_test/test_host_data.F90 | 96 - test/advection_test/test_host_data.meta | 70 - test/advection_test/test_host_mod.F90 | 176 -- test/advection_test/test_host_mod.meta | 64 - test/capgen_test/CMakeLists.txt | 95 - test/capgen_test/README.md | 23 - test/capgen_test/adjust/temp_kinds.F90 | 12 - test/capgen_test/capgen_test_reports.py | 152 - test/capgen_test/ddt2.F90 | 12 - test/capgen_test/ddt_suite.xml | 8 - test/capgen_test/environ_conditions.meta | 111 - test/capgen_test/make_ddt.F90 | 142 - test/capgen_test/make_ddt.meta | 128 - test/capgen_test/setup_coeffs.F90 | 24 - test/capgen_test/setup_coeffs.meta | 29 - .../source_dir1/environ_conditions.F90 | 96 - test/capgen_test/source_dir2/temp_set.F90 | 124 - test/capgen_test/temp_adjust.F90 | 127 - test/capgen_test/temp_adjust.meta | 156 - test/capgen_test/temp_calc_adjust.F90 | 111 - test/capgen_test/temp_calc_adjust.meta | 111 - test/capgen_test/temp_set.meta | 214 -- test/capgen_test/temp_suite.xml | 12 - .../test_capgen_host_integration.F90 | 89 - test/capgen_test/test_host.F90 | 306 -- test/capgen_test/test_host.meta | 38 - test/capgen_test/test_host_data.F90 | 60 - test/capgen_test/test_host_data.meta | 59 - test/capgen_test/test_host_mod.F90 | 164 - test/capgen_test/test_host_mod.meta | 133 - test/ddthost_test/CMakeLists.txt | 53 - test/ddthost_test/README.md | 16 - test/ddthost_test/ddt_suite.xml | 8 - test/ddthost_test/ddthost_test_reports.py | 139 - test/ddthost_test/environ_conditions.F90 | 96 - test/ddthost_test/environ_conditions.meta | 110 - test/ddthost_test/host_ccpp_ddt.F90 | 16 - test/ddthost_test/host_ccpp_ddt.meta | 31 - test/ddthost_test/make_ddt.F90 | 133 - test/ddthost_test/make_ddt.meta | 132 - test/ddthost_test/setup_coeffs.F90 | 24 - test/ddthost_test/setup_coeffs.meta | 29 - test/ddthost_test/temp_adjust.F90 | 84 - test/ddthost_test/temp_adjust.meta | 119 - test/ddthost_test/temp_calc_adjust.F90 | 95 - test/ddthost_test/temp_calc_adjust.meta | 87 - test/ddthost_test/temp_set.F90 | 113 - test/ddthost_test/temp_set.meta | 181 -- test/ddthost_test/temp_suite.xml | 12 - .../test_ddt_host_integration.F90 | 82 - test/ddthost_test/test_host.F90 | 273 -- test/ddthost_test/test_host.meta | 18 - test/ddthost_test/test_host_data.F90 | 51 - test/ddthost_test/test_host_data.meta | 52 - test/ddthost_test/test_host_mod.F90 | 141 - test/ddthost_test/test_host_mod.meta | 98 - test/hash_table_tests/Makefile | 38 - test/hash_table_tests/test_hash.F90 | 218 -- test/nested_suite_test/CMakeLists.txt | 49 - test/nested_suite_test/README.md | 18 - test/nested_suite_test/ccpp_kinds.F90 | 27 - test/nested_suite_test/effr_calc.F90 | 84 - test/nested_suite_test/effr_calc.meta | 163 - test/nested_suite_test/effr_diag.F90 | 68 - test/nested_suite_test/effr_diag.meta | 65 - test/nested_suite_test/effr_post.F90 | 61 - test/nested_suite_test/effr_post.meta | 65 - test/nested_suite_test/effr_pre.F90 | 60 - test/nested_suite_test/effr_pre.meta | 66 - test/nested_suite_test/effrs_calc.F90 | 32 - test/nested_suite_test/effrs_calc.meta | 25 - test/nested_suite_test/main_suite.xml | 18 - test/nested_suite_test/module_rad_ddt.F90 | 23 - test/nested_suite_test/module_rad_ddt.meta | 40 - test/nested_suite_test/rad_lw.F90 | 35 - test/nested_suite_test/rad_lw.meta | 35 - test/nested_suite_test/rad_sw.F90 | 35 - test/nested_suite_test/rad_sw.meta | 41 - test/nested_suite_test/radiation2_suite.xml | 10 - .../nested_suite_test/radiation3_subsuite.xml | 7 - test/nested_suite_test/radiation3_suite.xml | 7 - test/nested_suite_test/radiation4_suite.xml | 7 - test/nested_suite_test/test_host.F90 | 264 -- test/nested_suite_test/test_host.meta | 38 - test/nested_suite_test/test_host_data.F90 | 103 - test/nested_suite_test/test_host_data.meta | 128 - test/nested_suite_test/test_host_mod.F90 | 132 - test/nested_suite_test/test_host_mod.meta | 42 - .../test_nested_suite_integration.F90 | 91 - test/pylint_test.sh | 28 - test/test_fortran_to_metadata.sh | 37 - test/test_offline_metadata_checker.sh | 35 - test/test_stub.py | 162 - test/unit_tests/README.md | 29 - .../bad_kind_spec_table_properties.meta | 5 - .../check_fortran_to_metadata.meta | 31 - .../sample_files/double_header.meta | 12 - .../sample_files/double_table_properties.meta | 13 - .../duplicate_kind_spec_table_properties.meta | 6 - .../fortran_files/array_parsing_test.F90 | 33 - .../fortran_files/comments_test.F90 | 34 - .../fortran_files/linebreak_test.F90 | 52 - .../fortran_files/long_string_test.F90 | 97 - .../good_kind_spec_table_properties.meta | 6 - .../missing_table_properties.meta | 3 - .../test_bad_1st_arg_table_header.meta | 23 - .../test_bad_2nd_arg_table_header.meta | 23 - .../sample_files/test_bad_dimension.meta | 15 - .../sample_files/test_bad_line_split.meta | 16 - .../sample_files/test_bad_table_key.meta | 10 - .../sample_files/test_bad_table_type.meta | 15 - .../sample_files/test_bad_type_name.meta | 9 - .../test_bad_var_property_name.meta | 35 - .../sample_files/test_dependencies_path.meta | 11 - .../sample_files/test_duplicate_variable.meta | 21 - .../sample_files/test_fortran_to_metadata.F90 | 28 - test/unit_tests/sample_files/test_host.meta | 34 - .../sample_files/test_invalid_intent.meta | 23 - .../test_invalid_table_properties_type.meta | 9 - .../test_mismatch_section_table_title.meta | 9 - .../sample_files/test_missing_intent.meta | 22 - .../sample_files/test_missing_table_name.meta | 14 - .../sample_files/test_missing_table_type.meta | 14 - .../sample_files/test_missing_units.meta | 16 - .../test_multi_ccpp_arg_tables.meta | 115 - .../sample_files/test_unknown_ddt_type.meta | 14 - .../sample_host_files/data1_mod.F90 | 11 - .../sample_host_files/data1_mod.meta | 25 - test/unit_tests/sample_host_files/ddt1.F90 | 17 - test/unit_tests/sample_host_files/ddt1.meta | 20 - .../sample_host_files/ddt1_plus.F90 | 33 - .../sample_host_files/ddt1_plus.meta | 20 - test/unit_tests/sample_host_files/ddt2.F90 | 24 - test/unit_tests/sample_host_files/ddt2.meta | 29 - .../sample_host_files/ddt2_extra_var.F90 | 34 - .../sample_host_files/ddt2_extra_var.meta | 34 - .../sample_host_files/ddt_data1_mod.F90 | 30 - .../sample_host_files/ddt_data1_mod.meta | 56 - .../sample_host_files/mismatch_hdim_mod.F90 | 11 - .../sample_host_files/mismatch_hdim_mod.meta | 25 - .../CCPPeq1_var_in_fort_meta.F90 | 38 - .../CCPPeq1_var_in_fort_meta.meta | 37 - .../CCPPeq1_var_missing_in_fort.F90 | 38 - .../CCPPeq1_var_missing_in_fort.meta | 37 - .../CCPPeq1_var_missing_in_meta.F90 | 38 - .../CCPPeq1_var_missing_in_meta.meta | 29 - .../CCPPgt1_var_in_fort_meta.F90 | 38 - .../CCPPgt1_var_in_fort_meta.meta | 37 - .../CCPPnotset_var_missing_in_meta.F90 | 38 - .../CCPPnotset_var_missing_in_meta.meta | 29 - .../sample_scheme_files/invalid_dummy_arg.F90 | 43 - .../invalid_dummy_arg.meta | 66 - .../invalid_subr_stmnt.F90 | 30 - .../invalid_subr_stmnt.meta | 23 - .../sample_scheme_files/mismatch_hdim.F90 | 48 - .../sample_scheme_files/mismatch_hdim.meta | 55 - .../sample_scheme_files/mismatch_intent.F90 | 75 - .../sample_scheme_files/mismatch_intent.meta | 102 - .../sample_scheme_files/missing_arg_table.F90 | 75 - .../missing_arg_table.meta | 41 - .../missing_fort_header.F90 | 73 - .../missing_fort_header.meta | 102 - .../sample_scheme_files/reorder.F90 | 73 - .../sample_scheme_files/reorder.meta | 102 - .../sample_scheme_files/temp_adjust.F90 | 96 - .../sample_scheme_files/temp_adjust.meta | 134 - .../sample_suite_files/another_suite.xml | 10 - .../sample_suite_files/another_suite2.xml | 16 - .../sample_suite_files/nested_full_suite.xml | 10 - .../sample_suite_files/subsuite1.xml | 7 - .../sample_suite_files/subsuite_inline.xml | 9 - .../suite_bad_v2_duplicate_group.xml | 16 - .../suite_bad_v2_suite_tag.xml | 7 - .../suite_bad_version01.xml | 8 - .../suite_bad_version02.xml | 8 - .../suite_bad_version03.xml | 8 - .../suite_bad_version04.xml | 8 - .../suite_good_v1_test01.xml | 8 - .../suite_good_v1_test02.xml | 11 - .../suite_good_v2_test01.xml | 9 - .../suite_good_v2_test01_exp.xml | 11 - .../suite_good_v2_test02.xml | 10 - .../suite_good_v2_test02_exp.xml | 13 - .../suite_good_v2_test03.xml | 19 - .../suite_good_v2_test03_exp.xml | 30 - .../suite_good_v2_test04.xml | 18 - .../suite_good_v2_test04_exp.xml | 26 - .../suite_invalid_group_fortran_id.xml | 8 - .../suite_invalid_scheme_fortran_id.xml | 8 - .../suite_invalid_suite_fortran_id.xml | 8 - .../sample_suite_files/suite_missing_file.xml | 9 - .../suite_missing_group.xml | 7 - .../suite_missing_loaded_suite.xml | 16 - .../suite_missing_version.xml | 8 - .../suite_recurse_level2.xml | 10 - .../suite_recurse_level2a.xml | 10 - .../suite_recurse_level3.xml | 10 - .../suite_recurse_level3a.xml | 10 - .../sample_suite_files/suite_recurse_top1.xml | 18 - .../sample_suite_files/suite_recurse_top2.xml | 18 - test/unit_tests/test_common.py | 92 - test/unit_tests/test_fortran_parse.py | 101 - test/unit_tests/test_fortran_write.py | 172 -- test/unit_tests/test_metadata_host_file.py | 280 -- test/unit_tests/test_metadata_scheme_file.py | 357 --- test/unit_tests/test_metadata_table.py | 445 --- test/unit_tests/test_sdf.py | 569 ---- test/unit_tests/test_var_transforms.py | 475 --- test/unit_tests/xmllint_wrapper/xmllint | 24 - test/utils/CMakeLists.txt | 1 - test/utils/test_utils.F90 | 88 - test/var_compatibility_test/CMakeLists.txt | 49 - test/var_compatibility_test/README.md | 23 - test/var_compatibility_test/effr_calc.F90 | 84 - test/var_compatibility_test/effr_calc.meta | 163 - test/var_compatibility_test/effr_diag.F90 | 68 - test/var_compatibility_test/effr_diag.meta | 65 - test/var_compatibility_test/effr_post.F90 | 61 - test/var_compatibility_test/effr_post.meta | 65 - test/var_compatibility_test/effr_pre.F90 | 60 - test/var_compatibility_test/effr_pre.meta | 66 - test/var_compatibility_test/effrs_calc.F90 | 32 - test/var_compatibility_test/effrs_calc.meta | 25 - .../var_compatibility_test/module_rad_ddt.F90 | 23 - .../module_rad_ddt.meta | 40 - test/var_compatibility_test/rad_lw.F90 | 35 - test/var_compatibility_test/rad_lw.meta | 35 - test/var_compatibility_test/rad_sw.F90 | 35 - test/var_compatibility_test/rad_sw.meta | 41 - test/var_compatibility_test/test_host.F90 | 264 -- test/var_compatibility_test/test_host.meta | 38 - .../var_compatibility_test/test_host_data.F90 | 103 - .../test_host_data.meta | 128 - test/var_compatibility_test/test_host_mod.F90 | 132 - .../var_compatibility_test/test_host_mod.meta | 42 - .../test_var_compatibility_integration.F90 | 88 - .../var_compatibility_suite.xml | 21 - .../var_compatibility_test_reports.py | 116 - test_prebuild/run_all_tests.sh | 12 - .../test_blocked_data/CMakeLists.txt | 98 - test_prebuild/test_blocked_data/README.md | 14 - .../test_blocked_data/blocked_data_scheme.F90 | 126 - .../blocked_data_scheme.meta | 147 - .../test_blocked_data/blocked_data_suite.xml | 9 - .../test_blocked_data/ccpp_prebuild_config.py | 81 - test_prebuild/test_blocked_data/data.F90 | 41 - test_prebuild/test_blocked_data/data.meta | 69 - test_prebuild/test_blocked_data/main.F90 | 117 - test_prebuild/test_blocked_data/run_test.sh | 11 - .../test_chunked_data/CMakeLists.txt | 98 - test_prebuild/test_chunked_data/README.md | 16 - .../test_chunked_data/ccpp_prebuild_config.py | 81 - .../test_chunked_data/chunked_data_scheme.F90 | 126 - .../chunked_data_scheme.meta | 147 - test_prebuild/test_chunked_data/data.F90 | 43 - test_prebuild/test_chunked_data/data.meta | 76 - test_prebuild/test_chunked_data/main.F90 | 114 - test_prebuild/test_chunked_data/run_test.sh | 13 - .../suite_chunked_data_suite.xml | 9 - test_prebuild/test_opt_arg/CMakeLists.txt | 98 - test_prebuild/test_opt_arg/ccpp_kinds.F90 | 13 - test_prebuild/test_opt_arg/ccpp_kinds.meta | 15 - .../test_opt_arg/ccpp_prebuild_config.py | 81 - test_prebuild/test_opt_arg/data.F90 | 23 - test_prebuild/test_opt_arg/data.meta | 46 - test_prebuild/test_opt_arg/main.F90 | 125 - test_prebuild/test_opt_arg/opt_arg_scheme.F90 | 90 - .../test_opt_arg/opt_arg_scheme.meta | 157 - test_prebuild/test_opt_arg/run_test.sh | 13 - .../test_opt_arg/suite_opt_arg_suite.xml | 9 - test_prebuild/test_track_variables.py | 130 - .../ccpp_prebuild_config.py | 69 - .../test_track_variables/scheme_1.meta | 25 - .../test_track_variables/scheme_2.meta | 87 - .../test_track_variables/scheme_3.meta | 143 - .../test_track_variables/scheme_4.meta | 31 - .../test_track_variables/scheme_A.meta | 31 - .../test_track_variables/scheme_B.meta | 48 - .../test_track_variables/suite_TEST_SUITE.xml | 24 - .../suite_small_suite.xml | 14 - test_prebuild/test_unit_conv/CMakeLists.txt | 98 - test_prebuild/test_unit_conv/README.md | 16 - test_prebuild/test_unit_conv/ccpp_kinds.F90 | 13 - test_prebuild/test_unit_conv/ccpp_kinds.meta | 15 - .../test_unit_conv/ccpp_prebuild_config.py | 82 - test_prebuild/test_unit_conv/data.F90 | 24 - test_prebuild/test_unit_conv/data.meta | 66 - test_prebuild/test_unit_conv/main.F90 | 97 - test_prebuild/test_unit_conv/run_test.sh | 13 - .../test_unit_conv/suite_unit_conv_suite.xml | 11 - .../test_unit_conv/unit_conv_scheme_1.F90 | 70 - .../test_unit_conv/unit_conv_scheme_1.meta | 49 - .../test_unit_conv/unit_conv_scheme_2.F90 | 69 - .../test_unit_conv/unit_conv_scheme_2.meta | 49 - test_prebuild/unit_tests/run_tests.sh | 10 - .../unit_tests/test_metadata_parser.py | 57 - test_prebuild/unit_tests/test_mkstatic.py | 22 - 376 files changed, 3 insertions(+), 50930 deletions(-) delete mode 100644 CMakeLists.txt delete mode 100644 cmake/ccpp_capgen.cmake delete mode 100644 pytest.ini delete mode 100644 requirements.txt delete mode 100755 run_codee_tmp.sh delete mode 100644 schema/suite_v1_0.xsd delete mode 100644 schema/suite_v2_0.xsd delete mode 100755 scripts/ccpp_capgen.py delete mode 100644 scripts/ccpp_database_obj.py delete mode 100755 scripts/ccpp_datafile.py delete mode 100755 scripts/ccpp_fortran_to_metadata.py delete mode 100755 scripts/ccpp_prebuild.py delete mode 100644 scripts/ccpp_state_machine.py delete mode 100644 scripts/ccpp_suite.py delete mode 100755 scripts/ccpp_track_variables.py delete mode 100644 scripts/code_block.py delete mode 100755 scripts/common.py delete mode 100644 scripts/constituents.py delete mode 100644 scripts/conversion_tools/__init__.py delete mode 100755 scripts/conversion_tools/unit_conversion.py delete mode 100644 scripts/ddt_library.py delete mode 100644 scripts/file_utils.py delete mode 100644 scripts/fortran_tools/__init__.py delete mode 100644 scripts/fortran_tools/fortran_write.py delete mode 100755 scripts/fortran_tools/offline_check_fortran_vs_metadata.py delete mode 100644 scripts/fortran_tools/parse_fortran.py delete mode 100644 scripts/fortran_tools/parse_fortran_file.py delete mode 100644 scripts/framework_env.py delete mode 100644 scripts/host_cap.py delete mode 100644 scripts/host_model.py delete mode 100755 scripts/metadata2html.py delete mode 100755 scripts/metadata_parser.py delete mode 100755 scripts/metadata_table.py delete mode 100755 scripts/metavar.py delete mode 100755 scripts/mkcap.py delete mode 100755 scripts/mkdoc.py delete mode 100755 scripts/mkstatic.py delete mode 100644 scripts/parse_tools/__init__.py delete mode 100755 scripts/parse_tools/fortran_conditional.py delete mode 100755 scripts/parse_tools/parse_checkers.py delete mode 100644 scripts/parse_tools/parse_log.py delete mode 100644 scripts/parse_tools/parse_object.py delete mode 100644 scripts/parse_tools/parse_source.py delete mode 100755 scripts/parse_tools/preprocess.py delete mode 100644 scripts/parse_tools/xml_tools.py delete mode 100755 scripts/state_machine.py delete mode 100755 scripts/suite_objects.py delete mode 100755 scripts/var_props.py delete mode 100644 src/CMakeLists.txt delete mode 100644 src/ccpp_constituent_prop_mod.F90 delete mode 100644 src/ccpp_constituent_prop_mod.meta delete mode 100644 src/ccpp_hash_table.F90 delete mode 100644 src/ccpp_hashable.F90 delete mode 100644 src/ccpp_scheme_utils.F90 delete mode 100644 src/ccpp_types.F90 delete mode 100644 src/ccpp_types.meta delete mode 100644 test/.pylintrc delete mode 100644 test/CMakeLists.txt delete mode 100644 test/README.md delete mode 100644 test/advection_test/CMakeLists.txt delete mode 100644 test/advection_test/README.md delete mode 100644 test/advection_test/advection_test_reports.py delete mode 100644 test/advection_test/apply_constituent_tendencies.F90 delete mode 100644 test/advection_test/apply_constituent_tendencies.meta delete mode 100644 test/advection_test/cld_ice.F90 delete mode 100644 test/advection_test/cld_ice.meta delete mode 100644 test/advection_test/cld_liq.F90 delete mode 100644 test/advection_test/cld_liq.meta delete mode 100644 test/advection_test/cld_suite.xml delete mode 100644 test/advection_test/cld_suite_error.xml delete mode 100644 test/advection_test/const_indices.F90 delete mode 100644 test/advection_test/const_indices.meta delete mode 100644 test/advection_test/dlc_liq.F90 delete mode 100644 test/advection_test/dlc_liq.meta delete mode 100644 test/advection_test/test_advection_host_integration.F90 delete mode 100644 test/advection_test/test_host.F90 delete mode 100644 test/advection_test/test_host.meta delete mode 100644 test/advection_test/test_host_data.F90 delete mode 100644 test/advection_test/test_host_data.meta delete mode 100644 test/advection_test/test_host_mod.F90 delete mode 100644 test/advection_test/test_host_mod.meta delete mode 100644 test/capgen_test/CMakeLists.txt delete mode 100644 test/capgen_test/README.md delete mode 100644 test/capgen_test/adjust/temp_kinds.F90 delete mode 100644 test/capgen_test/capgen_test_reports.py delete mode 100644 test/capgen_test/ddt2.F90 delete mode 100644 test/capgen_test/ddt_suite.xml delete mode 100644 test/capgen_test/environ_conditions.meta delete mode 100644 test/capgen_test/make_ddt.F90 delete mode 100644 test/capgen_test/make_ddt.meta delete mode 100644 test/capgen_test/setup_coeffs.F90 delete mode 100644 test/capgen_test/setup_coeffs.meta delete mode 100644 test/capgen_test/source_dir1/environ_conditions.F90 delete mode 100644 test/capgen_test/source_dir2/temp_set.F90 delete mode 100644 test/capgen_test/temp_adjust.F90 delete mode 100644 test/capgen_test/temp_adjust.meta delete mode 100644 test/capgen_test/temp_calc_adjust.F90 delete mode 100644 test/capgen_test/temp_calc_adjust.meta delete mode 100644 test/capgen_test/temp_set.meta delete mode 100644 test/capgen_test/temp_suite.xml delete mode 100644 test/capgen_test/test_capgen_host_integration.F90 delete mode 100644 test/capgen_test/test_host.F90 delete mode 100644 test/capgen_test/test_host.meta delete mode 100644 test/capgen_test/test_host_data.F90 delete mode 100644 test/capgen_test/test_host_data.meta delete mode 100644 test/capgen_test/test_host_mod.F90 delete mode 100644 test/capgen_test/test_host_mod.meta delete mode 100644 test/ddthost_test/CMakeLists.txt delete mode 100644 test/ddthost_test/README.md delete mode 100644 test/ddthost_test/ddt_suite.xml delete mode 100644 test/ddthost_test/ddthost_test_reports.py delete mode 100644 test/ddthost_test/environ_conditions.F90 delete mode 100644 test/ddthost_test/environ_conditions.meta delete mode 100644 test/ddthost_test/host_ccpp_ddt.F90 delete mode 100644 test/ddthost_test/host_ccpp_ddt.meta delete mode 100644 test/ddthost_test/make_ddt.F90 delete mode 100644 test/ddthost_test/make_ddt.meta delete mode 100644 test/ddthost_test/setup_coeffs.F90 delete mode 100644 test/ddthost_test/setup_coeffs.meta delete mode 100644 test/ddthost_test/temp_adjust.F90 delete mode 100644 test/ddthost_test/temp_adjust.meta delete mode 100644 test/ddthost_test/temp_calc_adjust.F90 delete mode 100644 test/ddthost_test/temp_calc_adjust.meta delete mode 100644 test/ddthost_test/temp_set.F90 delete mode 100644 test/ddthost_test/temp_set.meta delete mode 100644 test/ddthost_test/temp_suite.xml delete mode 100644 test/ddthost_test/test_ddt_host_integration.F90 delete mode 100644 test/ddthost_test/test_host.F90 delete mode 100644 test/ddthost_test/test_host.meta delete mode 100644 test/ddthost_test/test_host_data.F90 delete mode 100644 test/ddthost_test/test_host_data.meta delete mode 100644 test/ddthost_test/test_host_mod.F90 delete mode 100644 test/ddthost_test/test_host_mod.meta delete mode 100644 test/hash_table_tests/Makefile delete mode 100644 test/hash_table_tests/test_hash.F90 delete mode 100644 test/nested_suite_test/CMakeLists.txt delete mode 100644 test/nested_suite_test/README.md delete mode 100644 test/nested_suite_test/ccpp_kinds.F90 delete mode 100644 test/nested_suite_test/effr_calc.F90 delete mode 100644 test/nested_suite_test/effr_calc.meta delete mode 100644 test/nested_suite_test/effr_diag.F90 delete mode 100644 test/nested_suite_test/effr_diag.meta delete mode 100644 test/nested_suite_test/effr_post.F90 delete mode 100644 test/nested_suite_test/effr_post.meta delete mode 100644 test/nested_suite_test/effr_pre.F90 delete mode 100644 test/nested_suite_test/effr_pre.meta delete mode 100644 test/nested_suite_test/effrs_calc.F90 delete mode 100644 test/nested_suite_test/effrs_calc.meta delete mode 100644 test/nested_suite_test/main_suite.xml delete mode 100644 test/nested_suite_test/module_rad_ddt.F90 delete mode 100644 test/nested_suite_test/module_rad_ddt.meta delete mode 100644 test/nested_suite_test/rad_lw.F90 delete mode 100644 test/nested_suite_test/rad_lw.meta delete mode 100644 test/nested_suite_test/rad_sw.F90 delete mode 100644 test/nested_suite_test/rad_sw.meta delete mode 100644 test/nested_suite_test/radiation2_suite.xml delete mode 100644 test/nested_suite_test/radiation3_subsuite.xml delete mode 100644 test/nested_suite_test/radiation3_suite.xml delete mode 100644 test/nested_suite_test/radiation4_suite.xml delete mode 100644 test/nested_suite_test/test_host.F90 delete mode 100644 test/nested_suite_test/test_host.meta delete mode 100644 test/nested_suite_test/test_host_data.F90 delete mode 100644 test/nested_suite_test/test_host_data.meta delete mode 100644 test/nested_suite_test/test_host_mod.F90 delete mode 100644 test/nested_suite_test/test_host_mod.meta delete mode 100644 test/nested_suite_test/test_nested_suite_integration.F90 delete mode 100755 test/pylint_test.sh delete mode 100755 test/test_fortran_to_metadata.sh delete mode 100755 test/test_offline_metadata_checker.sh delete mode 100644 test/test_stub.py delete mode 100644 test/unit_tests/README.md delete mode 100644 test/unit_tests/sample_files/bad_kind_spec_table_properties.meta delete mode 100644 test/unit_tests/sample_files/check_fortran_to_metadata.meta delete mode 100644 test/unit_tests/sample_files/double_header.meta delete mode 100644 test/unit_tests/sample_files/double_table_properties.meta delete mode 100644 test/unit_tests/sample_files/duplicate_kind_spec_table_properties.meta delete mode 100644 test/unit_tests/sample_files/fortran_files/array_parsing_test.F90 delete mode 100644 test/unit_tests/sample_files/fortran_files/comments_test.F90 delete mode 100644 test/unit_tests/sample_files/fortran_files/linebreak_test.F90 delete mode 100644 test/unit_tests/sample_files/fortran_files/long_string_test.F90 delete mode 100644 test/unit_tests/sample_files/good_kind_spec_table_properties.meta delete mode 100644 test/unit_tests/sample_files/missing_table_properties.meta delete mode 100644 test/unit_tests/sample_files/test_bad_1st_arg_table_header.meta delete mode 100644 test/unit_tests/sample_files/test_bad_2nd_arg_table_header.meta delete mode 100644 test/unit_tests/sample_files/test_bad_dimension.meta delete mode 100644 test/unit_tests/sample_files/test_bad_line_split.meta delete mode 100644 test/unit_tests/sample_files/test_bad_table_key.meta delete mode 100644 test/unit_tests/sample_files/test_bad_table_type.meta delete mode 100644 test/unit_tests/sample_files/test_bad_type_name.meta delete mode 100644 test/unit_tests/sample_files/test_bad_var_property_name.meta delete mode 100644 test/unit_tests/sample_files/test_dependencies_path.meta delete mode 100644 test/unit_tests/sample_files/test_duplicate_variable.meta delete mode 100644 test/unit_tests/sample_files/test_fortran_to_metadata.F90 delete mode 100644 test/unit_tests/sample_files/test_host.meta delete mode 100644 test/unit_tests/sample_files/test_invalid_intent.meta delete mode 100644 test/unit_tests/sample_files/test_invalid_table_properties_type.meta delete mode 100644 test/unit_tests/sample_files/test_mismatch_section_table_title.meta delete mode 100644 test/unit_tests/sample_files/test_missing_intent.meta delete mode 100644 test/unit_tests/sample_files/test_missing_table_name.meta delete mode 100644 test/unit_tests/sample_files/test_missing_table_type.meta delete mode 100644 test/unit_tests/sample_files/test_missing_units.meta delete mode 100644 test/unit_tests/sample_files/test_multi_ccpp_arg_tables.meta delete mode 100644 test/unit_tests/sample_files/test_unknown_ddt_type.meta delete mode 100644 test/unit_tests/sample_host_files/data1_mod.F90 delete mode 100644 test/unit_tests/sample_host_files/data1_mod.meta delete mode 100644 test/unit_tests/sample_host_files/ddt1.F90 delete mode 100644 test/unit_tests/sample_host_files/ddt1.meta delete mode 100644 test/unit_tests/sample_host_files/ddt1_plus.F90 delete mode 100644 test/unit_tests/sample_host_files/ddt1_plus.meta delete mode 100644 test/unit_tests/sample_host_files/ddt2.F90 delete mode 100644 test/unit_tests/sample_host_files/ddt2.meta delete mode 100644 test/unit_tests/sample_host_files/ddt2_extra_var.F90 delete mode 100644 test/unit_tests/sample_host_files/ddt2_extra_var.meta delete mode 100644 test/unit_tests/sample_host_files/ddt_data1_mod.F90 delete mode 100644 test/unit_tests/sample_host_files/ddt_data1_mod.meta delete mode 100644 test/unit_tests/sample_host_files/mismatch_hdim_mod.F90 delete mode 100644 test/unit_tests/sample_host_files/mismatch_hdim_mod.meta delete mode 100644 test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.F90 delete mode 100644 test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.meta delete mode 100644 test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.F90 delete mode 100644 test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.meta delete mode 100644 test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.F90 delete mode 100644 test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.meta delete mode 100644 test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.F90 delete mode 100644 test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.meta delete mode 100644 test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.F90 delete mode 100644 test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.meta delete mode 100644 test/unit_tests/sample_scheme_files/invalid_dummy_arg.F90 delete mode 100644 test/unit_tests/sample_scheme_files/invalid_dummy_arg.meta delete mode 100644 test/unit_tests/sample_scheme_files/invalid_subr_stmnt.F90 delete mode 100644 test/unit_tests/sample_scheme_files/invalid_subr_stmnt.meta delete mode 100644 test/unit_tests/sample_scheme_files/mismatch_hdim.F90 delete mode 100644 test/unit_tests/sample_scheme_files/mismatch_hdim.meta delete mode 100644 test/unit_tests/sample_scheme_files/mismatch_intent.F90 delete mode 100644 test/unit_tests/sample_scheme_files/mismatch_intent.meta delete mode 100644 test/unit_tests/sample_scheme_files/missing_arg_table.F90 delete mode 100644 test/unit_tests/sample_scheme_files/missing_arg_table.meta delete mode 100644 test/unit_tests/sample_scheme_files/missing_fort_header.F90 delete mode 100644 test/unit_tests/sample_scheme_files/missing_fort_header.meta delete mode 100644 test/unit_tests/sample_scheme_files/reorder.F90 delete mode 100644 test/unit_tests/sample_scheme_files/reorder.meta delete mode 100644 test/unit_tests/sample_scheme_files/temp_adjust.F90 delete mode 100644 test/unit_tests/sample_scheme_files/temp_adjust.meta delete mode 100644 test/unit_tests/sample_suite_files/another_suite.xml delete mode 100644 test/unit_tests/sample_suite_files/another_suite2.xml delete mode 100644 test/unit_tests/sample_suite_files/nested_full_suite.xml delete mode 100644 test/unit_tests/sample_suite_files/subsuite1.xml delete mode 100644 test/unit_tests/sample_suite_files/subsuite_inline.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_bad_v2_duplicate_group.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_bad_v2_suite_tag.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_bad_version01.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_bad_version02.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_bad_version03.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_bad_version04.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_good_v1_test01.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_good_v1_test02.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test01.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test01_exp.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test02.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test02_exp.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test03.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test03_exp.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test04.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_good_v2_test04_exp.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_invalid_group_fortran_id.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_invalid_suite_fortran_id.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_missing_file.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_missing_group.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_missing_loaded_suite.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_missing_version.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_recurse_level2.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_recurse_level2a.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_recurse_level3.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_recurse_level3a.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_recurse_top1.xml delete mode 100644 test/unit_tests/sample_suite_files/suite_recurse_top2.xml delete mode 100644 test/unit_tests/test_common.py delete mode 100644 test/unit_tests/test_fortran_parse.py delete mode 100644 test/unit_tests/test_fortran_write.py delete mode 100644 test/unit_tests/test_metadata_host_file.py delete mode 100644 test/unit_tests/test_metadata_scheme_file.py delete mode 100644 test/unit_tests/test_metadata_table.py delete mode 100644 test/unit_tests/test_sdf.py delete mode 100644 test/unit_tests/test_var_transforms.py delete mode 100755 test/unit_tests/xmllint_wrapper/xmllint delete mode 100644 test/utils/CMakeLists.txt delete mode 100644 test/utils/test_utils.F90 delete mode 100644 test/var_compatibility_test/CMakeLists.txt delete mode 100644 test/var_compatibility_test/README.md delete mode 100644 test/var_compatibility_test/effr_calc.F90 delete mode 100644 test/var_compatibility_test/effr_calc.meta delete mode 100644 test/var_compatibility_test/effr_diag.F90 delete mode 100644 test/var_compatibility_test/effr_diag.meta delete mode 100644 test/var_compatibility_test/effr_post.F90 delete mode 100644 test/var_compatibility_test/effr_post.meta delete mode 100644 test/var_compatibility_test/effr_pre.F90 delete mode 100644 test/var_compatibility_test/effr_pre.meta delete mode 100644 test/var_compatibility_test/effrs_calc.F90 delete mode 100644 test/var_compatibility_test/effrs_calc.meta delete mode 100644 test/var_compatibility_test/module_rad_ddt.F90 delete mode 100644 test/var_compatibility_test/module_rad_ddt.meta delete mode 100644 test/var_compatibility_test/rad_lw.F90 delete mode 100644 test/var_compatibility_test/rad_lw.meta delete mode 100644 test/var_compatibility_test/rad_sw.F90 delete mode 100644 test/var_compatibility_test/rad_sw.meta delete mode 100644 test/var_compatibility_test/test_host.F90 delete mode 100644 test/var_compatibility_test/test_host.meta delete mode 100644 test/var_compatibility_test/test_host_data.F90 delete mode 100644 test/var_compatibility_test/test_host_data.meta delete mode 100644 test/var_compatibility_test/test_host_mod.F90 delete mode 100644 test/var_compatibility_test/test_host_mod.meta delete mode 100644 test/var_compatibility_test/test_var_compatibility_integration.F90 delete mode 100644 test/var_compatibility_test/var_compatibility_suite.xml delete mode 100755 test/var_compatibility_test/var_compatibility_test_reports.py delete mode 100755 test_prebuild/run_all_tests.sh delete mode 100644 test_prebuild/test_blocked_data/CMakeLists.txt delete mode 100644 test_prebuild/test_blocked_data/README.md delete mode 100644 test_prebuild/test_blocked_data/blocked_data_scheme.F90 delete mode 100644 test_prebuild/test_blocked_data/blocked_data_scheme.meta delete mode 100644 test_prebuild/test_blocked_data/blocked_data_suite.xml delete mode 100644 test_prebuild/test_blocked_data/ccpp_prebuild_config.py delete mode 100644 test_prebuild/test_blocked_data/data.F90 delete mode 100644 test_prebuild/test_blocked_data/data.meta delete mode 100644 test_prebuild/test_blocked_data/main.F90 delete mode 100755 test_prebuild/test_blocked_data/run_test.sh delete mode 100644 test_prebuild/test_chunked_data/CMakeLists.txt delete mode 100644 test_prebuild/test_chunked_data/README.md delete mode 100755 test_prebuild/test_chunked_data/ccpp_prebuild_config.py delete mode 100644 test_prebuild/test_chunked_data/chunked_data_scheme.F90 delete mode 100644 test_prebuild/test_chunked_data/chunked_data_scheme.meta delete mode 100644 test_prebuild/test_chunked_data/data.F90 delete mode 100644 test_prebuild/test_chunked_data/data.meta delete mode 100644 test_prebuild/test_chunked_data/main.F90 delete mode 100755 test_prebuild/test_chunked_data/run_test.sh delete mode 100644 test_prebuild/test_chunked_data/suite_chunked_data_suite.xml delete mode 100644 test_prebuild/test_opt_arg/CMakeLists.txt delete mode 100644 test_prebuild/test_opt_arg/ccpp_kinds.F90 delete mode 100644 test_prebuild/test_opt_arg/ccpp_kinds.meta delete mode 100755 test_prebuild/test_opt_arg/ccpp_prebuild_config.py delete mode 100644 test_prebuild/test_opt_arg/data.F90 delete mode 100644 test_prebuild/test_opt_arg/data.meta delete mode 100644 test_prebuild/test_opt_arg/main.F90 delete mode 100644 test_prebuild/test_opt_arg/opt_arg_scheme.F90 delete mode 100644 test_prebuild/test_opt_arg/opt_arg_scheme.meta delete mode 100755 test_prebuild/test_opt_arg/run_test.sh delete mode 100644 test_prebuild/test_opt_arg/suite_opt_arg_suite.xml delete mode 100755 test_prebuild/test_track_variables.py delete mode 100755 test_prebuild/test_track_variables/ccpp_prebuild_config.py delete mode 100644 test_prebuild/test_track_variables/scheme_1.meta delete mode 100644 test_prebuild/test_track_variables/scheme_2.meta delete mode 100644 test_prebuild/test_track_variables/scheme_3.meta delete mode 100644 test_prebuild/test_track_variables/scheme_4.meta delete mode 100644 test_prebuild/test_track_variables/scheme_A.meta delete mode 100644 test_prebuild/test_track_variables/scheme_B.meta delete mode 100644 test_prebuild/test_track_variables/suite_TEST_SUITE.xml delete mode 100644 test_prebuild/test_track_variables/suite_small_suite.xml delete mode 100644 test_prebuild/test_unit_conv/CMakeLists.txt delete mode 100644 test_prebuild/test_unit_conv/README.md delete mode 100644 test_prebuild/test_unit_conv/ccpp_kinds.F90 delete mode 100644 test_prebuild/test_unit_conv/ccpp_kinds.meta delete mode 100755 test_prebuild/test_unit_conv/ccpp_prebuild_config.py delete mode 100644 test_prebuild/test_unit_conv/data.F90 delete mode 100644 test_prebuild/test_unit_conv/data.meta delete mode 100644 test_prebuild/test_unit_conv/main.F90 delete mode 100755 test_prebuild/test_unit_conv/run_test.sh delete mode 100644 test_prebuild/test_unit_conv/suite_unit_conv_suite.xml delete mode 100644 test_prebuild/test_unit_conv/unit_conv_scheme_1.F90 delete mode 100644 test_prebuild/test_unit_conv/unit_conv_scheme_1.meta delete mode 100644 test_prebuild/test_unit_conv/unit_conv_scheme_2.F90 delete mode 100644 test_prebuild/test_unit_conv/unit_conv_scheme_2.meta delete mode 100755 test_prebuild/unit_tests/run_tests.sh delete mode 100644 test_prebuild/unit_tests/test_metadata_parser.py delete mode 100644 test_prebuild/unit_tests/test_mkstatic.py diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index c0aa6cee..00000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,122 +0,0 @@ -cmake_minimum_required(VERSION 3.18) - -project(ccpp_framework - VERSION 5.0.0 - LANGUAGES Fortran) - -include(cmake/ccpp_capgen.cmake) - -#------------------------------------------------------------------------------ -# Set package definitions -set(PACKAGE "ccpp-framework") -string(TIMESTAMP YEAR "%Y") - -option(OPENMP "Enable OpenMP support for the framework" OFF) -option(BUILD_SHARED_LIBS "Build using shared libraries" ON) -option(CCPP_FRAMEWORK_BUILD_DOCUMENTATION - "Create and install the HTML documentation (requires Doxygen)" OFF) -option(CCPP_FRAMEWORK_ENABLE_TESTS "Enable building/running CCPP regression tests" OFF) -option(CCPP_RUN_ADVECTION_TEST "Enable advection regression test" OFF) -option(CCPP_RUN_CAPGEN_TEST "Enable capgen regression test" OFF) -option(CCPP_RUN_DDT_HOST_TEST "Enable ddt host regression test" OFF) -option(CCPP_RUN_VAR_COMPATIBILITY_TEST "Enable variable compatibility regression test" OFF) -option(CCPP_RUN_NESTED_SUITE_TEST "Enable nested suite regression test" OFF) - -message("") -message("OPENMP .............................. ${OPENMP}") -message("BUILD_SHARED_LIBS ................... ${BUILD_SHARED_LIBS}") -message("") -message("CCPP_FRAMEWORK_BUILD_DOCUMENTATION ...${CCPP_FRAMEWORK_BUILD_DOCUMENTATION}") -message("CCPP_FRAMEWORK_ENABLE_TESTS ......... ${CCPP_FRAMEWORK_ENABLE_TESTS}") -message("CCPP_RUN_ADVECTION_TEST ............. ${CCPP_RUN_ADVECTION_TEST}") -message("CCPP_RUN_CAPGEN_TEST ................ ${CCPP_RUN_CAPGEN_TEST}") -message("CCPP_RUN_DDT_HOST_TEST .............. ${CCPP_RUN_DDT_HOST_TEST}") -message("CCPP_RUN_VAR_COMPATIBILITY_TEST ..... ${CCPP_RUN_VAR_COMPATIBILITY_TEST}") -message("CCPP_RUN_NESTED_SUITE_TEST .......... ${CCPP_RUN_NESTED_SUITE_TEST}") -message("") - -set(CCPP_VERBOSITY "0" CACHE STRING "Verbosity level of output (default: 0)") - -# Warn user on conflicting test options -if(CCPP_RUN_ADVECTION_TEST OR - CCPP_RUN_CAPGEN_TEST OR - CCPP_RUN_DDT_HOST_TEST OR - CCPP_RUN_VAR_COMPATIBILITY_TEST) - set(CCPP_MANUALLY_DECLARED_TEST ON BOOL) -endif() -if(CCPP_MANUALLY_DECLARED_TEST AND CCPP_FRAMEWORK_ENABLE_TESTS) - message(WARNING "Detected a manual test flag and the flag to run all tests. If only expected to run a single test, please unset CCPP_FRAMEWORK_ENABLE_TESTS option.") -endif() -set(CCPP_RUNNING_TESTS CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_MANUALLY_DECLARED_TEST) - -# If running tests, set appropriate flags to help with debugging test issues. -if(CCPP_RUNNING_TESTS) - if(${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU") - ADD_COMPILE_OPTIONS(-fcheck=all) - ADD_COMPILE_OPTIONS(-fbacktrace) - ADD_COMPILE_OPTIONS(-ffpe-trap=zero) - ADD_COMPILE_OPTIONS(-finit-real=nan) - ADD_COMPILE_OPTIONS(-ggdb) - ADD_COMPILE_OPTIONS(-ffree-line-length-none) - ADD_COMPILE_OPTIONS(-cpp) - elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel") - ADD_COMPILE_OPTIONS(-fpe0) - ADD_COMPILE_OPTIONS(-warn) - ADD_COMPILE_OPTIONS(-traceback) - ADD_COMPILE_OPTIONS(-debug extended) - ADD_COMPILE_OPTIONS(-fpp) - ADD_COMPILE_OPTIONS(-diag-disable=10448) - elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL "IntelLLVM") - ADD_COMPILE_OPTIONS(-fpe0) - ADD_COMPILE_OPTIONS(-warn) - ADD_COMPILE_OPTIONS(-traceback) - ADD_COMPILE_OPTIONS(-debug full) - ADD_COMPILE_OPTIONS(-fpp) - elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVIDIA" OR ${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVHPC") - ADD_COMPILE_OPTIONS(-Mnoipa) - ADD_COMPILE_OPTIONS(-traceback) - ADD_COMPILE_OPTIONS(-Mfree) - ADD_COMPILE_OPTIONS(-Ktrap=fp) - ADD_COMPILE_OPTIONS(-Mpreprocess) - else() - message (WARNING "This program may not be able to be compiled with compiler :${CMAKE_Fortran_COMPILER_ID}") - endif() -endif() - -# Use rpaths on MacOSX -set(CMAKE_MACOSX_RPATH 1) - -#------------------------------------------------------------------------------ -# Set MPI flags for Fortran with MPI F08 interface -find_package(MPI COMPONENTS Fortran REQUIRED) -if(NOT MPI_Fortran_HAVE_F08_MODULE) - message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") -endif() - -#------------------------------------------------------------------------------ -# Set OpenMP flags for C/C++/Fortran -if(OPENMP) - find_package(OpenMP REQUIRED) -endif() - -#------------------------------------------------------------------------------ -# Set a default build type if none was specified -if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) - message(STATUS "Setting build type to 'Release' as none was specified.") - set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) -endif() - -#------------------------------------------------------------------------------ -# Add the sub-directories -add_subdirectory(src) - -if(CCPP_RUNNING_TESTS) - enable_testing() - add_subdirectory(test) -endif() - -if (CCPP_FRAMEWORK_BUILD_DOCUMENTATION) - find_package(Doxygen REQUIRED) - add_subdirectory(doc) -endif() - diff --git a/LICENSE b/LICENSE index 996dc27b..add90622 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2017, NOAA, UCAR/NCAR CU/CIRES +Copyright 2026, NOAA, NAVY, UCAR/NCAR, CU/CIRES, CSU/CIRA Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 3447598e..11c71e52 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +TODO UPDATE ME + # CCPP Framework This repository contains the Common Community Physics Package (CCPP) Framework: The infrastructure that connects CCPP physics schemes with a host model, as well as stand-alone tools for use with CCPP. diff --git a/cmake/ccpp_capgen.cmake b/cmake/ccpp_capgen.cmake deleted file mode 100644 index 599a28ff..00000000 --- a/cmake/ccpp_capgen.cmake +++ /dev/null @@ -1,137 +0,0 @@ -# CMake wrapper for ccpp_capgen.py -# Currently meant to be a CMake API needed for generating caps for regression tests. -# -# CAPGEN_EXPECT_THROW_ERROR - ON/OFF (Default: OFF) - Scans ccpp_capgen.py log for error string and errors if not found. -# HOST_NAME - String name of host -# OUTPUT_ROOT - String path to put generated caps -# VERBOSITY - Number of --verbose flags to pass to capgen -# HOSTFILES - CMake list of host metadata filenames -# SCHEMEFILES - CMake list of scheme metadata files -# SUITES - CMake list of suite xml files -function(ccpp_capgen) - set(optionalArgs CAPGEN_EXPECT_THROW_ERROR) - set(oneValueArgs HOST_NAME OUTPUT_ROOT VERBOSITY KIND_SPECS) - set(multi_value_keywords HOSTFILES SCHEMEFILES SUITES) - - cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN}) - - # Error if script file not found. - set(CCPP_CAPGEN_CMD_LIST "${CMAKE_SOURCE_DIR}/scripts/ccpp_capgen.py") - if(NOT EXISTS ${CCPP_CAPGEN_CMD_LIST}) - message(FATAL_ERROR "function(ccpp_capgen): Could not find ccpp_capgen.py. Looked for ${CCPP_CAPGEN_CMD_LIST}.") - endif() - - # Interpret parsed arguments - if(DEFINED arg_HOSTFILES) - list(JOIN arg_HOSTFILES "," HOSTFILES_SEPARATED) - list(APPEND CCPP_CAPGEN_CMD_LIST "--host-files" "${HOSTFILES_SEPARATED}") - endif() - if(DEFINED arg_SCHEMEFILES) - list(JOIN arg_SCHEMEFILES "," SCHEMEFILES_SEPARATED) - list(APPEND CCPP_CAPGEN_CMD_LIST "--scheme-files" "${SCHEMEFILES_SEPARATED}") - endif() - if(DEFINED arg_SUITES) - list(JOIN arg_SUITES "," SUITES_SEPARATED) - list(APPEND CCPP_CAPGEN_CMD_LIST "--suites" "${SUITES_SEPARATED}") - endif() - if(DEFINED arg_HOST_NAME) - list(APPEND CCPP_CAPGEN_CMD_LIST "--host-name" "${arg_HOST_NAME}") - endif() - if(DEFINED arg_OUTPUT_ROOT) - message(STATUS "Creating output directory: ${arg_OUTPUT_ROOT}") - file(MAKE_DIRECTORY "${arg_OUTPUT_ROOT}") - list(APPEND CCPP_CAPGEN_CMD_LIST "--output-root" "${arg_OUTPUT_ROOT}") - endif() - if(DEFINED arg_VERBOSITY) - string(REPEAT "--verbose " ${arg_VERBOSITY} VERBOSE_PARAMS_SEPARATED) - separate_arguments(VERBOSE_PARAMS UNIX_COMMAND "${VERBOSE_PARAMS_SEPARATED}") - list(APPEND CCPP_CAPGEN_CMD_LIST ${VERBOSE_PARAMS}) - endif() - if(DEFINED arg_KIND_SPECS) - string(REPLACE "," ";" KIND_SPEC_LIST "${arg_KIND_SPECS}") - set(KIND_ARGS "") # start empty - foreach(pair IN LISTS KIND_SPEC_LIST) - # Append each pair prefixed with --kind-type and quoted. - # The surrounding double‑quotes are added explicitly so the - # resulting string contains them. - set(KIND_ARGS "${KIND_ARGS}--kind-type \"${pair}\"") - string(STRIP "${KIND_ARGS}" KIND_ARGS) - endforeach() - - list(APPEND CCPP_CAPGEN_CMD_LIST ${KIND_SPEC_PARAMS}) - endif() - - message(STATUS "Running ccpp_capgen.py from ${CMAKE_CURRENT_SOURCE_DIR}") - - unset(CAPGEN_OUT) # Unset CAPGEN_OUT to prevent incorrect output on subsequent ccpp_capgen(...) calls. - execute_process(COMMAND ${CCPP_CAPGEN_CMD_LIST} - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - OUTPUT_VARIABLE CAPGEN_OUT - ERROR_VARIABLE CAPGEN_OUT - RESULT_VARIABLE RES - COMMAND_ECHO STDOUT) - - message(STATUS "ccpp-capgen stdout: ${CAPGEN_OUT}") - - if(arg_CAPGEN_EXPECT_THROW_ERROR) - # Determine if the process succeeded but had an expected string in the process log. - string(FIND "${CAPGEN_OUT}" "Variables of type ccpp_constituent_properties_t only allowed in register phase" ERROR_INDEX) - - if (ERROR_INDEX GREATER -1) - message(STATUS "Capgen build produces expected error message.") - else() - message(FATAL_ERROR "CCPP cap generation did not generate expected error. Expected 'Variables of type constituent_properties_t only allowed in register phase.") - endif() - else() - if(RES EQUAL 0) - message(STATUS "ccpp-capgen completed successfully") - else() - message(FATAL_ERROR "CCPP cap generation FAILED: result = ${RES}") - endif() - endif() -endfunction() - -# CMake wrapper for ccpp_datafile.py -# Currently meant to be a CMake API needed for generating caps for regression tests. -# -# DATATABLE - Path to generated datatable.xml file -# REPORT_NAME - String report name to get list of generated files form capgen (typically --ccpp-files) -function(ccpp_datafile) - set(oneValueArgs DATATABLE REPORT_NAME) - cmake_parse_arguments(arg "" "${oneValueArgs}" "" ${ARGN}) - - set(CCPP_DATAFILE_CMD "${CMAKE_SOURCE_DIR}/scripts/ccpp_datafile.py") - - if(NOT EXISTS ${CCPP_DATAFILE_CMD}) - message(FATAL_ERROR "function(ccpp_datafile): Could not find ccpp_datafile.py. Looked for ${CCPP_DATAFILE_CMD}.") - endif() - - if(NOT DEFINED arg_REPORT_NAME) - message(FATAL_ERROR "function(ccpp_datafile): REPORT_NAME not set. Must specify the report to generate to run cpp_datafile.py") - endif() - list(APPEND CCPP_DATAFILE_CMD "${arg_REPORT_NAME}") - - if(NOT DEFINED arg_DATATABLE) - message(FATAL_ERROR "function(ccpp_datafile): DATATABLE not set. A datatable file must be configured to call ccpp_datafile.") - endif() - list(APPEND CCPP_DATAFILE_CMD "${arg_DATATABLE}") - - message(STATUS "Running ccpp_datafile from ${CMAKE_CURRENT_SOURCE_DIR}") - - unset(CCPP_CAPS) # Unset CCPP_CAPS to prevent incorrect output on subsequent ccpp_datafile(...) calls. - execute_process(COMMAND ${CCPP_DATAFILE_CMD} - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - OUTPUT_VARIABLE CCPP_CAPS - RESULT_VARIABLE RES - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_STRIP_TRAILING_WHITESPACE - COMMAND_ECHO STDOUT) - message(STATUS "CCPP_CAPS = ${CCPP_CAPS}") - if(RES EQUAL 0) - message(STATUS "CCPP cap files retrieved") - else() - message(FATAL_ERROR "CCPP cap file retrieval FAILED: result = ${RES}") - endif() - string(REPLACE "," ";" CCPP_CAPS_LIST ${CCPP_CAPS}) # Convert "," separated list from python back to ";" separated list for CMake. - set(CCPP_CAPS_LIST "${CCPP_CAPS_LIST}" PARENT_SCOPE) -endfunction() diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 59386b60..00000000 --- a/pytest.ini +++ /dev/null @@ -1,3 +0,0 @@ -[pytest] -addopts = -ra --ignore=scripts/metadata2html.py --ignore-glob=test/**/test_reports.py - diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index be9f272b..00000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -black -flake8 -pytest diff --git a/run_codee_tmp.sh b/run_codee_tmp.sh deleted file mode 100755 index 4483b1ff..00000000 --- a/run_codee_tmp.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -files=( - "src/ccpp_constituent_prop_mod.F90" - "src/ccpp_hashable.F90" - "src/ccpp_hash_table.F90" - "src/ccpp_scheme_utils.F90" - "src/ccpp_types.F90" -) - -for entry in "${files[@]}"; do - file=${entry} - git checkout origin/develop -- $file - codee format --verbose --on-error force $file - echo "" - echo "-------------------------------------------------" -done diff --git a/schema/suite_v1_0.xsd b/schema/suite_v1_0.xsd deleted file mode 100644 index dfa96cc5..00000000 --- a/schema/suite_v1_0.xsd +++ /dev/null @@ -1,128 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/schema/suite_v2_0.xsd b/schema/suite_v2_0.xsd deleted file mode 100644 index 9a69efa5..00000000 --- a/schema/suite_v2_0.xsd +++ /dev/null @@ -1,125 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/scripts/ccpp_capgen.py b/scripts/ccpp_capgen.py deleted file mode 100755 index ef512c33..00000000 --- a/scripts/ccpp_capgen.py +++ /dev/null @@ -1,768 +0,0 @@ -#!/usr/bin/env python3 - -""" -Create CCPP parameterization caps, host-model interface code, -physics suite runtime code, and CCPP framework documentation. -""" - -# Python library imports -import sys -import os -import logging -import re -# CCPP framework imports -from ccpp_database_obj import CCPPDatabaseObj -from ccpp_datafile import generate_ccpp_datatable -from ccpp_suite import API -from file_utils import check_for_writeable_file, remove_dir, replace_paths -from file_utils import create_file_list, move_modified_files -from file_utils import KINDS_FILENAME, KINDS_MODULE -from fortran_tools import parse_fortran_file, FortranWriter -from framework_env import parse_command_line -from host_cap import write_host_cap -from host_model import HostModel -from metadata_table import parse_metadata_file, register_ddts, SCHEME_HEADER_TYPE -from parse_tools import init_log, set_log_level, context_string -from parse_tools import register_fortran_ddt_name -from parse_tools import CCPPError, ParseInternalError - -## Capture the Framework root -_SCRIPT_PATH = os.path.dirname(__file__) -_FRAMEWORK_ROOT = os.path.abspath(os.path.join(_SCRIPT_PATH, os.pardir)) -_SRC_ROOT = os.path.join(_FRAMEWORK_ROOT, "src") -## Init this now so that all Exceptions can be trapped -_LOGGER = init_log(os.path.basename(__file__)) - -## Recognized Fortran filename extensions -_FORTRAN_FILENAME_EXTENSIONS = ['F90', 'f90', 'F', 'f'] - -## Metadata table types which can have extra variables in Fortran -_EXTRA_VARIABLE_TABLE_TYPES = ['module', 'host', 'ddt'] - -## Metadata table types where order is significant -_ORDERED_TABLE_TYPES = [] - -## CCPP Framework supported DDT types -_CCPP_FRAMEWORK_DDT_TYPES = ["ccpp_hash_table_t", - "ccpp_hashable_t", - "ccpp_hashable_char_t"] - -############################################################################### -def delete_pathnames_from_file(capfile, logger): -############################################################################### - """Remove all the filenames found in , then delete """ - root_path = os.path.dirname(os.path.abspath(capfile)) - success = True - with open(capfile, 'r') as infile: - for line in infile.readlines(): - path = line.strip() - # Skip blank lines and lines which appear to start with a comment. - if path and (path[0] != '#') and (path[0] != '!'): - # Check for an absolute path - if not os.path.isabs(path): - # Assume relative pathnames are relative to pathsfile - path = os.path.normpath(os.path.join(root_path, path)) - # end if - logger.info("Clean: Removing {}".format(path)) - try: - os.remove(path) - except OSError as oserr: - success = False - errmsg = 'Unable to remove {}\n{}' - logger.warning(errmsg.format(path, oserr)) - # end try - # end if (else skip blank or comment line) - # end for - # end with open - logger.info("Clean: Removing {}".format(capfile)) - try: - os.remove(capfile) - except OSError as oserr: - success = False - errmsg = 'Unable to remove {}\n{}' - logger.warning(errmsg.format(capfile, oserr)) - # end try - if success: - logger.info("ccpp_capgen clean successful, exiting") - else: - logger.info("ccpp_capgen clean encountered errors, exiting") - # end if - -############################################################################### -def find_associated_fortran_file(filename, fortran_source_path): -############################################################################### - """Find the Fortran file associated with metadata file, . - Fortran files should be in . - """ - fort_filename = None - lastdot = filename.rfind('.') - if lastdot < 0: - base = os.path.basename(filename + '.') - else: - base = os.path.basename(filename[0:lastdot+1]) - # end if - for extension in _FORTRAN_FILENAME_EXTENSIONS: - test_name = os.path.join(fortran_source_path, base + extension) - if os.path.exists(test_name): - fort_filename = test_name - break - # end if - # end for - if fort_filename is None: - emsg = f"Cannot find Fortran file associated with '{filename}'." - emsg += f"\nfortran_src_path = '{fortran_source_path}'" - raise CCPPError(emsg) - # end if - return fort_filename - -############################################################################### -def create_kinds_file(run_env, output_dir): -############################################################################### - "Create the kinds.F90 file to be used by CCPP schemes and suites" - kinds_filepath = os.path.join(output_dir, KINDS_FILENAME) - if run_env.logger is not None: - msg = 'Writing {} to {}' - run_env.logger.info(msg.format(KINDS_FILENAME, output_dir)) - # end if - kind_types = run_env.kind_types() - with FortranWriter(kinds_filepath, "w", - "kinds for CCPP", KINDS_MODULE) as kindf: - for kind_type in kind_types: - kind_spec = run_env.kind_spec(kind_type) - use_stmt = f"use {run_env.kind_module(kind_type)}," - if kind_spec == kind_type: - use_stmt += f" only: {kind_type}" - else: - use_stmt += f" only: {kind_type} => {kind_spec}" - # end if - kindf.write(use_stmt, 1) - # end for - kindf.write_preamble() - for kind_type in kind_types: - kindf.write("public :: {}".format(kind_type), 1) - # end for - # end with - return kinds_filepath - -############################################################################### -def add_error(error_string, new_error): -############################################################################### - '''Add an error () to , separating errors by a - newline''' - if error_string: - error_string += '\n' - # end if - return error_string + new_error - -############################################################################### -def is_arrayspec(local_name): -############################################################################### - "Return True iff is an array reference" - return '(' in local_name - -############################################################################### -def find_var_in_list(local_name, var_list): -############################################################################### - """Find a variable, , in . - local name is used because Fortran metadata variables do not have - real standard names. - Note: The search is case insensitive. - Return both the variable and the index where it was found. - If not found, return None for the variable and -1 for the index - """ - vvar = None - vind = -1 - lname = local_name.lower() - for lind, lvar in enumerate(var_list): - if lvar.get_prop_value('local_name').lower() == lname: - vvar = lvar - vind = lind - break - # end if - # end for - return vvar, vind - -############################################################################### -def var_comp(prop_name, mvar, fvar, title, case_sensitive=False): -############################################################################### - "Compare a property between two variables" - errors = '' - mprop = mvar.get_prop_value(prop_name) - fprop = fvar.get_prop_value(prop_name) - if not case_sensitive: - if isinstance(mprop, str): - mprop = mprop.lower() - # end if - if isinstance(fprop, str): - fprop = fprop.lower() - # end if - # end if - comp = mprop == fprop - if not comp: - errmsg = '{} mismatch ({} != {}) in {}{}' - ctx = context_string(mvar.context) - errors = add_error(errors, - errmsg.format(prop_name, mprop, fprop, title, ctx)) - # end if - return errors - -############################################################################### -def dims_comp(mheader, mvar, fvar, title, logger, case_sensitive=False): -############################################################################### - "Compare the dimensions attribute of two variables" - errors = '' - mdims = mvar.get_dimensions() - fdims = mheader.convert_dims_to_standard_names(fvar, logger=logger) - comp = len(mdims) == len(fdims) - if not comp: - errmsg = 'Error: rank mismatch in {}/{} ({} != {}){}' - stdname = mvar.get_prop_value('standard_name') - ctx = context_string(mvar.context) - errors = add_error(errors, errmsg.format(title, stdname, - len(mdims), len(fdims), ctx)) - # end if - if comp: - # Now, compare the dims - for dim_ind, mdim in enumerate(mdims): - if ':' in mdim: - mdim = ':'.join([x.strip() for x in mdim.split(':')]) - # end if - fdim = fdims[dim_ind].strip() - if ':' in fdim: - fdim = ':'.join([x.strip() for x in fdim.split(':')]) - # end if - if not case_sensitive: - mdim = mdim.lower() - fdim = fdim.lower() - # end if - # Naked colon is okay for Fortran side - comp = fdim in (':', fdim) - if not comp: - errmsg = 'Error: dim {} mismatch ({} != {}) in {}/{}{}' - stdname = mvar.get_prop_value('standard_name') - ctx = context_string(mvar.context) - errmsg = errmsg.format(dim_ind+1, mdim, fdims[dim_ind], - title, stdname, ctx) - errors = add_error(errors, errmsg) - # end if - # end for - # end if - return errors - -############################################################################### -def compare_fheader_to_mheader(meta_header, fort_header, logger): -############################################################################### - """Compare a metadata header against the header generated from the - corresponding code in the associated Fortran file. - Return a string with any errors found (empty string is no errors). - """ - errors_found = '' - title = meta_header.title - mht = meta_header.header_type - fht = fort_header.header_type - if mht != fht: - # Special case, host metadata can be in a Fortran module or scheme - if (mht != 'host') or (fht not in ('module', SCHEME_HEADER_TYPE)): - errmsg = 'Metadata table type mismatch for {}, {} != {}{}' - ctx = meta_header.start_context() - raise CCPPError(errmsg.format(title, meta_header.header_type, - fort_header.header_type, ctx)) - # end if - else: - # The headers should have the same variables in the same order - # The exception is that a Fortran module can have variable declarations - # after all the metadata variables. - mlist = meta_header.variable_list() - mlen = len(mlist) - flist = fort_header.variable_list() - flen = len(flist) - # Remove array references from mlist before checking lengths - for mvar in mlist: - if is_arrayspec(mvar.get_prop_value('local_name')): - mlen -= 1 - # end if - # end for - list_match = mlen == flen - # Check for optional Fortran variables that are not in metadata - if flen > mlen: - for find, fvar in enumerate(flist): - lname = fvar.get_prop_value('local_name') - _, mind = find_var_in_list(lname, mlist) - if mind < 0: - if fvar.get_prop_value('optional'): - # This is an optional variable - flen -= 1 - # end if - # end if - # end for - list_match = mlen == flen - # end if - if not list_match: - if fht in _EXTRA_VARIABLE_TABLE_TYPES: - if flen > mlen: - list_match = True - else: - etype = 'Fortran {}'.format(fht) - # end if - elif flen > mlen: - etype = 'metadata header' - else: - etype = 'Fortran {}'.format(fht) - # end if - # end if - if not list_match: - errmsg = 'Variable mismatch in {}, variables missing from {}.' - errors_found = add_error(errors_found, errmsg.format(title, etype)) - if etype == "metadata header": - # Look for missing metadata variables - for fvar in flist: - lname = fvar.get_prop_value('local_name') - _, find = find_var_in_list(lname, mlist) - if (find < 0) and (not fvar.get_prop_value('optional')): - errmsg = f"Fortran variable, {lname}, not in metadata" - errors_found = add_error(errors_found, errmsg) - # end if - # end for - # end if - # end if - for mind, mvar in enumerate(mlist): - lname = mvar.get_prop_value('local_name') - mname = mvar.get_prop_value('standard_name') - arrayref = is_arrayspec(lname) - fvar, find = find_var_in_list(lname, flist) - # Check for consistency between optional variables in metadata and - # optional variables in fortran. Error if optional attribute is - # missing from fortran declaration. - # first check: if metadata says the variable is optional, does the fortran match? - mopt = mvar.get_prop_value('optional') - if find and mopt: - fopt = fvar.get_prop_value('optional') - if (not fopt): - errmsg = f'Missing "optional" attribute in fortran declaration for variable {mname}, ' \ - f'for {title}' - errors_found = add_error(errors_found, errmsg) - # end if - # end if - # now check: if fortran says the variable is optional, does the metadata match? - if fvar: - fopt = fvar.get_prop_value('optional') - if (fopt and not mopt): - errmsg = f'Missing "optional" metadata property for variable {mname}, ' \ - f'for {title}' - errors_found = add_error(errors_found, errmsg) - # end if - # end if - if mind >= flen: - if arrayref: - # Array reference, variable not in Fortran table - pass - elif fvar is None: - errmsg = 'No Fortran variable for {} in {}' - errors_found = add_error(errors_found, - errmsg.format(lname, title)) - # end if (no else, we already reported an out-of-place error - # Do not break to collect all missing variables - continue - # end if - # At this point, we should have a Fortran variable - if (not arrayref) and (fvar is None): - errmsg = 'Variable mismatch in {}, no Fortran variable {}.' - errors_found = add_error(errors_found, errmsg.format(title, - lname)) - continue - # end if - # Check order dependence - if fht in _ORDERED_TABLE_TYPES: - if find != mind: - errmsg = 'Out of order argument, {} in {}' - errors_found = add_error(errors_found, - errmsg.format(lname, title)) - continue - # end if - # end if - if arrayref: - # Array reference, do not look for this in Fortran table - continue - # end if - errs = var_comp('local_name', mvar, fvar, title) - if errs: - errors_found = add_error(errors_found, errs) - else: - errs = var_comp('type', mvar, fvar, title) - if errs: - errors_found = add_error(errors_found, errs) - # end if - errs = var_comp('kind', mvar, fvar, title) - if errs: - errors_found = add_error(errors_found, errs) - # end if - if meta_header.header_type == SCHEME_HEADER_TYPE: - errs = var_comp('intent', mvar, fvar, title) - if errs: - errors_found = add_error(errors_found, errs) - # end if - # end if - # Compare dimensions - errs = dims_comp(meta_header, mvar, fvar, title, logger) - if errs: - errors_found = add_error(errors_found, errs) - # end if - # end if - # end for - # end if - return errors_found - -############################################################################### -def check_fortran_against_metadata(meta_headers, fort_headers, - mfilename, ffilename, logger, - fortran_routines=None): -############################################################################### - """Compare a set of metadata headers from against the - code in the associated Fortran file, . - NB: This routine destroys the list, but returns the - contents in an association dictionary on successful completion.""" - header_dict = {} # Associate a Fortran header for every metadata header - for mheader in meta_headers: - fheader = None - mtitle = mheader.title - for findex in range(len(fort_headers)): #pylint: disable=consider-using-enumerate - if fort_headers[findex].title == mtitle: - fheader = fort_headers.pop(findex) - break - # end if - # end for - if fheader is None: - tlist = '\n '.join([x.title for x in fort_headers]) - logger.debug("CCPP routines in {}:{}".format(ffilename, tlist)) - errmsg = "No matching Fortran routine found for {} in {}" - raise CCPPError(errmsg.format(mtitle, ffilename)) - # end if - header_dict[mheader] = fheader - # end if - # end while - if fort_headers: - errmsgs = [] - estr = "No matching metadata header found for {} in {}" - for fheader in fort_headers: - if fheader.has_variables: - errmsgs.append(estr.format(fheader.title, mfilename)) - # end if - # end for - if errmsgs: - mheads = ', '.join([x.name for x in meta_headers]) - errmsgs.append(f'Metadata headers in file: {mheads}') - raise CCPPError('\n'.join(errmsgs)) - # end if - # end if - # We have a one-to-one set, compare headers - errors_found = '' - for mheader in header_dict: - fheader = header_dict[mheader] - errors_found += compare_fheader_to_mheader(mheader, fheader, logger) - # end for - if errors_found: - num_errors = len(re.findall(r'\n', errors_found)) + 1 - errmsg = "{}\n{} error{} found comparing {} to {}" - raise CCPPError(errmsg.format(errors_found, num_errors, - 's' if num_errors > 1 else '', - mfilename, ffilename)) - # end if - # No return, an exception is raised on error - -############################################################################### -def duplicate_item_error(title, filename, itype, orig_item): -############################################################################### - """Raise an error indicating a duplicate item of type, """ - errmsg = "Duplicate {typ}, {title}, found in {file}" - edict = {'title':title, 'file':filename, 'typ':itype} - ofile = orig_item.context.filename - if ofile is not None: - errmsg = errmsg + ", original found in {ofile}" - edict['ofile'] = ofile - # end if - raise CCPPError(errmsg.format(**edict)) - -############################################################################### -def parse_host_model_files(host_filenames, host_name, run_env, - known_ddts=list()): -############################################################################### - """ - Gather information from host files (e.g., DDTs, registry) and - return a host model object with the information. - """ - header_dict = {} - table_dict = {} - logger = run_env.logger - for filename in host_filenames: - logger.info('Reading host model data from {}'.format(filename)) - # parse metadata file - mtables = parse_metadata_file(filename, known_ddts, run_env) - fortran_source_path = mtables[0].fortran_source_path - fort_file = find_associated_fortran_file(filename, fortran_source_path) - ftables, _ = parse_fortran_file(fort_file, run_env) - # Check Fortran against metadata (will raise an exception on error) - mheaders = list() - for sect in [x.sections() for x in mtables]: - mheaders.extend(sect) - # end for - fheaders = list() - for sect in [x.sections() for x in ftables]: - fheaders.extend(sect) - # end for - check_fortran_against_metadata(mheaders, fheaders, - filename, fort_file, logger) - # Check for duplicate tables, then add to dict - for table in mtables: - if table.table_name in table_dict: - duplicate_item_error(table.table_name, filename, - table.table_type, table_dict[header.title]) - else: - table_dict[table.table_name] = table - # end if - # end for - # Check for duplicate headers, then add to dict - for header in mheaders: - if header.title in header_dict: - duplicate_item_error(header.title, filename, - header.header_type, - header_dict[header.title]) - else: - header_dict[header.title] = header - if header.header_type == 'ddt': - known_ddts.append(header.title) - # end if - # end for - # end for - if not host_name: - host_name = None - # end if - host_model = HostModel(table_dict, host_name, run_env) - return host_model - -############################################################################### -def parse_scheme_files(scheme_filenames, run_env, skip_ddt_check=False, - known_ddts=list(), relative_source_path=False): -############################################################################### - """ - Gather information from scheme files (e.g., init, run, and finalize - methods) and return resulting dictionary. - """ - table_dict = {} # Duplicate check and for dependencies processing - header_dict = {} # To check for duplicates - logger = run_env.logger - for filename in scheme_filenames: - logger.info('Reading CCPP schemes from {}'.format(filename)) - # parse metadata file - mtables = parse_metadata_file(filename, known_ddts, run_env, - skip_ddt_check=skip_ddt_check, - relative_source_path=relative_source_path) - fortran_source_path = mtables[0].fortran_source_path - fort_file = find_associated_fortran_file(filename, fortran_source_path) - ftables, additional_routines = parse_fortran_file(fort_file, run_env) - # Check Fortran against metadata (will raise an exception on error) - mheaders = list() - for sect in [x.sections() for x in mtables]: - mheaders.extend(sect) - # end for - fheaders = list() - for sect in [x.sections() for x in ftables]: - fheaders.extend(sect) - # end for - check_fortran_against_metadata(mheaders, fheaders, - filename, fort_file, logger, - fortran_routines=additional_routines) - # Check for duplicate tables, then add to dict - for table in mtables: - if table.table_name in table_dict: - duplicate_item_error(table.table_name, filename, - table.table_type, table_dict[header.title]) - else: - table_dict[table.table_name] = table - # end if - # end for - # Check for duplicate headers, then add to dict - for header in mheaders: - if header.title in header_dict: - duplicate_item_error(header.title, filename, header.header_type, - header_dict[header.title]) - else: - header_dict[header.title] = header - if header.header_type == 'ddt': - known_ddts.append(header.title) - # end if - # end if - # end for - # end for - - return header_dict.values(), table_dict - -############################################################################### -def clean_capgen(cap_output_file, logger): -############################################################################### - """Attempt to remove the files created by the last invocation of capgen""" - log_level = logger.getEffectiveLevel() - set_log_level(logger, logging.INFO) - if os.path.exists(cap_output_file): - logger.info("Cleaning capgen files from {}".format(cap_output_file)) - delete_pathnames_from_file(cap_output_file, logger) - else: - emsg = "Unable to run clean, {} not found" - logger.error(emsg.format(cap_output_file)) - # end if - set_log_level(logger, log_level) - -############################################################################### -def capgen(run_env, return_db=False): -############################################################################### - """Parse indicated host, scheme, and suite files. - Generate code to allow host model to run indicated CCPP suites.""" - ## A few sanity checks - ## Make sure output directory is legit - if os.path.exists(run_env.output_dir): - if not os.path.isdir(run_env.output_dir): - errmsg = "output-root, '{}', is not a directory" - raise CCPPError(errmsg.format(run_env.output_root)) - # end if - if not os.access(run_env.output_dir, os.W_OK): - errmsg = "Cannot write files to output-root ({})" - raise CCPPError(errmsg.format(run_env.output_root)) - # end if (output_dir is okay) - else: - # Try to create output_dir (let it crash if it fails) - os.makedirs(run_env.output_dir) - # end if - # Pre-register base CCPP DDT types: - for ddt_name in _CCPP_FRAMEWORK_DDT_TYPES: - register_fortran_ddt_name(ddt_name) - # end for - src_dir = os.path.join(_FRAMEWORK_ROOT, "src") - host_files = run_env.host_files - host_name = run_env.host_name - scheme_files = run_env.scheme_files - # We need to create three lists of files, hosts, schemes, and SDFs - host_files = create_file_list(run_env.host_files, ['meta'], 'Host', - run_env.logger) - # The host model needs to know about the constituents module - const_mod = os.path.join(_SRC_ROOT, "ccpp_constituent_prop_mod.meta") - if const_mod not in host_files: - host_files.append(const_mod) - # end if - scheme_files = create_file_list(run_env.scheme_files, ['meta'], - 'Scheme', run_env.logger) - sdfs = create_file_list(run_env.suites, ['xml'], 'Suite', run_env.logger) - check_for_writeable_file(run_env.datatable_file, "Cap output datatable") - ##XXgoldyXX: Temporary warning - if run_env.generate_docfiles: - raise CCPPError("--generate-docfiles not yet supported") - # end if - # The host model may depend on suite DDTs - scheme_ddts = register_ddts(scheme_files) - # Handle the host files - host_model = parse_host_model_files(host_files, host_name, run_env, - known_ddts=scheme_ddts) - # Next, parse the scheme files - # We always need to parse the constituent DDTs - const_prop_mod = os.path.join(src_dir, "ccpp_constituent_prop_mod.meta") - if const_prop_mod not in scheme_files: - scheme_files = [const_prop_mod] + scheme_files - # end if - host_ddts = register_ddts(host_files) - scheme_headers, scheme_tdict = parse_scheme_files(scheme_files, run_env, - known_ddts=host_ddts) - if run_env.verbose: - ddts = host_model.ddt_lib.keys() - if ddts: - run_env.logger.debug("DDT definitions = {}".format(ddts)) - # end if - # end if - plist = host_model.prop_list('local_name') - if run_env.verbose: - run_env.logger.debug("{} variables = {}".format(host_model.name, plist)) - run_env.logger.debug("schemes = {}".format([x.title - for x in scheme_headers])) - # Finally, we can get on with writing suites - # Make sure to write to temporary location if files exist in - if not os.path.exists(run_env.output_dir): - # Try to create output_dir (let it crash if it fails) - os.makedirs(run_env.output_dir) - # Nothing here, use it for output - outtemp_dir = run_env.output_dir - elif not os.listdir(run_env.output_dir): - # Nothing here, use it for output - outtemp_dir = run_env.output_dir - else: - # We need to create a temporary staging area, create it here - outtemp_name = "ccpp_temp_scratch_dir" - outtemp_dir = os.path.join(run_env.output_dir, outtemp_name) - if os.path.exists(outtemp_dir): - remove_dir(outtemp_dir, force=True) - # end if - os.makedirs(outtemp_dir) - # end if - ccpp_api = API(sdfs, host_model, scheme_headers, run_env) - cap_filenames = ccpp_api.write(outtemp_dir, run_env) - if run_env.generate_host_cap: - # Create a cap file - cap_module = host_model.ccpp_cap_name() - host_files = [write_host_cap(host_model, ccpp_api, cap_module, - outtemp_dir, run_env)] - else: - host_files = list() - # end if - # Create the kinds file - kinds_file = create_kinds_file(run_env, outtemp_dir) - # Move any changed files to output_dir and remove outtemp_dir - move_modified_files(outtemp_dir, run_env.output_dir, - overwrite=run_env.force_overwrite, remove_src=True) - # We have to rename the files we created - if outtemp_dir != run_env.output_dir: - replace_paths(cap_filenames, outtemp_dir, run_env.output_dir) - replace_paths(host_files, outtemp_dir, run_env.output_dir) - kinds_file = kinds_file.replace(outtemp_dir, run_env.output_dir) - # end if - # Finally, create the database of generated files and caps - # This can be directly in output_dir because it will not affect dependencies - generate_ccpp_datatable(run_env, host_model, ccpp_api, - scheme_headers, scheme_tdict, host_files, - cap_filenames, kinds_file, src_dir) - if return_db: - return CCPPDatabaseObj(run_env, host_model=host_model, api=ccpp_api) - # end if - return None - -############################################################################### -def _main_func(): -############################################################################### - """Parse command line, then parse indicated host, scheme, and suite files. - Finally, generate code to allow host model to run indicated CCPP suites.""" - framework_env = parse_command_line(sys.argv[1:], __doc__, logger=_LOGGER) - if framework_env.verbosity > 1: - set_log_level(framework_env.logger, logging.DEBUG) - elif framework_env.verbosity > 0: - set_log_level(framework_env.logger, logging.INFO) - # end if - if framework_env.clean: - clean_capgen(framework_env.datatable_file, framework_env.logger) - else: - _ = capgen(framework_env) - # end if (clean) - -############################################################################### - -if __name__ == "__main__": - try: - _main_func() - sys.exit(0) - except ParseInternalError as pie: - _LOGGER.exception(pie) - sys.exit(-1) - except CCPPError as ccpp_err: - if _LOGGER.getEffectiveLevel() <= logging.DEBUG: - _LOGGER.exception(ccpp_err) - else: - _LOGGER.error(ccpp_err) - # end if - sys.exit(1) - finally: - logging.shutdown() - # end try diff --git a/scripts/ccpp_database_obj.py b/scripts/ccpp_database_obj.py deleted file mode 100644 index 24579750..00000000 --- a/scripts/ccpp_database_obj.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 - -""" -Define the CCPPDatabaseObj object -Object definition and methods to provide information from a run of capgen. -""" - -from host_model import HostModel -from ccpp_suite import API - -class CCPPDatabaseObjError(ValueError): - """Error class specific to CCPPDatabaseObj. - All uses of this error should be internal (i.e., programmer error, - not user error).""" - - def __init__(self, message): - """Initialize this exception""" - super().__init__(message) - -class CCPPDatabaseObj: - """Object with data and methods to provide information from a run of capgen. - """ - - def __init__(self, run_env, host_model=None, api=None, database_file=None): - """Initialize this CCPPDatabaseObj. - If is not None, all other inputs MUST be None and - the object is created from the database table created by capgen. - To initialize the object from an in-memory capgen run, ALL other - inputs MUST be passed (i.e., not None) and it is an error to pass - a value for . - """ - - runtime_obj = all([host_model is not None, api is not None]) - self.__host_model = None - self.__api = None - self.__database_file = None - if runtime_obj and database_file: - emsg = "Cannot provide both runtime arguments and database_file." - elif (not runtime_obj) and (not database_file): - emsg = "Must provide either database_file or all runtime arguments." - else: - emsg = "" - # end if - if emsg: - raise CCPPDatabaseObjError(f"ERROR: {emsg}") - # end if - if runtime_obj: - self.__host_model = host_model - self.__api = api - else: - self.db_from_file(run_env, database_file) - # end if - - def db_from_file(self, run_env, database_file): - """Create the necessary internal data structures from a CCPP - datatable.xml file created by capgen. - """ - metadata_tables = {} - host_name = "host" - self.__host_model = HostModel(metadata_tables, host_name, run_env) - self.__api = API(sdfs, host_model, scheme_headers, run_env) - raise CCPPDatabaseObjError("ERROR: not supported") - - def host_model_dict(self): - """Return the host model dictionary for this CCPP DB object""" - if self.__host_model is not None: - return self.__host_model - # end if - raise CCPPDatabaseObjError("ERROR: not supported") - - def suite_list(self): - """Return a list of suites built into the API""" - if self.__api is not None: - return list(self.__api.suites) - # end if - raise CCPPDatabaseObjError("ERROR: not supported") - - def constituent_dictionary(self, suite): - """Return the constituent dictionary for """ - return suite.constituent_dictionary() - - def call_list(self, phase): - """Return the API call list for """ - if self.__api is not None: - return self.__api.call_list(phase) - # end if - raise CCPPDatabaseObjError("ERROR: not supported") diff --git a/scripts/ccpp_datafile.py b/scripts/ccpp_datafile.py deleted file mode 100755 index 2c546528..00000000 --- a/scripts/ccpp_datafile.py +++ /dev/null @@ -1,1214 +0,0 @@ -#!/usr/bin/env python3 - -"""Code to generate and query the CCPP datafile returned by capgen. -The CCPP datafile is a database consisting of several tables: -- A list of all generated files, broken into groups for host cap, - suite caps, and ccpp_kinds. -- A list of scheme entries, keyed by scheme name -- A list of CCPP metadata files actually used by capgen, broken into groups - for host-model metadata and scheme metadata. These filenames may serve - as keys -- A list of variable entries, keyed by standard name. -""" - -## NB: A new report must be added in two places: -## 1) In the list of DatatableReport._valid_reports -## 2) As an option in datatable_report - -# Python library imports -import argparse -import logging -import os -import sys -import xml.etree.ElementTree as ET -# CCPP framework imports -from framework_env import CCPPFrameworkEnv -from metadata_table import UNKNOWN_PROCESS_TYPE -from metavar import Var -from parse_tools import read_xml_file, write_xml_file -from parse_tools import ParseContext, ParseSource -from suite_objects import Subcycle - -# Global data -_INDENT_STR = " " - -# Used for creating template variables -_MVAR_DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -## datatable_report must have an action for each report type -_VALID_REPORTS = [{"report" : "host_files", "type" : bool, - "help" : - "Return a list of host CAP files created by capgen"}, - {"report" : "suite_files", "type" : bool, - "help" : - "Return a list of suite CAP files created by capgen"}, - {"report" : "utility_files", "type" : bool, - "help" : ("Return a list of utility files created by " - "capgen (e.g., ccpp_kinds.F90)")}, - {"report" : "ccpp_files", "type" : bool, - "help" : "Return a list of all files created by capgen"}, - {"report" : "process_list", "type" : bool, - "help" : ("Return a list of process types and implementing " - "scheme name")}, - {"report" : "module_list", "type" : bool, - "help" : - "Return a list of module names used in this set of suites"}, - {"report" : "dependencies", "type" : bool, - "help" : ("Return a list of scheme and host " - "dependency module names")}, - {"report" : "suite_list", "type" : bool, - "help" : "Return a list of configured suite names"}, - {"report" : "required_variables", "type" : str, - "help" : ("Return a list of required variable " - "standard names for suite, "), - "metavar" : "SUITE_NAME"}, - {"report" : "input_variables", "type" : str, - "help" : ("Return a list of required input variable " - "standard names for suite, "), - "metavar" : "SUITE_NAME"}, - {"report" : "output_variables", "type" : str, - "help" : ("Return a list of required output variable " - "standard names for suite, "), - "metavar" : "SUITE_NAME"}, - {"report" : "host_variables", "type" : bool, - "help" : ("Return a list of required host model variable " - "standard names")}, - {"report" : "show", "type" : bool, - "help" : - "Pretty print the database contents to the screen"}] - -### -### Utilities -### - -class CCPPDatatableError(ValueError): - """Error specific to errors found in the CCPP capgen datafile""" - pass - -class DatatableInternalError(ValueError): - """Error class for reporting internal errors""" - def __init__(self, message): - """Initialize this exception""" - logging.shutdown() - super(DatatableInternalError, self).__init__(message) - -class DatatableReport(object): - """A class to hold a database report type and inquiry function""" - - __valid_actions = [x["report"] for x in _VALID_REPORTS] - - def __init__(self, action, value=True): - """Initialize this report as report-type, - # Test a valid action - >>> DatatableReport('input_variables', False).action - 'input_variables' - - # Test an invalid action - >>> DatatableReport('banana', True).value - Traceback (most recent call last): - ... - ValueError: Invalid action, 'banana' - - """ - if action in DatatableReport.__valid_actions: - self.__action = action - self.__value = value - else: - raise ValueError("Invalid action, '{}'".format(action)) - # end if - - def action_is(self, action): - """If matches this report type, return True. - Otherwise, return False - >>> DatatableReport('suite_files', False).action_is('suite_files') - True - - >>> DatatableReport('suite_files', False).action_is('banana') - False - """ - return action == self.__action - - @property - def action(self): - """Return this action's action""" - return self.__action - - @property - def value(self): - """Return this action's value""" - return self.__value - - @classmethod - def valid_actions(cls): - """Return the list of valid actions for this class""" - return cls.__valid_actions - -### -### Interface for retrieving datatable information -### - -############################################################################### -def _command_line_parser(): -############################################################################### - "Create and return an ArgumentParser for parsing the command line" - description = """ - Retrieve information about a ccpp_capgen run. - The returned information is controlled by selecting an action from - the list of optional arguments below. - Note that exactly one action is required. - """ - parser = argparse.ArgumentParser(description=description) - parser.add_argument("datatable", type=str, - help="Path to a data table XML file created by capgen") - ### Only one action per call - group = parser.add_mutually_exclusive_group(required=True) - for report in _VALID_REPORTS: - rep_type = "--{}".format(report["report"].replace("_", "-")) - if report["type"] is bool: - group.add_argument(rep_type, action='store_true', default=False, - help=report["help"]) - elif report["type"] is str: - if "metavar" in report: - group.add_argument(rep_type, required=False, type=str, - metavar=report["metavar"], default='', - help=report["help"]) - else: - group.add_argument(rep_type, required=False, type=str, - default='', help=report["help"]) - # end if - else: - raise ValueError("Unknown report type, '{}'".format(report["type"])) - # end if - # end for - ### - defval = "," - help_str = "String to separate items in a list (default: '{}')" - parser.add_argument("--separator", type=str, required=False, default=defval, - metavar="SEP", dest="sep", help=help_str.format(defval)) - defval = False - help_str = ("Exclude protected variables (only has an effect if the " - "requested report is returning a list of variables)." - " (default: {})") - parser.add_argument("--exclude-protected", action='store_true', - required=False, - default=defval, help=help_str.format(defval)) - defval = -1 - help_str = ("Screen width for '--show' line wrapping. -1 means do not " - "wrap. (default: {})") - parser.add_argument("--line-wrap", type=int, required=False, - metavar="LINE_WIDTH", dest="line_wrap", - default=defval, help=help_str.format(defval)) - defval = 2 - help_str = "Indent depth for '--show' output (default: {})" - parser.add_argument("--indent", type=int, required=False, default=2, - help=help_str.format(defval)) - return parser - -############################################################################### -def parse_command_line(args): -############################################################################### - """Create an ArgumentParser to parse and return command-line arguments""" - parser = _command_line_parser() - pargs = parser.parse_args(args) - return pargs - -### -### Accessor functions to retrieve information from a datatable file -### - -############################################################################### -def _read_datatable(datatable): -############################################################################### - """Read the XML file, and return its root node""" - _, datatable = read_xml_file(datatable, None) # No logger - return datatable - -############################################################################### -def _find_table_section(table, elem_type): -############################################################################### - """Look for and return an element type, , in . - Raise an exception if the element is not found. - # Test present section - >>> table = ET.fromstring("") - >>> _find_table_section(table, "ccpp_files").tag - 'ccpp_files' - - # Test missing section - >>> table = ET.fromstring("") - >>> _find_table_section(table, "ccpp_files").tag - Traceback (most recent call last): - ... - ccpp_datafile.CCPPDatatableError: Element type, 'ccpp_files', not found in table - """ - found = table.find(elem_type) - if found is None: - emsg = "Element type, '{}', not found in table" - raise CCPPDatatableError(emsg.format(elem_type)) - # end if - return found - -############################################################################### -def _retrieve_ccpp_files(table, file_type=None): -############################################################################### - """Find and retrieve a list of generated filenames from
. - If is not None, only return that file type. - # Test valid ccpp files - >>> table = ET.fromstring(""\ - "/path/to/file1"\ - "/path/to/file2"\ - "/path/to/file3"\ - "/path/to/file4"\ - "") - >>> _retrieve_ccpp_files(table) - ['/path/to/file1', '/path/to/file2', '/path/to/file3', '/path/to/file4'] - - # Test invalid file type - >>> table = ET.fromstring(""\ - "/path/to/file1"\ - "") - >>> _retrieve_ccpp_files(table) - Traceback (most recent call last): - ... - ccpp_datafile.CCPPDatatableError: Invalid file list entry type, 'banana' - """ - ccpp_files = list() - # Find the files section - for section in _find_table_section(table, "ccpp_files"): - if (not file_type) or (section.tag == file_type): - for entry in section: - if entry.tag == "file": - ccpp_files.append(entry.text) - else: - emsg = "Invalid file list entry type, '{}'" - raise CCPPDatatableError(emsg.format(entry.tag)) - # end if - # end for - # end if - # end if - return ccpp_files - -############################################################################### -def _retrieve_process_list(table): -############################################################################### - """Find and return a list of all physics scheme processes in
. - # Test valid module - >>> table = ET.fromstring(""\ - ""\ - ""\ - "") - >>> _retrieve_process_list(table) - ['four=scheme2', 'three=scheme1'] - - # Test no schemes element - >>> table = ET.fromstring("") - >>> _retrieve_process_list(table) - Traceback (most recent call last): - ... - ccpp_datafile.CCPPDatatableError: Could not find 'schemes' element - """ - - result = list() - schemes = table.find("schemes") - if schemes is None: - raise CCPPDatatableError("Could not find 'schemes' element") - # end if - for scheme in schemes: - name = scheme.get("name") - proc = scheme.get("process") - if proc: - result.append("{}={}".format(proc, name)) - # end if - # end for - return sorted(result) - -############################################################################### -def _retrieve_module_list(table): -############################################################################### - """Find and return a list of all scheme modules in
. - # Test valid module - >>> table = ET.fromstring(""\ - ""\ - ""\ - "") - >>> _retrieve_module_list(table) - ['partridge', 'turtle_dove'] - - # Test no schemes element - >>> table = ET.fromstring("") - >>> _retrieve_module_list(table) - Traceback (most recent call last): - ... - ccpp_datafile.CCPPDatatableError: Could not find 'schemes' element - - """ - result = set() - schemes = table.find("schemes") - if schemes is None: - raise CCPPDatatableError("Could not find 'schemes' element") - # end if - for scheme in schemes: - for phase in scheme: - module = phase.get("module") - if module is not None: - result.add(module) - # end if - # end for - # end for - return sorted(result) - -############################################################################### -def _retrieve_dependencies(table): -############################################################################### - """Find and return a list of all host and scheme dependencies. - # Test valid dependencies - >>> table = ET.fromstring("" \ - "bananaorange" \ - "") - >>> _retrieve_dependencies(table) - ['banana', 'orange'] - - # Test no dependencies - >>> table = ET.fromstring("" \ - "") - >>> _retrieve_dependencies(table) - [] - - # Test missing dependencies tag - >>> table = ET.fromstring("") - >>> _retrieve_dependencies(table) - Traceback (most recent call last): - ... - ccpp_datafile.CCPPDatatableError: Could not find 'dependencies' element - """ - - result = set() - depends = table.find("dependencies") - if depends is None: - raise CCPPDatatableError("Could not find 'dependencies' element") - # end if - for dependency in depends: - dep_file = dependency.text - if dep_file is not None: - result.add(dep_file) - # end if - # end for - return sorted(result) - -############################################################################### -def _find_var_dictionary(table, dict_name=None, dict_type=None): -############################################################################### - """Find and return a var_dictionary named, in
. - If not found, return None - # Test valid table with dict_name provided - >>> table = ET.fromstring(""\ - ""\ - ""\ - ""\ - "") - >>> _find_var_dictionary(table, dict_name='orange').get("name") - 'orange' - - # Test valid table with dict_type provided - >>> _find_var_dictionary(table, dict_type='host').get("name") - 'banana' - - # Test valid table with both dict_type and dict_name provided - >>> _find_var_dictionary(table, dict_type='host', dict_name='banana').get("name") - 'banana' - - # Test no table found (expect None) - >>> _find_var_dictionary(table, dict_name='apple') is None - True - - # Test error handling - >>> _find_var_dictionary(table) - Traceback (most recent call last): - ... - ValueError: At least one of or must contain a string - """ - var_dicts = table.find("var_dictionaries") - target_dict = None - if (dict_name is None) and (dict_type is None): - raise ValueError(("At least one of or must " - "contain a string")) - # end if - for vdict in var_dicts: - if (((dict_name is None) or (vdict.get("name") == dict_name)) and - ((dict_type is None) or (vdict.get("type") == dict_type))): - target_dict = vdict - break - # end if - # end for - return target_dict - -############################################################################### -def _retrieve_suite_list(table): -############################################################################### - """Find and return a list of all suites found in
. - # Test suites are found - >>> table = ET.fromstring(""\ - ""\ - "") - >>> _retrieve_suite_list(table) - ['umbrella', 'galoshes'] - - # Test suites not found - >>> table = ET.fromstring("") - >>> _retrieve_suite_list(table) - [] - """ - result = list() - # First, find the API variable dictionary - api_elem = table.find("api") - if api_elem is not None: - suites_elem = api_elem.find("suites") - if suites_elem is not None: - for suite in suites_elem: - result.append(suite.get("name")) - # end for - # end if - # end if - return result - -############################################################################### -def _retrieve_suite_group_names(table, suite_name): -############################################################################### - """Find and return a list of the group names for this suite. - # Test suites are found - >>> table = ET.fromstring(""\ - ""\ - ""\ - ""\ - "") - >>> _retrieve_suite_group_names(table, 'umbrella') - ['florence', 'delores', 'edna'] - - # Test non-present suite - >>> _retrieve_suite_group_names(table, 'poncho') - [] - """ - - result = list() - # First, find the API variable dictionary - api_elem = table.find("api") - if api_elem is not None: - suites_elem = api_elem.find("suites") - if suites_elem is not None: - for suite in suites_elem: - if suite.get("name") == suite_name: - for item in suite: - if item.tag == "group": - result.append(item.get("name")) - # end if - # end for - # end if - # end for - # end if - # end if - return result - -############################################################################### -def _is_variable_protected(table, var_name, var_dict): -############################################################################### - """Determine whether variable, , from is protected. - So this by checking for the 'protected' attribute for in - or any of 's ancestors (parent dictionaries). - # Test found variable - >>> table = ET.fromstring(""\ - ""\ - ""\ - ""\ - "") - >>> var_dict = _find_var_dictionary(table, dict_name="banana") - >>> _is_variable_protected(table, "hi", var_dict) - True - - >>> var_dict = _find_var_dictionary(table, dict_type="api") - >>> _is_variable_protected(table, "hello", var_dict) - False - - # Test non-present variable also returns False - >>> _is_variable_protected(table, "hiya", var_dict) - False - """ - protected = False - while (not protected) and (var_dict is not None): - dvars = var_dict.find("variables") - if dvars is not None: - for var in dvars: - if var.get("name") == var_name: - protected = var.get("protected", default="False") == "True" - break - # end if - # end for - # end if - parent = var_dict.get("parent") - if parent is not None: - var_dict = _find_var_dictionary(table, dict_name=parent) - else: - var_dict = None - # end if - # end while - return protected - -############################################################################### -def _retrieve_variable_list(table, suite_name, - intent_type=None, exclude_protected=True): -############################################################################### - """Find and return a list of all the required variables in . - If suite, , is not found in
, return an empty list. - If is present, return only that variable type (input or - output). - If is True, do not include protected variables - >>> table = ET.fromstring(""\ - ""\ - ""\ - ""\ - ""\ - ""\ - "") - - # Test group variable retrieval - >>> _retrieve_variable_list(table, 'fruit', exclude_protected=False) - ['var3', 'var4', 'var5'] - - >>> _retrieve_variable_list(table, 'fruit') - ['var3'] - - >>> _retrieve_variable_list(table, 'fruit', intent_type='input', exclude_protected=False) - ['var3', 'var5'] - - >>> _retrieve_variable_list(table, 'fruit', intent_type='output', exclude_protected=False) - ['var4', 'var5'] - - >>> _retrieve_variable_list(table, 'fruit', intent_type='input') - ['var3'] - - >>> _retrieve_variable_list(table, 'fruit', intent_type='output') - [] - - # Test host variable retrieval - >>> _retrieve_variable_list(table, 'fruit', intent_type='host', exclude_protected=False) - ['var1', 'var2'] - - >>> _retrieve_variable_list(table, 'fruit', intent_type='host') - ['var1'] - - # Test invalid intent type - >>> _retrieve_variable_list(table, 'fruit', intent_type='banana') - Traceback (most recent call last): - ... - ccpp_datafile.CCPPDatatableError: Invalid intent_type, 'banana' - - """ - # Note that suites do not have call lists so we have to collect - # all the variables from the suite's groups. - var_set = set() - excl_vars = list() - if intent_type == "host": - allowed_intents = list() - elif intent_type is None: - allowed_intents = ['in', 'out', 'inout'] - elif intent_type == "input": - allowed_intents = ['in', 'inout'] - elif intent_type == "output": - allowed_intents = ['out', 'inout'] - else: - emsg = "Invalid intent_type, '{}'" - raise CCPPDatatableError(emsg.format(intent_type)) - # end if - if exclude_protected or (intent_type == "host"): - host_dict = _find_var_dictionary(table, dict_type="host") - if host_dict is not None: - hvars = host_dict.find("variables") - if hvars is not None: - for var in hvars: - vname = var.get("name") - if exclude_protected: - exclude = _is_variable_protected(table, vname, - host_dict) - else: - exclude = False - # end if - if intent_type == "host": - if not exclude: - # Add to host variable set - var_set.add(vname) - # end if - else: - if exclude: - # Add to list of protected variables - excl_vars.append(vname) - # end if - # end if - # end for - # end if - # end if - # end if - if intent_type != "host": - group_names = _retrieve_suite_group_names(table, suite_name) - for group in group_names: - cl_name = group + "_call_list" - group_dict = _find_var_dictionary(table, dict_name=cl_name, - dict_type="group_call_list") - if group_dict is not None: - gvars = group_dict.find("variables") - if gvars is not None: - for var in gvars: - vname = var.get("name") - vintent = var.get("intent") - if exclude_protected: - exclude = vname in excl_vars - if not exclude: - exclude = _is_variable_protected(table, vname, - group_dict) - # end if - else: - exclude = False - # end if - if (vintent in allowed_intents) and (not exclude): - var_set.add(vname) - # end if - # end for - # end if - # end if - # end for - # end if - return sorted(var_set) - -############################################################################### -def datatable_report(datatable, action, sep, exclude_protected=False): -############################################################################### - """Perform a lookup on and return the result. - """ - if not action: - emsg = "datatable_report: An action is required\n" - emsg += _command_line_parser().format_usage() - raise ValueError(emsg) - # end if - if not sep: - emsg = "datatable_report: A separator character () is required\n" - emsg += _command_line_parser().format_usage() - raise ValueError(emsg) - # end if - table = _read_datatable(datatable) - if action.action_is("ccpp_files"): - result = _retrieve_ccpp_files(table) - elif action.action_is("host_files"): - result = _retrieve_ccpp_files(table, file_type="host_files") - elif action.action_is("suite_files"): - result = _retrieve_ccpp_files(table, file_type="suite_files") - elif action.action_is("utility_files"): - result = _retrieve_ccpp_files(table, file_type="utilities") - elif action.action_is("process_list"): - result = _retrieve_process_list(table) - elif action.action_is("module_list"): - result = _retrieve_module_list(table) - elif action.action_is("dependencies"): - result = _retrieve_dependencies(table) - elif action.action_is("suite_list"): - result = _retrieve_suite_list(table) - elif action.action_is("required_variables"): - result = _retrieve_variable_list(table, action.value, - exclude_protected=exclude_protected) - elif action.action_is("input_variables"): - result = _retrieve_variable_list(table, action.value, - intent_type="input", - exclude_protected=exclude_protected) - elif action.action_is("output_variables"): - result = _retrieve_variable_list(table, action.value, - intent_type="output", - exclude_protected=exclude_protected) - elif action.action_is("host_variables"): - result = _retrieve_variable_list(table, "host", exclude_protected=exclude_protected, - intent_type="host") - else: - result = '' - # end if - if isinstance(result, list): - result = sep.join(result) - # end if - return result - -############################################################################### -def _indent_str(indent): -############################################################################### - """Return the line start string for indent level, .""" - return _INDENT_STR*indent - -############################################################################### -def _format_line(line_in, indent, line_wrap, increase_indent=True): -############################################################################### - """Format into separate lines in an attempt to not have the - length of any line greater than characters including any - indent (with indent level specified by ). - If is True, increase the indent level for new lines - created by the process. - A value of less one means do not wrap the line. - >>> line = "This is a very long string that should be wrapped hopefully" - >>> _format_line(line, 1, 50) - ' This is a very long string that should be\\n wrapped hopefully\\n' - - >>> _format_line(line, 1, 50, increase_indent=False) - ' This is a very long string that should be\\n wrapped hopefully\\n' - - >>> _format_line(line, 0, 50) - 'This is a very long string that should be wrapped\\n hopefully\\n' - - >>> line = 'short line' - >>> _format_line(line, 0, 2, increase_indent=False) - 'short\\nline\\n' - """ - in_squote = False - in_dquote = False - outline = '' - indent_str = _indent_str(indent) - curr_indent = len(indent_str) - wrap_points = list() - line = line_in.strip() - llen = len(line) - # Do we need to wrap the line? - if (line_wrap <= 0) or (llen + curr_indent <= line_wrap): - index = llen + 1 - else: - index = 0 - # end if - # Collect possible wrap points - while index < llen: - inchar = line[index] - if in_squote: - if inchar == "'": - in_squote = False - # end if (else do nothing) - elif in_dquote: - if inchar == '"': - in_dquote = False - # end if (else do nothing) - elif inchar == ' ': - wrap_points.append(index + curr_indent) - # end if (else it is not an interesting character) - index += 1 - # end while - if (line_wrap <= 0) or (llen + curr_indent <= line_wrap): - this_line = indent_str + line - next_line = "" - else: - # Find the best break point - good_points = [x for x in wrap_points if x <= line_wrap] - if increase_indent: - indent += 2 # To indent past child tags - # end if - if good_points: - wrap = max(good_points) - curr_indent - this_line = indent_str + line[0:wrap] - next_line = _format_line(line[wrap+1:], indent, line_wrap, - increase_indent=False) - elif wrap_points: - wrap = min(wrap_points) - curr_indent - this_line = indent_str + line[0:wrap] - next_line = _format_line(line[wrap+1:], indent, line_wrap, - increase_indent=False) - else: - this_line = indent_str + line - next_line = "" - # end if - # end if - outline = this_line + '\n' + next_line - return outline - -############################################################################### -def table_entry_pretty_print(entry, indent, line_wrap=-1): -############################################################################### - """Create and return a pretty print string of the contents of - >>> table = ET.fromstring("") - >>> table_entry_pretty_print(table, 0) - '\\n \\n\\n' - - >>> table_entry_pretty_print(table, 1, line_wrap=20) - ' \\n \\n \\n' - """ - output = "" - outline = "<{}".format(entry.tag) - for name in entry.attrib: - outline += " {}={}".format(name, entry.attrib[name]) - # end for - has_children = len(list(entry)) > 0 - has_text = entry.text - if has_children or has_text: - # We have sub-structure, close and print this tag - outline += ">" - output += _format_line(outline, indent, line_wrap) - else: - # No sub-structure, we are done with this tag - outline += " />" - output += _format_line(outline, indent, line_wrap) - # end if - if has_children: - for child in entry: - output += table_entry_pretty_print(child, indent+1, - line_wrap=line_wrap) - # end for - # end if - if has_text: - output += _format_line(entry.text, indent+1, line_wrap) - # end if - if has_children or has_text: - # We had sub-structure, print the close tag - outline = "".format(entry.tag) - output = output.rstrip() + '\n' + _format_line(outline, - indent, line_wrap) - # end if - return output - -############################################################################### -def datatable_pretty_print(datatable, indent, line_wrap): -############################################################################### - """Create and return a pretty print string of the contents of """ - indent = 0 - table = _read_datatable(datatable) - report = table_entry_pretty_print(table, indent, line_wrap=line_wrap) - return report - -### -### Functions to create the datatable file -### - -############################################################################### -def _object_type(pobj): -############################################################################### - """Return an XML-acceptable string for the type of .""" - return pobj.__class__.__name__.lower() - -############################################################################### -def _new_var_entry(parent, var, full_entry=True): -############################################################################### - """Create a variable sub-element of with information from . - If is False, only include standard name and intent. - >>> parent = ET.fromstring('') - >>> var = Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(horizontal_loop_extent)', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'DDT', ParseContext()), _MVAR_DUMMY_RUN_ENV) - >>> _new_var_entry(parent, var) - >>> table_entry_pretty_print(parent, 0) - '\\n \\n \\n horizontal_loop_extent\\n \\n \\n ddt\\n \\n \\n vname\\n \\n \\n\\n' - - >>> parent = ET.fromstring('') - >>> _new_var_entry(parent, var, full_entry=False) - >>> table_entry_pretty_print(parent, 0) - '\\n \\n\\n' - """ - prop_list = ["intent", "local_name"] - if full_entry: - prop_list.extend(["allocatable", "active", "default_value", - "diagnostic_name", "diagnostic_name_fixed", - "kind", "persistence", "polymorphic", "protected", - "state_variable", "type", "units", "molar_mass", - "advected", "top_at_one", "optional"]) - prop_list.extend(Var.constituent_property_names()) - # end if - ventry = ET.SubElement(parent, "var") - ventry.set("name", var.get_prop_value("standard_name")) - for prop in prop_list: - value = var.get_prop_value(prop) - if value: - ventry.set(prop, str(value)) - # end if - # end for - if full_entry: - dims = var.get_dimensions() - if dims: - v_entry = ET.SubElement(ventry, "dimensions") - v_entry.text = " ".join(dims) - # end if - v_entry = ET.SubElement(ventry, "source_type") - v_entry.text = var.source.ptype.lower() - v_entry = ET.SubElement(ventry, "source_name") - v_entry.text = var.source.name.lower() - # end if - -############################################################################### -def _new_scheme_entry(parent, scheme, group_name, scheme_headers): -############################################################################### - """Create a new XML entry for under """ - sch_name = scheme.name - sch_entry = parent.find(sch_name) - process = None - if not sch_entry: - sch_entry = ET.SubElement(parent, "scheme") - sch_entry.set("name", sch_name) - # end if - if scheme.run_phase(): - sch_tag = group_name - else: - sch_tag = scheme.phase() - # end if - if not sch_tag: - emsg = "No phase info for scheme, '{}', group = '{}" - raise CCPPDatatableError(emsg.format(sch_name, group_name)) - # end if - phase_entry = sch_entry.find(sch_tag) - if phase_entry: - pname = phase_entry.get("name") - if pname != sch_name: - emsg = "Scheme entry already exists for {} but name is {}" - raise CCPPDatatableError(emsg.format(sch_name, pname)) - # end if - # Special case: Scheme w/o run phase. - if not scheme._has_run_phase: - return - else: - phase_entry = ET.SubElement(sch_entry, sch_tag) - phase_entry.set("name", sch_name) - title = scheme.subroutine_name - phase_entry.set("subroutine_name", title) - phase_entry.set("filename", scheme.context.filename) - if title in scheme_headers: - header = scheme_headers[title] - proc = header.process_type - if proc != UNKNOWN_PROCESS_TYPE: - if process: - if process != proc: - emsg = 'Inconsistent process, {} != {}' - raise CCPPDatatableError(emsg.format(proc, process)) - # end if (no else, things are okay) - else: - process = proc - # end if - # end if - module = header.module - if module: - phase_entry.set("module", module) - # end if - else: - emsg = 'Could not find metadata header for {}' - raise CCPPDatatableError(emsg.format(sch_name)) - # end if - call_list = ET.SubElement(phase_entry, "call_list") - vlist = scheme.call_list.variable_list() - for var in vlist: - _new_var_entry(call_list, var, full_entry=False) - # end for - # end if - if process: - sch_entry.set("process", proc) - # end if - -############################################################################### -def _new_variable_dictionary(dictionaries, var_dict, dict_type, parent=None): -############################################################################### - """Create a new XML entry for under .""" - dict_entry = ET.SubElement(dictionaries, "var_dictionary") - dict_entry.set("name", var_dict.name) - dict_entry.set("type", dict_type) - if parent is not None: - dict_entry.set("parent", parent.name) - # end if - sub_dicts = var_dict.sub_dictionaries() - if sub_dicts: - sd_entry = ET.SubElement(dict_entry, "sub_dictionaries") - sd_entry.text = " ".join([x.name for x in sub_dicts]) - # end if - vars_entry = ET.SubElement(dict_entry, "variables") - for var in var_dict.variable_list(): - _new_var_entry(vars_entry, var, full_entry=True) - # end for - -############################################################################### -def _add_suite_object_dictionaries(dictionaries, suite_object): -############################################################################### - """Create new XML entries for under . - Add 's dictionary and its call_list dictionary (if present). - Recurse to this objects parts.""" - dict_type = _object_type(suite_object) - _new_variable_dictionary(dictionaries, suite_object, dict_type, - parent=suite_object.parent) - if suite_object.call_list: - dict_type += "_call_list" - _new_variable_dictionary(dictionaries, suite_object.call_list, - dict_type, parent=suite_object.parent) - # end if - for part in suite_object.parts: - _add_suite_object_dictionaries(dictionaries, part) - # end for - -############################################################################### -def _add_dependencies(parent, scheme_depends, host_depends): -############################################################################### - """Add a section to that lists all the dependencies - required by schemes or the host model. - >>> parent = ET.fromstring("") - >>> scheme_depends = ['file1', 'file2'] - >>> host_depends = ['file3', 'file4'] - >>> _add_dependencies(parent, scheme_depends, host_depends) - >>> table_entry_pretty_print(parent, 0) - '\\n \\n \\n file3\\n \\n \\n file4\\n \\n \\n file1\\n \\n \\n file2\\n \\n \\n\\n' - """ - file_entry = ET.SubElement(parent, "dependencies") - for hfile in host_depends: - entry = ET.SubElement(file_entry, "dependency") - entry.text = hfile - # end for - for sfile in scheme_depends: - entry = ET.SubElement(file_entry, "dependency") - entry.text = sfile - # end for - -############################################################################### -def _add_generated_files(parent, host_files, suite_files, ccpp_kinds, src_dir): -############################################################################### - """Add a section to that lists all the files generated - by in sections for host cap, suite caps, ccpp_kinds, and source files. - Also add existing utility files which are always needed by the framework. - """ - file_entry = ET.SubElement(parent, "ccpp_files") - utilities = ET.SubElement(file_entry, "utilities") - entry = ET.SubElement(utilities, "file") - entry.text = ccpp_kinds - entry = ET.SubElement(utilities, "file") - entry.text = os.path.join(src_dir, "ccpp_constituent_prop_mod.F90") - entry = ET.SubElement(utilities, "file") - entry.text = os.path.join(src_dir, "ccpp_scheme_utils.F90") - entry = ET.SubElement(utilities, "file") - entry.text = os.path.join(src_dir, "ccpp_hashable.F90") - entry = ET.SubElement(utilities, "file") - entry.text = os.path.join(src_dir, "ccpp_hash_table.F90") - host_elem = ET.SubElement(file_entry, "host_files") - for hfile in host_files: - entry = ET.SubElement(host_elem, "file") - entry.text = hfile - # end for - suite_elem = ET.SubElement(file_entry, "suite_files") - for sfile in suite_files: - entry = ET.SubElement(suite_elem, "file") - entry.text = sfile - # end for - -############################################################################### -def _add_suite_object(parent, suite_object): -############################################################################### - """Add an entry for under . This operation is - recursive to all the components inside of """ - obj_elem = ET.SubElement(parent, _object_type(suite_object)) - obj_elem.set("name", suite_object.name) - ptype = suite_object.phase_type - if ptype: - obj_elem.set("phase", ptype) - # end if - if isinstance(suite_object, Subcycle): - obj_elem.set("loop", suite_object._loop) - # end if - for obj_part in suite_object.parts: - _add_suite_object(obj_elem, obj_part) - # end for - -############################################################################### -def generate_ccpp_datatable(run_env, host_model, api, scheme_headers, - scheme_tdict, host_files, suite_files, - ccpp_kinds, source_dir): -############################################################################### - """Write a CCPP datatable for to . - The datatable includes the generated filenames for the host cap, - the suite caps, the ccpp_kinds module, and source code files. - """ - # Define new tree - datatable = ET.Element("ccpp_datatable") - datatable.set("version", "1.0") - # Write out the generated files - _add_generated_files(datatable, host_files, suite_files, - ccpp_kinds, source_dir) - # Write out scheme info - schemes = ET.SubElement(datatable, "schemes") - # Create a dictionary of the scheme headers for easy lookup - scheme_header_dict = {} - for header in scheme_headers: - if header.title in scheme_header_dict: - emsg = 'Header {} already in dictionary' - raise CCPPDatatableError(emsg.format(header.title)) - # end if - scheme_header_dict[header.title] = header - # end for - # Dump all scheme info from the suites - for suite in api.suites: - for group in suite.groups: - gname = group.name - for scheme in group.schemes(): - _new_scheme_entry(schemes, scheme, gname, scheme_header_dict) - # end for - # end for - # end for - # Write the API - api_elem = ET.SubElement(datatable, "api") - suites_elem = ET.SubElement(api_elem, "suites") - for suite in api.suites: - suite_elem = ET.SubElement(suites_elem, "suite") - suite_elem.set("name", suite.name) - suite_elem.set("filename", suite.sdf_name) - for group in suite.groups: - # Skip empty groups - if group.parts: - _add_suite_object(suite_elem, group) - # end if - # end for - # end for - # Dump the variable dictionaries - var_dicts = ET.SubElement(datatable, "var_dictionaries") - # First, the top-level dictionaries - _new_variable_dictionary(var_dicts, host_model, "host") - _new_variable_dictionary(var_dicts, api, "api", parent=api.parent) - # Now, the suite and group namelists, etc. (including call_lists) - for suite in api.suites: - _new_variable_dictionary(var_dicts, suite, "suite", parent=suite.parent) - for group in suite.groups: - _add_suite_object_dictionaries(var_dicts, group) - # end for - # end for - # end for - # Add in all dependencies - scheme_depends = set() - for table in scheme_tdict: - for dep_file in scheme_tdict[table].dependencies: - scheme_depends.add(dep_file) - # end for - # end for - host_depends = set() - host_tables = host_model.metadata_tables() - for table in host_tables: - for dep_file in host_tables[table].dependencies: - host_depends.add(dep_file) - # end for - # end for - _add_dependencies(datatable, scheme_depends, host_depends) - # Write tree - write_xml_file(datatable, run_env.datatable_file) - -############################################################################### - -if __name__ == "__main__": - PARGS = parse_command_line(sys.argv[1:]) - if PARGS.show: - _INDENT_STR = " "*PARGS.indent - LINE_WRAP = PARGS.line_wrap - REPORT = datatable_pretty_print(PARGS.datatable, 0, line_wrap=LINE_WRAP) - else: - ARG_VARS = vars(PARGS) - _ACTION = None - _ERRMSG = '' - _ESEP = '' - for opt in ARG_VARS: - if (opt in DatatableReport.valid_actions()) and ARG_VARS[opt]: - if _ACTION: - _ERRMSG += _ESEP + "Duplicate action, '{}'".format(opt) - _ESEP = '\n' - else: - _ACTION = DatatableReport(opt, ARG_VARS[opt]) - # end if - # end if - # end for - if _ERRMSG: - raise ValueError(_ERRMSG) - # end if - REPORT = datatable_report(PARGS.datatable, _ACTION, - PARGS.sep, PARGS.exclude_protected) - # end if - print("{}".format(REPORT.rstrip())) - sys.exit(0) diff --git a/scripts/ccpp_fortran_to_metadata.py b/scripts/ccpp_fortran_to_metadata.py deleted file mode 100755 index 56d466e4..00000000 --- a/scripts/ccpp_fortran_to_metadata.py +++ /dev/null @@ -1,258 +0,0 @@ -#!/usr/bin/env python3 - -#pylint: disable=anomalous-backslash-in-string -""" -Create prototype CCPP metadata tables from Fortran files - -Parses annotated Fortran files to produce metadata files where the -standard_name, units, and dimension standard names must be filled in. -The annotation is a two line comment for every physics scheme, derived -data type (DDT) definition, or host model data section. -The annotation form is: - -!> \section arg_table_ Argument Table -!! \htmlinclude arg_table_.html - -where is the name of the scheme, the name of the DDT, or the -name of the module containing data to be included in the metadata file. -For a scheme, the annotation must appear just before the subroutine statement. -For a DDT definition, the annotation must appear just before the type statement. -For module data, the annotation should occur after any module variables - which should not be included in the metadata file. -Note that only CCPP interfaces (e.g., _run, _init, _final) - will be documented in this manner. All other routines should be left as is. -""" -#pylint: enable=anomalous-backslash-in-string - -# Python library imports -import argparse -import sys -import os -import os.path -import logging -# CCPP framework imports -from framework_env import CCPPFrameworkEnv -from parse_tools import init_log, set_log_level -from parse_tools import CCPPError, ParseInternalError -from parse_tools import reset_standard_name_counter, unique_standard_name -from parse_tools import register_fortran_ddt_name -from fortran_tools import parse_fortran_file -from file_utils import create_file_list -from metadata_table import blank_metadata_line - -## Init this now so that all Exceptions can be trapped -_LOGGER = init_log(os.path.basename(__file__)) - -## Recognized Fortran filename extensions -_FORTRAN_FILENAME_EXTENSIONS = ['F90', 'f90', 'F', 'f'] - -############################################################################### -def parse_command_line(args, description): -############################################################################### - "Create an ArgumentParser to parse and return command-line arguments" - parser = argparse.ArgumentParser(description=description, - formatter_class=argparse.RawTextHelpFormatter) - - parser.add_argument("files", metavar='', - type=str, - help="""Comma separated list of filenames to process -Filenames with a '.meta' suffix are treated as host model metadata files -Filenames with a '.txt' suffix are treated as containing a list of .meta -filenames""") - - parser.add_argument("--preproc-directives", - metavar='VARDEF1[,VARDEF2 ...]', type=str, default='', - help="Proprocessor directives used to correctly parse source files") - - parser.add_argument("--ddt-names", - metavar='DDT_NAME1[,DDT_NAME2 ...]', type=str, default='', - help="Comma-separated DDT names that may be used in the parsed Fortran files") - - parser.add_argument("--output-root", type=str, - metavar='', - default=os.getcwd(), - help="directory for generated files") - - parser.add_argument("--section-separator", type=str, default='', - help="""Comment line to separate CCPP metadata tables -(must start with a # or ; character)""") - - parser.add_argument("--verbose", action='count', default=0, - help="Log more activity, repeat for increased output") - pargs = parser.parse_args(args) - return pargs - -############################################################################### -def write_metadata_file(mfilename, ftables, sep): -############################################################################### - """ - Write the prototype metadata file, , based on the - headers () parsed from Fortran. - """ - # Write the metadata file with all the items collected from Fortran - with open(mfilename, 'w') as outfile: - header_sep = '' - table_name = '' - for table in ftables: - # Write the table properties section - # Note that there may be extra tables depending on how the - # Fortran was parsed. - if (not table_name) or (table_name != table.table_name): - outfile.write("{}[ccpp-table-properties]{}".format(header_sep, - os.linesep)) - header_sep = sep + os.linesep - table_name = table.table_name - outfile.write(" name = {}{}".format(table_name, os.linesep)) - outfile.write(" type = {}{}".format(table.table_type, - os.linesep)) - # end if - for header in table.sections(): - lname_dict = {'1':'ccpp_constant_one'} - outfile.write('{}[ccpp-arg-table]{}'.format(header_sep, - os.linesep)) - outfile.write(' name = {}{}'.format(header.title, - os.linesep)) - outfile.write(' type = {}{}'.format(header.header_type, - os.linesep)) - for var in header.variable_list(): - lname = var.get_prop_value('local_name') - outfile.write('[ {} ]{}'.format(lname, os.linesep)) - prop = var.get_prop_value('standard_name') - outfile.write(' standard_name = {}{}'.format(prop, - os.linesep)) - lname_dict[lname] = prop - prop = var.get_prop_value('units') - if not prop: - prop = 'enter_units' - # end if - outfile.write(' units = {}{}'.format(prop, os.linesep)) - tprop = var.get_prop_value('type') - kprop = var.get_prop_value('kind') - if tprop == kprop: - outfile.write(' type = {}'.format(tprop)) - else: - outfile.write(' type = {}'.format(tprop.lower())) - if kprop: - outfile.write(' | kind = {}'.format(kprop.lower())) - # end if - # end if - outfile.write(os.linesep) - dims = var.get_dimensions() - # Fill in standard names for dimensions - dlist = list() - if dims: - for dim in dims: - dslist = list() - for dimspec in dim.split(':'): - if dimspec and (dimspec in lname_dict): - dstr = lname_dict[dimspec] - else: - dstr = unique_standard_name() - # end if - dslist.append(dstr) - # end for - dlist.append(':'.join(dslist)) - # end for - # end if - prop = '(' + ','.join(dlist) + ')' - outfile.write(' dimensions = {}{}'.format(prop, - os.linesep)) - if header.header_type == 'scheme': - prop = var.get_prop_value('intent') - outfile.write(' intent = {}{}'.format(prop, - os.linesep)) - # end if - # end for - # end for - # end for - # end with - -############################################################################### -def parse_fortran_files(filenames, run_env, output_dir, sep, logger): -############################################################################### - """ - Parse each file in and produce a prototype metadata file - with a metadata table for each arg_table entry in the file. - """ - meta_filenames = list() - for filename in filenames: - logger.info('Looking for arg_tables from {}'.format(filename)) - reset_standard_name_counter() - ftables, _ = parse_fortran_file(filename, run_env) - # Create metadata filename - filepath = '.'.join(os.path.basename(filename).split('.')[0:-1]) - fname = filepath + '.meta' - mfilename = os.path.join(output_dir, fname) - write_metadata_file(mfilename, ftables, sep) - meta_filenames.append(mfilename) - return meta_filenames - -############################################################################### -def _main_func(): -############################################################################### - """Parse command line, then parse indicated Fortran files. - Finally, generate a prototype metadata file for each Fortran file.""" - args = parse_command_line(sys.argv[1:], __doc__) - verbosity = args.verbose - if verbosity > 1: - set_log_level(_LOGGER, logging.DEBUG) - elif verbosity > 0: - set_log_level(_LOGGER, logging.INFO) - # end if - if args.ddt_names: - for dname in args.ddt_names.split(','): - register_fortran_ddt_name(dname) - # end for - # end if - # Make sure we know where output is going - output_dir = os.path.abspath(args.output_root) - # Optional table separator comment - section_sep = args.section_separator - if not blank_metadata_line(section_sep): - emsg = "Illegal section separator, '{}', first character must be # or ;" - raise CCPPError(emsg.format(section_sep)) - # We need to create a list of input Fortran files - fort_files = create_file_list(args.files, _FORTRAN_FILENAME_EXTENSIONS, - 'Fortran', _LOGGER) - preproc_defs = args.preproc_directives - ## A few sanity checks - ## Make sure output directory is legit - if os.path.exists(output_dir): - if not os.path.isdir(output_dir): - errmsg = "output-root, '{}', is not a directory" - raise CCPPError(errmsg.format(args.output_root)) - # end if - if not os.access(output_dir, os.W_OK): - errmsg = "Cannot write files to output-root ({})" - raise CCPPError(errmsg.format(args.output_root)) - # end if (output_dir is okay) - else: - # Try to create output_dir (let it crash if it fails) - os.makedirs(output_dir) - # end if - # Parse the files and create metadata - run_env = CCPPFrameworkEnv(_LOGGER, verbose=verbosity, - host_files="", scheme_files="", suites="", - preproc_directives=preproc_defs) - _ = parse_fortran_files(fort_files, run_env, - output_dir, section_sep, _LOGGER) - -############################################################################### - -if __name__ == "__main__": - try: - _main_func() - sys.exit(0) - except ParseInternalError as pie: - _LOGGER.exception(pie) - sys.exit(-1) - except CCPPError as ccpp_err: - if _LOGGER.getEffectiveLevel() <= logging.DEBUG: - _LOGGER.exception(ccpp_err) - else: - _LOGGER.error(ccpp_err) - # end if - sys.exit(1) - finally: - logging.shutdown() - # end try diff --git a/scripts/ccpp_prebuild.py b/scripts/ccpp_prebuild.py deleted file mode 100755 index c6198e27..00000000 --- a/scripts/ccpp_prebuild.py +++ /dev/null @@ -1,856 +0,0 @@ -#!/usr/bin/env python3 - -# Standard modules -import argparse -import collections -import copy -import filecmp -import importlib -import itertools -import logging -import os -import re -import sys - -# CCPP framework imports -from common import lowercase_keys_and_values -from common import encode_container, decode_container, decode_container_as_dict -from common import CCPP_STAGES, CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE -from common import STANDARD_VARIABLE_TYPES, STANDARD_INTEGER_TYPE, CCPP_TYPE -from common import SUITE_DEFINITION_FILENAME_PATTERN -from common import split_var_name_and_array_reference -from metadata_parser import merge_dictionaries, parse_scheme_tables, parse_variable_tables -from mkcap import CapsMakefile, CapsCMakefile, CapsSourcefile, \ - SchemesMakefile, SchemesCMakefile, SchemesSourcefile, \ - TypedefsMakefile, TypedefsCMakefile, TypedefsSourcefile -from mkdoc import metadata_to_html, metadata_to_latex -from mkstatic import API, Suite, Group -from mkstatic import CCPP_SUITE_VARIABLES - -############################################################################### -# Set up the command line argument parser and other global variables # -############################################################################### - -parser = argparse.ArgumentParser() -parser.add_argument('--config', action='store', help='path to CCPP prebuild configuration file', required=True) -parser.add_argument('--clean', action='store_true', help='remove files created by this script, then exit', default=False) -parser.add_argument('--verbose', action='store_true', help='enable verbose output from this script', default=False) -parser.add_argument('--debug', action='store_true', help='enable debugging features in auto-generated code', default=False) -parser.add_argument('--suites', action='store', help='suite definition files to use (comma-separated, without path)', default='') -parser.add_argument('--builddir', action='store', help='relative path to CCPP build directory', required=False, default=None) -parser.add_argument('--namespace', action='store', help='namespace suffix to be added to the name of static api module', required=False, default='') - -# BASEDIR is the current directory where this script is executed -BASEDIR = os.getcwd() - -############################################################################### -# Functions and subroutines # -############################################################################### - -def parse_arguments(): - """Parse command line arguments.""" - success = True - args = parser.parse_args() - configfile = args.config - clean = args.clean - verbose = args.verbose - debug = args.debug - if args.suites: - sdfs = ['{0}.xml'.format(x) for x in args.suites.split(',')] - else: - sdfs = None - builddir = args.builddir - namespace = args.namespace - return (success, configfile, clean, verbose, debug, sdfs, builddir, namespace) - -def import_config(configfile, builddir): - """Import the configuration from a given configuration file""" - success = True - config = {} - - if not os.path.isfile(configfile): - logging.error("Configuration file {0} not found".format(configfile)) - success = False - return(success, config) - - # Import the host-model specific CCPP prebuild config; - # split into path and module name for import - configpath = os.path.abspath(os.path.dirname(configfile)) - configmodule = os.path.splitext(os.path.basename(configfile))[0] - sys.path.append(configpath) - ccpp_prebuild_config = importlib.import_module(configmodule) - - # If the build directory for running ccpp_prebuild.py is not - # specified as command line argument, use value from config - if not builddir: - builddir = os.path.join(BASEDIR, ccpp_prebuild_config.DEFAULT_BUILD_DIR) - logging.info('Build directory not specified on command line, ' + \ - 'use "{}" from CCPP prebuild config'.format(ccpp_prebuild_config.DEFAULT_BUILD_DIR)) - - # Definitions in host-model dependent CCPP prebuild config script - config['variable_definition_files'] = ccpp_prebuild_config.VARIABLE_DEFINITION_FILES - config['typedefs_makefile'] = ccpp_prebuild_config.TYPEDEFS_MAKEFILE.format(build_dir=builddir) - config['typedefs_cmakefile'] = ccpp_prebuild_config.TYPEDEFS_CMAKEFILE.format(build_dir=builddir) - config['typedefs_sourcefile'] = ccpp_prebuild_config.TYPEDEFS_SOURCEFILE.format(build_dir=builddir) - config['scheme_files'] = ccpp_prebuild_config.SCHEME_FILES - config['schemes_makefile'] = ccpp_prebuild_config.SCHEMES_MAKEFILE.format(build_dir=builddir) - config['schemes_cmakefile'] = ccpp_prebuild_config.SCHEMES_CMAKEFILE.format(build_dir=builddir) - config['schemes_sourcefile'] = ccpp_prebuild_config.SCHEMES_SOURCEFILE.format(build_dir=builddir) - config['caps_makefile'] = ccpp_prebuild_config.CAPS_MAKEFILE.format(build_dir=builddir) - config['caps_cmakefile'] = ccpp_prebuild_config.CAPS_CMAKEFILE.format(build_dir=builddir) - config['caps_sourcefile'] = ccpp_prebuild_config.CAPS_SOURCEFILE.format(build_dir=builddir) - config['caps_dir'] = ccpp_prebuild_config.CAPS_DIR.format(build_dir=builddir) - config['suites_dir'] = ccpp_prebuild_config.SUITES_DIR.format(build_dir=builddir) - config['host_model'] = ccpp_prebuild_config.HOST_MODEL_IDENTIFIER - config['html_vartable_file'] = ccpp_prebuild_config.HTML_VARTABLE_FILE.format(build_dir=builddir) - config['latex_vartable_file'] = ccpp_prebuild_config.LATEX_VARTABLE_FILE.format(build_dir=builddir) - # Location of static API file, shell script to source, cmake include file - config['static_api_dir'] = ccpp_prebuild_config.STATIC_API_DIR.format(build_dir=builddir) - config['static_api_sourcefile'] = ccpp_prebuild_config.STATIC_API_SOURCEFILE.format(build_dir=builddir) - config['static_api_cmakefile'] = ccpp_prebuild_config.STATIC_API_CMAKEFILE.format(build_dir=builddir) - - # To handle new metadata: import DDT references (if exist) - try: - config['typedefs_new_metadata'] = ccpp_prebuild_config.TYPEDEFS_NEW_METADATA - logging.info("Found TYPEDEFS_NEW_METADATA dictionary in config, assume at least some data is in new metadata format") - except AttributeError: - config['typedefs_new_metadata'] = None - logging.info("Could not find TYPEDEFS_NEW_METADATA dictionary in config, assume all data is in old metadata format") - - return(success, config) - -def setup_logging(verbose): - """Sets up the logging module and logging level.""" - success = True - if verbose: - level = logging.DEBUG - else: - level = logging.INFO - logging.basicConfig(format='%(levelname)s: %(message)s', level=level) - if verbose: - logging.info('Logging level set to DEBUG') - else: - logging.info('Logging level set to INFO') - return success - -def clean_files(config, namespace): - """Clean files created by ccpp_prebuild.py""" - success = True - logging.info('Performing clean ....') - if namespace: - static_api_file = '{api}.F90'.format(api=CCPP_STATIC_API_MODULE+'_'+namespace) - else: - static_api_file = '{api}.F90'.format(api=CCPP_STATIC_API_MODULE) - # Create list of files to remove, use wildcards where necessary - files_to_remove = [ - config['typedefs_makefile'], - config['typedefs_cmakefile'], - config['typedefs_sourcefile'], - config['schemes_makefile'], - config['schemes_cmakefile'], - config['schemes_sourcefile'], - config['caps_makefile'], - config['caps_cmakefile'], - config['caps_sourcefile'], - config['html_vartable_file'], - config['latex_vartable_file'], - os.path.join(config['caps_dir'], 'ccpp_*_cap.F90'), - os.path.join(config['static_api_dir'], static_api_file), - config['static_api_sourcefile'], - ] - for f in files_to_remove: - try: - os.remove(f) - except FileNotFoundError: - pass - except Exception as e: - logging.error(f"Error removing {f}: {e}") - success = False - return success - -def get_all_suites(suites_dir): - """Assemble a list of all suite definition files in suites_dir""" - success = False - logging.info("No suites were given, compiling a list of all suites") - sdfs = [] - for f in os.listdir(suites_dir): - match = SUITE_DEFINITION_FILENAME_PATTERN.match(f) - if match: - logging.info('Adding suite definition file {}'.format(f)) - sdfs.append(f) - if sdfs: - success = True - return (success, sdfs) - -def parse_suites(suites_dir, sdfs): - """Parse suite definition files for prebuild""" - logging.info('Parsing suite definition files ...') - suites = [] - for sdf in sdfs: - sdf_file=os.path.join(suites_dir, sdf) - if not os.path.exists(sdf_file): - # If suite file not found, check old filename convention (suite_[suitename].xml) - sdf_file_legacy=os.path.join(suites_dir, f"suite_{sdf}") - if os.path.exists(sdf_file_legacy): - logging.warning("Parsing suite definition file using legacy naming convention") - logging.warning(f"Filename {os.path.basename(sdf_file_legacy)}") - logging.warning(f"Suite name {sdf}") - sdf_file=sdf_file_legacy - else: - logging.critical(f"Suite definition file {sdf_file} not found.") - success = False - return (success, suites) - - logging.info(f'Parsing suite definition file {sdf_file} ...') - suite = Suite(sdf_name=sdf_file) - success = suite.parse() - if not success: - logging.error('Parsing suite definition file {0} failed.'.format(sdf)) - break - suites.append(suite) - return (success, suites) - -def convert_local_name_from_new_metadata(metadata, standard_name, typedefs_new_metadata, converted_variables): - """Convert local names in new metadata format (no old-style DDT references, array references as - standard names) to old metadata format (with old-style DDT references, array references as local names).""" - success = True - var = metadata[standard_name][0] - # Check if this variable has already been converted - if standard_name in converted_variables: - logging.debug('Variable {0} was in old metadata format and has already been converted'.format(standard_name)) - return (success, var.local_name, converted_variables) - # Decode container into a dictionary - container = decode_container_as_dict(var.container) - # Check if variable is in old or new metadata format - module_name = container['MODULE'] - if not module_name in typedefs_new_metadata.keys(): - logging.debug('Variable {0} is in old metadata format, no conversion necessary'.format(standard_name)) - return (success, var.local_name, converted_variables) - # For module variables set type_name to module_name - if not 'TYPE' in container.keys(): - type_name = module_name - else: - type_name = container['TYPE'] - # Check that this module/type is configured (modules will have empty prefices) - if not type_name in typedefs_new_metadata[module_name].keys(): - logging.error("Module {0} uses the new metadata format, but module/type {1} is not configured".format(module_name, type_name)) - success = False - return (success, None, converted_variables) - - # The local name (incl. the array reference) is in new metadata format - local_name = var.local_name - logging.debug("Converting local name {0} of variable {1} from new to old metadata".format(local_name, standard_name)) - if "(" in local_name: - (actual_var_name, array_reference) = split_var_name_and_array_reference(local_name) - indices = array_reference.lstrip('(').rstrip(')').split(',') - indices_local_names = [] - for index_range in indices: - # Remove leading and trailing whitespaces - index_range = index_range.strip() - # Leave colons-only dimension alone - if index_range == ':': - indices_local_names.append(index_range) - continue - # Split by colons to get a pair of dimensions - dimensions = index_range.split(':') - dimensions_local_names = [] - for dimension in dimensions: - # Remove leading and trailing whitespaces - dimension = dimension.strip() - # Leave literals alone - try: - int(dimension) - dimensions_local_names.append(dimension) - continue - except ValueError: - pass - # Convert the local name of the dimension to old metadata standard, if necessary (recursive call) - (success, local_name_dim, converted_variables) = convert_local_name_from_new_metadata( - metadata, dimension, typedefs_new_metadata, converted_variables) - if not success: - return (success, None, converted_variables) - # Update the local name of the dimension, if necessary - if not metadata[dimension][0].local_name == local_name_dim: - logging.debug("Updating local name of variable {0} from {1} to {2}".format(dimension, - metadata[dimension][0].local_name, local_name_dim)) - metadata[dimension][0].local_name = local_name_dim - dimensions_local_names.append(local_name_dim) - indices_local_names.append(':'.join(dimensions_local_names)) - # Put back together the array reference with local names in old metadata format - array_reference_local_names = '(' + ','.join(indices_local_names) + ')' - # Compose local name (still without any DDT reference prefix) - local_name = actual_var_name + array_reference_local_names - - # Prefix the local name with the reference if not empty - if typedefs_new_metadata[module_name][type_name]: - local_name = typedefs_new_metadata[module_name][type_name] + '%' + local_name - if success: - converted_variables.append(standard_name) - - return (success, local_name, converted_variables) - -def gather_variable_definitions(variable_definition_files, typedefs_new_metadata): - """Scan all Fortran source files with variable definitions on the host model side. - If typedefs_new_metadata is not None, search all metadata entries and convert new metadata - (local names) into old metadata by prepending the DDT references.""" - # - logging.info('Parsing metadata tables for variables provided by host model ...') - success = True - metadata_define = collections.OrderedDict() - dependencies_define = collections.OrderedDict() - for variable_definition_file in variable_definition_files: - (filedir, filename) = os.path.split(os.path.abspath(variable_definition_file)) - # Change to directory of variable_definition_file and parse it - os.chdir(os.path.join(BASEDIR,filedir)) - (metadata, dependencies) = parse_variable_tables(filedir, filename) - metadata_define = merge_dictionaries(metadata_define, metadata) - dependencies_define.update(dependencies) - # Return to BASEDIR - os.chdir(BASEDIR) - # - if typedefs_new_metadata: - logging.info('Convert local names from new metadata format into old metadata format ...') - # Keep track of which variables have already been converted - converted_variables = [] - for key in metadata_define.keys(): - # Double-check that variable definitions are unique - if len(metadata_define[key])>1: - logging.error("Multiple definitions of standard_name {0} in type/variable defintions".format(key)) - success = False - return - (success, local_name, converted_variables) = convert_local_name_from_new_metadata( - metadata_define, key, typedefs_new_metadata, converted_variables) - if not success: - logging.error("An error occurred during the conversion of variable {0} from new to old metadata format".format(key)) - return (success, metadata_define) - # Update the local name of the variable, if necessary - if not metadata_define[key][0].local_name == local_name: - logging.debug("Updating local name of variable {0} from {1} to {2}".format(key, - metadata_define[key][0].local_name, local_name)) - metadata_define[key][0].local_name = local_name - # - return (success, metadata_define, dependencies_define) - -def collect_physics_subroutines(scheme_files): - """Scan all Fortran source files in scheme_files for subroutines with argument tables.""" - logging.info('Parsing metadata tables in physics scheme files ...') - success = True - # Parse all scheme files: record metadata, argument list, dependencies, and which scheme is in which file - metadata_request = collections.OrderedDict() - arguments_request = collections.OrderedDict() - dependencies_request = collections.OrderedDict() - schemes_in_files = collections.OrderedDict() - for scheme_file in scheme_files: - scheme_file_with_abs_path = os.path.abspath(scheme_file) - (scheme_filepath, scheme_filename) = os.path.split(scheme_file_with_abs_path) - # Change to directory where scheme_file lives - os.chdir(scheme_filepath) - (metadata, arguments, dependencies) = parse_scheme_tables(scheme_filepath, scheme_filename) - # Record which scheme is in which file - for scheme in arguments.keys(): - schemes_in_files[scheme] = scheme_file_with_abs_path - # Merge metadata, append to arguments and dependencies - metadata_request = merge_dictionaries(metadata_request, metadata) - arguments_request.update(arguments) - dependencies_request.update(dependencies) - os.chdir(BASEDIR) - # Return to BASEDIR - os.chdir(BASEDIR) - return (success, metadata_request, arguments_request, dependencies_request, schemes_in_files) - -def check_schemes_in_suites(arguments, suites): - """Check that all schemes that are requested in the suites exist""" - success = True - logging.info("Checking for existence of schemes in suites ...") - argument_keys = [x.lower() for x in arguments.keys()] - for suite in suites: - for group in suite.groups: - for subcycle in group.subcycles: - for scheme_name in subcycle.schemes: - if not scheme_name in argument_keys: - success = False - logging.critical("Scheme {} in suite {} cannot be found".format(scheme_name, suite.name)) - return success - -def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites): - """Remove all variables from metadata that are not used in the given suite; - also remove information on argument lists, dependencies and schemes in files""" - success = True - # Output: filtered dictionaries - metadata_filtered = collections.OrderedDict() - arguments_filtered = collections.OrderedDict() - dependencies_filtered = collections.OrderedDict() - schemes_in_files_filtered = collections.OrderedDict() - # Loop through all variables and check if the calling subroutine is in list of subroutines - for var_name in sorted(metadata.keys()): - keep = False - for var in metadata[var_name][:]: - container_string = decode_container(var.container) - subroutine = container_string[container_string.find('SUBROUTINE')+len('SUBROUTINE')+1:] - # Replace the full CCPP stage name with the abbreviated version - for ccpp_stage in CCPP_STAGES.keys(): - subroutine = subroutine.replace(ccpp_stage, CCPP_STAGES[ccpp_stage]) - for suite in suites: - if subroutine in suite.all_subroutines_called: - keep = True - break - if keep: - break - if keep: - metadata_filtered[var_name] = metadata[var_name] - else: - logging.info("filtering out variable {0}".format(var_name)) - # Filter argument lists - for scheme in arguments.keys(): - for suite in suites: - if scheme.lower() in suite.all_schemes_called: - arguments_filtered[scheme] = arguments[scheme] - break - # Filter dependencies - for scheme in dependencies.keys(): - for suite in suites: - if scheme.lower() in suite.all_schemes_called: - dependencies_filtered[scheme] = dependencies[scheme] - break - # Filter schemes_in_files - for scheme in schemes_in_files.keys(): - for suite in suites: - if scheme.lower() in suite.all_schemes_called: - schemes_in_files_filtered[scheme] = schemes_in_files[scheme] - return (success, metadata_filtered, arguments_filtered, dependencies_filtered, schemes_in_files_filtered) - -def add_ccpp_suite_variables(metadata): - """ Add variables that are required to construct CCPP suites to the list of requested variables""" - success = True - logging.info("Adding CCPP suite variables to list of requested variables") - for var_name in CCPP_SUITE_VARIABLES.keys(): - if not var_name in metadata.keys(): - metadata[var_name] = [copy.deepcopy(CCPP_SUITE_VARIABLES[var_name])] - logging.debug("Adding CCPP suite variable {0} to list of requested variables".format(var_name)) - return (success, metadata) - -def generate_list_of_schemes_and_dependencies_to_compile(schemes_in_files, dependencies1, dependencies2): - """Generate a flat list of schemes and dependencies in two dependency dictionaries to compile""" - success = True - # schemes_in_files is a dictionary with key scheme_name and value scheme_file - # dependencies is a dictionary with key scheme_name and value "list of dependencies" - schemes_and_dependencies_to_compile = list(schemes_in_files.values()) + \ - [dependency for dependency_list in list(dependencies1.values()) for dependency in dependency_list] + \ - [dependency for dependency_list in list(dependencies2.values()) for dependency in dependency_list] - # Remove duplicates - return (success, list(set(schemes_and_dependencies_to_compile))) - -def compare_metadata(metadata_define, metadata_request): - """Compare the requested metadata to the defined one. For each requested entry, a - single (i.e. non-ambiguous entry) must be present in the defined entries.""" - - logging.info('Comparing metadata for requested and provided variables ...') - success = True - modules = [] - metadata = collections.OrderedDict() - for var_name in sorted(metadata_request.keys()): - # Check that variable is provided by the model - if not var_name in metadata_define.keys(): - requested_by = ' & '.join(var.container for var in metadata_request[var_name]) - success = False - logging.error('Variable {0} requested by {1} not provided by the model'.format(var_name, requested_by)) - continue - # Check that an unambiguous target exists for this variable - if len(metadata_define[var_name]) > 1: - success = False - requested_by = ' & '.join(var.container for var in metadata_request[var_name]) - provided_by = ' & '.join(var.container for var in metadata_define[var_name]) - error_message = ' error, variable {0} requested by {1} cannot be identified unambiguously.'.format(var_name, requested_by) +\ - ' Multiple definitions in {0}'.format(provided_by) - logging.error(error_message) - continue - # Check that the variable properties are compatible between the model and the schemes; - # because we know that all variables in the metadata_request[var_name] list are compatible, - # it is sufficient to test the first entry against (the unique) metadata_define[var_name][0]. - if not metadata_request[var_name][0].compatible(metadata_define[var_name][0]): - success = False - error_message = ' incompatible entries in metadata for variable {0}:\n'.format(var_name) +\ - ' provided: {0}\n'.format(metadata_define[var_name][0].print_debug()) +\ - ' requested: {0}'.format(metadata_request[var_name][0].print_debug()) - logging.error(error_message) - continue - # Check for and register unit conversions if necessary. This must be done for each registered - # variable in the metadata_request[var_name] list (i.e. for each subroutine that is using it). - # Because var is an instance of the variable specific to the subroutine that uses it, and since - # each variable can be passed to a subroutine only once, there can be no overlapping/conflicting - # unit conversions. - for var in metadata_request[var_name]: - # Compare units - if var.units == metadata_define[var_name][0].units: - continue - # Register conversion, depending on the intent for this subroutine. - logging.debug('Registering unit conversion for variable {0} in {1}'.format(var_name, var.container)) - if var.intent=='inout': - var.convert_from(metadata_define[var_name][0].units) - var.convert_to(metadata_define[var_name][0].units) - elif var.intent=='in': - var.convert_from(metadata_define[var_name][0].units) - elif var.intent=='out': - var.convert_to(metadata_define[var_name][0].units) - # If the host model variable is allocated based on a condition, i.e. has an active attribute other - # than T (.true.), the scheme variable must be optional - if not metadata_define[var_name][0].active == 'T': - for var in metadata_request[var_name]: - if var.optional == 'F': - # DH 20241022 - change logging.error to logging.warn, because it is known - # that this strict check is not correct and will be reverted soon - #logging.error( - logging.warn("Conditionally allocated host-model variable {0} is not optional in {1}".format( - var_name, var.container)) - #success = False - # TEMPORARY CHECK - IF THE VARIABLE IS ALWAYS ALLOCATED, THE SCHEME VARIABLE SHOULDN'T BE OPTIONAL - else: - for var in metadata_request[var_name]: - if var.optional == 'T': - logging.warn("Unconditionally allocated host-model variable {0} is optional in {1}".format( - var_name, var.container)) - - # Construct the actual target variable and list of modules to use from the information in 'container' - var = metadata_define[var_name][0] - target = '' - for item in var.container.split(' '): - subitems = item.split('_') - if subitems[0] == 'MODULE': - # Add to list of required modules - modules.append('_'.join(subitems[1:])) - elif subitems[0] == 'TYPE': - pass - else: - logging.error('Unknown identifier {0} in container value of defined variable {1}'.format(subitems[0], var_name)) - success = False - target += var.local_name - # Copy the length kind from the variable definition to update len=* in the variable requests - if var.type == 'character': - kind = var.kind - metadata[var_name] = metadata_request[var_name] - # Set target and kind (if applicable) - for var in metadata[var_name]: - var.target = target - logging.debug('Requested variable {0} in {1} matched to target {2} in module {3}'.format( - var_name, var.container, target, modules[-1])) - # Update len=* for character variables - if var.type == 'character' and var.kind == 'len=*': - logging.debug('Update kind information for requested variable {0} in {1} from {2} to {3}'.format(var_name, - var.container, var.kind, kind)) - var.kind = kind - - # Remove duplicates from list of modules - modules = sorted(list(set(modules))) - return (success, modules, metadata) - -def generate_suite_and_group_caps(suites, metadata_request, metadata_define, arguments, caps_dir, debug): - """Generate for the suite and for all groups parsed.""" - logging.info("Generating suite and group caps ...") - suite_and_group_caps = [] - # Change to caps directory - os.chdir(caps_dir) - for suite in suites: - logging.debug("Generating suite and group caps for suite {0}...".format(suite.name)) - # Write caps for suite and groups in suite - suite.write(metadata_request, metadata_define, arguments, debug) - suite_and_group_caps += suite.caps - os.chdir(BASEDIR) - if suite_and_group_caps: - success = True - else: - success = False - return (success, suite_and_group_caps) - -def generate_static_api(suites, static_api_dir, namespace): - """Generate static API for given suite(s)""" - success = True - # Change to caps directory, create if necessary - if not os.path.isdir(static_api_dir): - os.makedirs(static_api_dir) - os.chdir(static_api_dir) - api = API(suites=suites, directory=static_api_dir) - if namespace: - base = os.path.splitext(os.path.basename(api.filename))[0] - logging.info('Static API file name is ''{}'''.format(api.filename)) - api.filename = base+'_'+namespace+'.F90' - api.module = base+'_'+namespace - logging.info('Static API file name is changed to ''{}'''.format(api.filename)) - logging.info('Generating static API {0} in {1} ...'.format(api.filename, static_api_dir)) - api.write() - os.chdir(BASEDIR) - return (success, api) - -def generate_typedefs_makefile(metadata_define, typedefs_makefile, typedefs_cmakefile, typedefs_sourcefile): - """Generate list of Fortran modules containing CCPP type/kind definitions, - and create makefile/cmakefile snippets for host model build system""" - logging.info('Generating list of Fortran modules containing CCPP type definitions ...') - success = True - # - typedefs = [] - # (1) Search for type definitions in the metadata, defined by: - # (a) the type not being a standard type, and - # (b) the type not being the CCPP framework internal type - # (c) the standard_name being identical to the type name - # (2) Search for kind definitions in the metadata, defined by: - # (a) the standard_name starting with "kind_" - # (b) the type being integer and the units being none - for key in metadata_define.keys(): - # derived data types - if not metadata_define[key][0].type in STANDARD_VARIABLE_TYPES and \ - not metadata_define[key][0].type == CCPP_TYPE and \ - metadata_define[key][0].type == metadata_define[key][0].standard_name: - container = decode_container_as_dict(metadata_define[key][0].container) - if not 'MODULE' in container.keys(): - logging.error("Invalid type definition for type {}: {}".format(metadata_define[key][0].type, metadata_define[key][0].print_debug())) - success = False - continue - # Fortran modules are lowercase and have the ending ".mod" - typedef_fortran_module = "{}.mod".format(container['MODULE']).lower() - if not typedef_fortran_module in typedefs: - typedefs.append(typedef_fortran_module) - # kind definitions - elif metadata_define[key][0].standard_name.startswith("kind_") and \ - metadata_define[key][0].type == STANDARD_INTEGER_TYPE and \ - metadata_define[key][0].units == 'none': - container = decode_container_as_dict(metadata_define[key][0].container) - if not 'MODULE' in container.keys(): - logging.error("Invalid kind definition for kind {}: {}".format(metadata_define[key][0].type, metadata_define[key][0].print_debug())) - success = False - continue - # Fortran modules are lowercase and have the ending ".mod" - typedef_fortran_module = "{}.mod".format(container['MODULE']).lower() - if not typedef_fortran_module in typedefs: - typedefs.append(typedef_fortran_module) - - logging.info('Generating typedefs makefile/cmakefile snippet ...') - # Write the Fortran modules without path - the build system knows where they are - makefile = TypedefsMakefile() - makefile.filename = typedefs_makefile + '.tmp' - cmakefile = TypedefsCMakefile() - cmakefile.filename = typedefs_cmakefile + '.tmp' - sourcefile = TypedefsSourcefile() - sourcefile.filename = typedefs_sourcefile + '.tmp' - # Sort typedefs so that the order remains the same (for cmake to avoid) recompiling - typedefs.sort() - # Generate list of type definitions - makefile.write(typedefs) - cmakefile.write(typedefs) - sourcefile.write(typedefs) - if os.path.isfile(typedefs_makefile) and \ - filecmp.cmp(typedefs_makefile, makefile.filename): - os.remove(makefile.filename) - os.remove(cmakefile.filename) - os.remove(sourcefile.filename) - else: - if os.path.isfile(typedefs_makefile): - os.remove(typedefs_makefile) - if os.path.isfile(typedefs_cmakefile): - os.remove(typedefs_cmakefile) - if os.path.isfile(typedefs_sourcefile): - os.remove(typedefs_sourcefile) - os.rename(makefile.filename, typedefs_makefile) - os.rename(cmakefile.filename, typedefs_cmakefile) - os.rename(sourcefile.filename, typedefs_sourcefile) - # - logging.info('Added {0} typedefs to {1}, {2}, {3}'.format( - len(typedefs), typedefs_makefile, typedefs_cmakefile, typedefs_sourcefile)) - return success - -def generate_schemes_makefile(schemes, schemes_makefile, schemes_cmakefile, schemes_sourcefile): - """Generate makefile/cmakefile snippets for all schemes.""" - logging.info('Generating schemes makefile/cmakefile snippet ...') - success = True - makefile = SchemesMakefile() - makefile.filename = schemes_makefile + '.tmp' - cmakefile = SchemesCMakefile() - cmakefile.filename = schemes_cmakefile + '.tmp' - sourcefile = SchemesSourcefile() - sourcefile.filename = schemes_sourcefile + '.tmp' - # Sort schemes so that the order remains the same (for cmake to avoid) recompiling - schemes.sort() - # Generate list of schemes with absolute path - schemes_with_abspath = [ os.path.abspath(scheme) for scheme in schemes ] - makefile.write(schemes_with_abspath) - cmakefile.write(schemes_with_abspath) - sourcefile.write(schemes_with_abspath) - if os.path.isfile(schemes_makefile) and \ - filecmp.cmp(schemes_makefile, makefile.filename): - os.remove(makefile.filename) - os.remove(cmakefile.filename) - os.remove(sourcefile.filename) - else: - if os.path.isfile(schemes_makefile): - os.remove(schemes_makefile) - if os.path.isfile(schemes_cmakefile): - os.remove(schemes_cmakefile) - if os.path.isfile(schemes_sourcefile): - os.remove(schemes_sourcefile) - os.rename(makefile.filename, schemes_makefile) - os.rename(cmakefile.filename, schemes_cmakefile) - os.rename(sourcefile.filename, schemes_sourcefile) - # - logging.info('Added {0} schemes to {1}, {2}, {3}'.format( - len(schemes_with_abspath), schemes_makefile, schemes_cmakefile, schemes_sourcefile)) - return success - -def generate_caps_makefile(caps, caps_makefile, caps_cmakefile, caps_sourcefile, caps_dir): - """Generate makefile/cmakefile snippets for all caps.""" - logging.info('Generating caps makefile/cmakefile snippet ...') - success = True - makefile = CapsMakefile() - makefile.filename = caps_makefile + '.tmp' - cmakefile = CapsCMakefile() - cmakefile.filename = caps_cmakefile + '.tmp' - sourcefile = CapsSourcefile() - sourcefile.filename = caps_sourcefile + '.tmp' - # Sort caps so that the order remains the same (for cmake to avoid) recompiling - caps.sort() - # Generate list of caps with absolute path - caps_with_abspath = [ os.path.abspath(os.path.join(caps_dir, cap)) for cap in caps ] - makefile.write(caps_with_abspath) - cmakefile.write(caps_with_abspath) - sourcefile.write(caps_with_abspath) - if os.path.isfile(caps_makefile) and \ - filecmp.cmp(caps_makefile, makefile.filename): - os.remove(makefile.filename) - os.remove(cmakefile.filename) - os.remove(sourcefile.filename) - else: - if os.path.isfile(caps_makefile): - os.remove(caps_makefile) - if os.path.isfile(caps_cmakefile): - os.remove(caps_cmakefile) - if os.path.isfile(caps_sourcefile): - os.remove(caps_sourcefile) - os.rename(makefile.filename, caps_makefile) - os.rename(cmakefile.filename, caps_cmakefile) - os.rename(sourcefile.filename, caps_sourcefile) - # - logging.info('Added {0} auto-generated caps to {1} and {2}, {3}'.format( - len(caps_with_abspath), caps_makefile, caps_cmakefile, caps_sourcefile)) - return success - -def main(): - """Main routine that handles the CCPP prebuild for different host models.""" - # Parse command line arguments - (success, configfile, clean, verbose, debug, sdfs, builddir, namespace) = parse_arguments() - if not success: - raise Exception('Call to parse_arguments failed.') - - success = setup_logging(verbose) - if not success: - raise Exception('Call to setup_logging failed.') - - (success, config) = import_config(configfile, builddir) - if not success: - raise Exception('Call to import_config failed.') - - # Perform clean if requested, then exit - if clean: - success = clean_files(config, namespace) - logging.info('CCPP prebuild clean completed successfully, exiting.') - sys.exit(0) - - # Convert TYPEDEFS_NEW_METATA config to lowercase - config['typedefs_new_metadata'] = lowercase_keys_and_values(config['typedefs_new_metadata']) - - # If no suite definition files were given, get all of them - if not sdfs: - (success, sdfs) = get_all_suites(config['suites_dir']) - if not success: - raise Exception('Call to get_all_sdfs failed.') - - # Parse suite definition files for prebuild - (success, suites) = parse_suites(config['suites_dir'], sdfs) - if not success: - raise Exception('Parsing suite definition files failed.') - - # Variables defined by the host model - (success, metadata_define, dependencies_define) = gather_variable_definitions(config['variable_definition_files'], config['typedefs_new_metadata']) - if not success: - raise Exception('Call to gather_variable_definitions failed.') - - # Create an HTML table with all variables provided by the model - success = metadata_to_html(metadata_define, config['host_model'], config['html_vartable_file']) - if not success: - raise Exception('Call to metadata_to_html failed.') - - # Variables requested by the CCPP physics schemes - (success, metadata_request, arguments_request, dependencies_request, schemes_in_files) = collect_physics_subroutines(config['scheme_files']) - if not success: - raise Exception('Call to collect_physics_subroutines failed.') - - # Check that the schemes requested in the suites exist - success = check_schemes_in_suites(arguments_request, suites) - if not success: - raise Exception('Call to check_schemes_in_suites failed.') - - # Filter metadata/arguments - remove whatever is not included in suite definition files - (success, metadata_request, arguments_request, dependencies_request, schemes_in_files) = filter_metadata( - metadata_request, arguments_request, dependencies_request, schemes_in_files, suites) - if not success: - raise Exception('Call to filter_metadata failed.') - - # Add variables that are required to construct CCPP suites to the list of requested variables - (success, metadata_request) = add_ccpp_suite_variables(metadata_request) - if not success: - raise Exception('Call to add_ccpp_suite_variables failed.') - - (success, schemes_and_dependencies_to_compile) = generate_list_of_schemes_and_dependencies_to_compile( - schemes_in_files, dependencies_request, dependencies_define) - if not success: - raise Exception('Call to generate_list_of_schemes_and_dependencies_to_compile failed.') - - # Create a LaTeX table with all variables requested by the pool of physics and/or provided by the host model - success = metadata_to_latex(metadata_define, metadata_request, config['host_model'], config['latex_vartable_file']) - if not success: - raise Exception('Call to metadata_to_latex failed.') - - # Check requested against defined arguments to generate metadata (list/dict of variables for CCPP) - (success, modules, metadata) = compare_metadata(metadata_define, metadata_request) - if not success: - raise Exception('Call to compare_metadata failed.') - - # Add Fortran module files of typedefs to makefile/cmakefile/shell script - success = generate_typedefs_makefile(metadata_define, config['typedefs_makefile'], - config['typedefs_cmakefile'], config['typedefs_sourcefile']) - if not success: - raise Exception('Call to generate_typedefs_makefile failed.') - - # Add filenames of schemes and variable definition files (types) to makefile/cmakefile/shell script - success = generate_schemes_makefile(schemes_and_dependencies_to_compile + config['variable_definition_files'], - config['schemes_makefile'], config['schemes_cmakefile'], - config['schemes_sourcefile']) - if not success: - raise Exception('Call to generate_schemes_makefile failed.') - - # Static build: generate caps for entire suite and groups in the specified suite; generate API - (success, suite_and_group_caps) = generate_suite_and_group_caps(suites, metadata_request, metadata_define, - arguments_request, config['caps_dir'], debug) - if not success: - raise Exception('Call to generate_suite_and_group_caps failed.') - - (success, api) = generate_static_api(suites, config['static_api_dir'], namespace) - if not success: - raise Exception('Call to generate_static_api failed.') - - success = api.write_includefile(config['static_api_sourcefile'], type='shell') - if not success: - raise Exception("Writing API sourcefile {sourcefile} failed".format(sourcefile=config['static_api_sourcefile'])) - - success = api.write_includefile(config['static_api_cmakefile'], type='cmake') - if not success: - raise Exception("Writing API cmakefile {cmakefile} failed".format(cmakefile=config['static_api_cmakefile'])) - - # Add filenames of caps to makefile/cmakefile/shell script - all_caps = suite_and_group_caps - - success = generate_caps_makefile(all_caps, config['caps_makefile'], config['caps_cmakefile'], - config['caps_sourcefile'], config['caps_dir']) - if not success: - raise Exception('Call to generate_caps_makefile failed.') - - logging.info('CCPP prebuild step completed successfully.') - -if __name__ == '__main__': - main() diff --git a/scripts/ccpp_state_machine.py b/scripts/ccpp_state_machine.py deleted file mode 100644 index 832e0073..00000000 --- a/scripts/ccpp_state_machine.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Definition of the state machine used by the CCPP""" - -# CCPP framework imports -from state_machine import StateMachine - -_REG_ST = r"(?:register)" -_INIT_ST = r"(?:init(?:ial(?:ize)?)?)" -_FINAL_ST = r"(?:final(?:ize)?)" -_RUN_ST = r"(?:run)" -_TS_INIT_ST = r"(?:timestep_init(?:ial(?:ize)?)?)" -_TS_FINAL_ST = r"(?:timestep_final(?:ize)?)" - -# Allowed CCPP transitions -# pylint: disable=bad-whitespace -RUN_PHASE_NAME = 'run' -CCPP_STATE_MACH = StateMachine((('register', 'uninitialized', - 'uninitialized', _REG_ST), - ('initialize', 'uninitialized', - 'initialized', _INIT_ST), - ('timestep_initial', 'initialized', - 'in_time_step', _TS_INIT_ST), - (RUN_PHASE_NAME, 'in_time_step', - 'in_time_step', _RUN_ST), - ('timestep_final', 'in_time_step', - 'initialized', _TS_FINAL_ST), - ('finalize', 'initialized', - 'uninitialized', _FINAL_ST))) -# pylint: enable=bad-whitespace diff --git a/scripts/ccpp_suite.py b/scripts/ccpp_suite.py deleted file mode 100644 index d1a6e968..00000000 --- a/scripts/ccpp_suite.py +++ /dev/null @@ -1,1270 +0,0 @@ -#!/usr/bin/env python3 -# - -"""Classes and methods to create a Fortran suite-implementation file -to implement calls to a set of suites for a given host model.""" - -# Python library imports -import os.path -import logging -import xml.etree.ElementTree as ET -# CCPP framework imports -from ccpp_state_machine import CCPP_STATE_MACH, RUN_PHASE_NAME -from code_block import CodeBlock -from constituents import ConstituentVarDict -from ddt_library import DDTLibrary -from file_utils import KINDS_MODULE -from fortran_tools import FortranWriter -from framework_env import CCPPFrameworkEnv -from metavar import Var, VarDictionary, ccpp_standard_var -from parse_tools import ParseContext, ParseSource -from parse_tools import ParseInternalError, CCPPError -from parse_tools import read_xml_file, validate_xml_file, write_xml_file -from parse_tools import find_schema_version, expand_nested_suites -from parse_tools import init_log, set_log_to_null -from suite_objects import CallList, Group, Scheme -from metavar import CCPP_LOOP_VAR_STDNAMES -from var_props import is_horizontal_dimension - -# pylint: disable=too-many-lines - -############################################################################### -# Module (global) variables -############################################################################### - -# Source for internally generated variables. -API_SOURCE_NAME = "CCPP_API" -# Use the constituent source type for consistency -_API_SUITE_VAR_NAME = ConstituentVarDict.constitutent_source_type() -_API_SCHEME_VAR_NAME = "scheme" -_API_CONTEXT = ParseContext(filename="ccpp_suite.py") -_API_SOURCE = ParseSource(API_SOURCE_NAME, _API_SCHEME_VAR_NAME, _API_CONTEXT) -_API_LOGGING = init_log('ccpp_suite') -set_log_to_null(_API_LOGGING) -_API_DUMMY_RUN_ENV = CCPPFrameworkEnv(_API_LOGGING, - ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -# Required variables for inclusion in auto-generated schemes -CCPP_REQUIRED_VARS = [ccpp_standard_var('ccpp_error_code', - _API_SCHEME_VAR_NAME, - _API_DUMMY_RUN_ENV, - context=_API_CONTEXT), - ccpp_standard_var('ccpp_error_message', - _API_SCHEME_VAR_NAME, - _API_DUMMY_RUN_ENV, - context=_API_CONTEXT)] - -############################################################################### - -class Suite(VarDictionary): - """Class to hold, process, and output a CAP for an entire CCPP suite. - The Suite includes initialization and finalization Group objects as - well as a Group for every suite part.""" - - __state_machine_initial_state = 'uninitialized' - __state_machine_var_name = 'ccpp_suite_state' - - __state_machine_init = ''' -character(len=16) :: {css_var_name} = '{state}' -''' - - # Note that these group names need to match CCPP_STATE_MACH - __register_group_name = 'register' - - __initial_group_name = 'initialize' - - __final_group_name = 'finalize' - - __timestep_initial_group_name = 'timestep_initial' - - __timestep_final_group_name = 'timestep_final' - - __scheme_template = '{}' - - def __init__(self, filename, suite_xml, api, run_env): - """Initialize this Suite object from the SDF, . - serves as the Suite's parent.""" - self.__run_env = run_env - self.__name = None - self.__sdf_name = filename - self.__groups = list() - self.__suite_init_group = None - self.__suite_final_group = None - self.__timestep_init_group = None - self.__timestep_final_group = None - self.__context = None - self.__host_arg_list_full = None - self.__host_arg_list_noloop = None - self.__module = None - self.__ddt_library = None - # Full phases/groups are special groups where the entire state is passed - self.__full_groups = {} - self._full_phases = {} - self.__gvar_stdnames = {} # Standard names of group-created vars - # Initialize our dictionary - # Create a 'parent' to hold the constituent variables - # The parent for the constituent dictionary is the API. - temp_name = os.path.splitext(os.path.basename(filename))[0] - const_dict = ConstituentVarDict(temp_name+'_constituents', - api, run_env) - super().__init__(self.sdf_name, run_env, parent_dict=const_dict) - if not os.path.exists(self.__sdf_name): - emsg = "Suite definition file {0} not found." - raise CCPPError(emsg.format(self.__sdf_name)) - # end if - # Parse the SDF - self.parse(suite_xml, run_env) - - @property - def name(self): - """Get the name of the suite.""" - return self.__name - - @property - def sdf_name(self): - """Get the name of the suite definition file.""" - return self.__sdf_name - - @classmethod - def check_suite_state(cls, stage): - """Return a list of CCPP state check statements for """ - check_stmts = list() - if stage in CCPP_STATE_MACH.transitions(): - # We need to make sure we are an allowed previous state - prev_state = CCPP_STATE_MACH.initial_state(stage) - css = "trim({})".format(Suite.__state_machine_var_name) - prev_str = "({} /= '{}')".format(css, prev_state) - check_stmts.append(("if {} then".format(prev_str), 1)) - check_stmts.append(("{errcode} = 1", 2)) - errmsg_str = "write({errmsg}, '(3a)') " - errmsg_str += "\"Invalid initial CCPP state, '\", " + css + ', ' - errmsg_str += "\"' in {funcname}\"" - check_stmts.append((errmsg_str, 2)) - check_stmts.append(("return", 2)) - check_stmts.append(("end if", 1)) - else: - raise ParseInternalError("Unknown stage, '{}'".format(stage)) - # end if - return CodeBlock(check_stmts) - - @classmethod - def set_suite_state(cls, phase): - """Return the code string to set the current suite state to . - If the initial and final states of are identical, return blank. - """ - initial = CCPP_STATE_MACH.initial_state(phase) - final = CCPP_STATE_MACH.final_state(phase) - if initial == final: - stmt = '! Suite state does not change' - else: - stmt = "ccpp_suite_state = '{}'".format(final) - # end if - return CodeBlock([(stmt, 1)]) - - def new_group(self, group_string, transition, run_env): - """Create a new Group object from the a XML description""" - if isinstance(group_string, str): - gxml = ET.fromstring(group_string) - else: - gxml = group_string - # end if - group = Group(gxml, transition, self, self.__context, run_env) - for svar in CCPP_REQUIRED_VARS: - group.add_call_list_variable(svar) - # end for - if transition != RUN_PHASE_NAME: - self.__full_groups[group.name] = group - self._full_phases[group.phase()] = group - # end if - return group - - def new_group_from_name(self, group_name, run_env): - '''Create an XML string for Group, , and use it to - create the corresponding group. - Note: must be the a transition string''' - group_xml = ''.format(group_name) - return self.new_group(group_xml, group_name, run_env) - - def parse(self, suite_xml, run_env): - """Parse the suite definition file.""" - success = True - # We do not have line number information for the XML file - self.__context = ParseContext(filename=self.__sdf_name) - self.__name = suite_xml.get('name') - self.__module = 'ccpp_{}_cap'.format(self.name) - gname = Suite.__register_group_name - self.__suite_reg_group = self.new_group_from_name(gname, run_env) - gname = Suite.__initial_group_name - self.__suite_init_group = self.new_group_from_name(gname, run_env) - gname = Suite.__final_group_name - self.__suite_final_group = self.new_group_from_name(gname, run_env) - gname = Suite.__timestep_initial_group_name - self.__timestep_init_group = self.new_group_from_name(gname, run_env) - gname = Suite.__timestep_final_group_name - self.__timestep_final_group = self.new_group_from_name(gname, run_env) - # Set up some groupings for later efficiency - self._beg_groups = [self.__suite_reg_group.name, - self.__suite_init_group.name, - self.__timestep_init_group.name] - self._end_groups = [self.__suite_final_group.name, - self.__timestep_final_group.name] - # Build hierarchical structure as in SDF - self.__groups.append(self.__suite_reg_group) - self.__groups.append(self.__suite_init_group) - self.__groups.append(self.__timestep_init_group) - for suite_item in suite_xml: - item_type = suite_item.tag.lower() - # Suite item is a group or a suite-wide init or final method - if item_type == 'group': - # Parse a group - self.__groups.append(self.new_group(suite_item, RUN_PHASE_NAME, - run_env)) - else: - match_trans = CCPP_STATE_MACH.function_match(item_type) - if match_trans is None: - emsg = "Unknown CCPP suite component tag type, '{}'" - raise CCPPError(emsg.format(item_type)) - # end if - if match_trans in self._full_phases: - # Parse a suite-wide initialization scheme - scheme = Scheme(suite_item, self.__context, - self, run_env) - self._full_phases[match_trans].add_item(scheme) - else: - emsg = "Unhandled CCPP suite component tag type, '{}'" - raise ParseInternalError(emsg.format(match_trans)) - # end if - # end for - self.__groups.append(self.__timestep_final_group) - self.__groups.append(self.__suite_final_group) - return success - - def suite_dicts(self): - """Return a list of this Suite's dictionaries. - A Suite's dictionaries are itself plus its constituent dictionary""" - return [self, self.parent] - - @property - def module(self): - """Get the list of the module generated for this suite.""" - return self.__module - - @property - def groups(self): - """Get the list of groups in this suite.""" - return self.__groups - - def find_variable(self, standard_name=None, source_var=None, - any_scope=True, clone=None, - search_call_list=False, loop_subst=False): - """Attempt to return the variable matching . - if is None, the standard name from is used. - It is an error to pass both and if - the standard name of is not the same as . - If is True, search parent scopes if not in current scope. - If the variable is not found this Suite's groups are searched for - a matching output variable. If found that variable is promoted to be a - Suite module variable and that variable is returned. - If the variable is not found and is not None, add a clone of - to this dictionary. - If the variable is not found and is None, return None. - """ - # First, see if the variable is already in our path - srch_clist = search_call_list - var = super().find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, - clone=None, - search_call_list=srch_clist, - loop_subst=loop_subst) - if var is None: - # No dice? Check for a group variable which can be promoted - # Don't promote loop standard names - if (standard_name in self.__gvar_stdnames and standard_name - not in CCPP_LOOP_VAR_STDNAMES): - group = self.__gvar_stdnames[standard_name] - var = group.find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=False, - search_call_list=srch_clist, - loop_subst=loop_subst) - - if var is not None: - # Promote variable to suite level - # Remove this entry to avoid looping back here - del self.__gvar_stdnames[standard_name] - # Let everyone know this is now a Suite variable - var.source = ParseSource(API_SOURCE_NAME, - _API_SUITE_VAR_NAME, - var.context) - self.add_variable(var, self.__run_env) - # Remove the variable from the group - group.remove_variable(standard_name) - # Make sure the variable's dimensions are available - # at the init stage (for allocation) - for group in self.groups: - # only add dimension variables to init phase calling list - # if they're not module-level "suite" variables - if group.name == self.__suite_init_group.name: - dims = var.get_dimensions() - # replace horizontal loop dimension if necessary - for idx, dim in enumerate(dims): - if is_horizontal_dimension(dim): - if 'horizontal_loop' in dim: - dims[idx] = 'ccpp_constant_one:horizontal_dimension' - # end if - # end if - # end for - subst_dict = {'dimensions': dims} - prop_dict = var.copy_prop_dict(subst_dict=subst_dict) - temp_var = Var(prop_dict, - ParseSource(var.get_prop_value('scheme'), - var.get_prop_value('local_name'), var.context), - self.__run_env) - # Add dimensions if they're not already there - group.add_variable_dimensions(temp_var, [], - _API_SUITE_VAR_NAME, - adjust_intent=True, - to_dict=group.call_list) - # end if - # end for - else: - emsg = ("Group, {}, claimed it had created {} " - "but variable was not found") - raise CCPPError(emsg.format(group.name, standard_name)) - # end if - # end if - # end if - if (var is None) and (clone is not None): - # Guess it is time to clone a different variable - var = super().find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, clone=clone) - # end if - return var - - def analyze(self, host_model, scheme_library, ddt_library, run_env): - """Collect all information needed to write a suite file - >>> CCPP_STATE_MACH.transition_match('init') - 'initialize' - >>> CCPP_STATE_MACH.transition_match('init', transition='finalize') - - >>> CCPP_STATE_MACH.transition_match('INIT') - 'initialize' - >>> CCPP_STATE_MACH.transition_match('initial') - 'initialize' - >>> CCPP_STATE_MACH.transition_match('timestep_initial') - 'timestep_initial' - >>> CCPP_STATE_MACH.transition_match('timestep_initialize') - 'timestep_initial' - >>> CCPP_STATE_MACH.transition_match('timestep_init') - 'timestep_initial' - >>> CCPP_STATE_MACH.transition_match('initialize') - 'initialize' - >>> CCPP_STATE_MACH.transition_match('initialize')[0:4] - 'init' - >>> CCPP_STATE_MACH.transition_match('initize') - - >>> CCPP_STATE_MACH.transition_match('run') - 'run' - >>> CCPP_STATE_MACH.transition_match('finalize') - 'finalize' - >>> CCPP_STATE_MACH.transition_match('finalize')[0:5] - 'final' - >>> CCPP_STATE_MACH.transition_match('final') - 'finalize' - >>> CCPP_STATE_MACH.transition_match('finalize_bar') - - >>> CCPP_STATE_MACH.function_match('foo_init') - ('foo', 'init', 'initialize') - >>> CCPP_STATE_MACH.function_match('foo_init', transition='finalize') - (None, None, None) - >>> CCPP_STATE_MACH.function_match('FOO_INIT') - ('FOO', 'INIT', 'initialize') - >>> CCPP_STATE_MACH.function_match('foo_initial') - ('foo', 'initial', 'initialize') - >>> CCPP_STATE_MACH.function_match('foo_initialize') - ('foo', 'initialize', 'initialize') - >>> CCPP_STATE_MACH.function_match('foo_initialize')[1][0:4] - 'init' - >>> CCPP_STATE_MACH.function_match('foo_initize') - (None, None, None) - >>> CCPP_STATE_MACH.function_match('foo_timestep_initial') - ('foo', 'timestep_initial', 'timestep_initial') - >>> CCPP_STATE_MACH.function_match('foo_timestep_init') - ('foo', 'timestep_init', 'timestep_initial') - >>> CCPP_STATE_MACH.function_match('foo_timestep_initialize') - ('foo', 'timestep_initialize', 'timestep_initial') - >>> CCPP_STATE_MACH.function_match('foo_run') - ('foo', 'run', 'run') - >>> CCPP_STATE_MACH.function_match('foo_finalize') - ('foo', 'finalize', 'finalize') - >>> CCPP_STATE_MACH.function_match('foo_finalize')[1][0:5] - 'final' - >>> CCPP_STATE_MACH.function_match('foo_final') - ('foo', 'final', 'finalize') - >>> CCPP_STATE_MACH.function_match('foo_finalize_bar') - (None, None, None) - >>> CCPP_STATE_MACH.function_match('foo_timestep_final') - ('foo', 'timestep_final', 'timestep_final') - >>> CCPP_STATE_MACH.function_match('foo_timestep_finalize') - ('foo', 'timestep_finalize', 'timestep_final') - """ - self.__ddt_library = ddt_library - # Collect all relevant schemes - # For all groups, find associated init and final methods - scheme_list = list() - for group in self.groups: - for scheme in group.schemes(): - scheme_list.append(scheme.name) - # end for - # end for - no_scheme_entries = {} # Skip schemes that are not in this suite - for module in scheme_list: - if scheme_library[module]: - scheme_entries = scheme_library[module] - else: - scheme_entries = no_scheme_entries - # end if - for phase in self._full_phases: - if phase in scheme_entries: - header = scheme_entries[phase] - # Add this scheme's init or final routine - pgroup = self._full_phases[phase] - if not pgroup.has_item(header.title): - sstr = Suite.__scheme_template.format(module) - sxml = ET.fromstring(sstr) - scheme = Scheme(sxml, self.__context, pgroup, run_env) - pgroup.add_part(scheme) - # end if (no else, scheme is already in group) - # end if (no else, phase not in scheme set) - # end for - # end for - # Grab the host model argument list - self.__host_arg_list_full = host_model.argument_list() - self.__host_arg_list_noloop = host_model.argument_list(loop_vars=False) - # First pass, create init, run, and finalize sequences - for item in self.groups: - if item.name in self.__full_groups: - phase = self.__full_groups[item.name].phase() - else: - phase = RUN_PHASE_NAME - # end if - lmsg = "Group {}, schemes = {}" - if run_env.verbose: - run_env.logger.debug(lmsg.format(item.name, - [x.name - for x in item.schemes()])) - item.analyze(phase, self, scheme_library, ddt_library, - self.check_suite_state(phase), - self.set_suite_state(phase)) - # Look for group variables that need to be promoted to the suite - # We need to promote any variable used later to the suite, however, - # we do not yet know if it will be used. - # Add new group-created variables - gvars = item.variable_list() - for gvar in gvars: - stdname = gvar.get_prop_value('standard_name') - if not stdname in self.__gvar_stdnames: - self.__gvar_stdnames[stdname] = item - # end if - # end for - # end for - - def is_run_group(self, group): - """Method to separate out run-loop groups from special initial - and final groups - """ - return ((group.name not in self._beg_groups) and - (group.name not in self._end_groups)) - - def max_part_len(self): - """What is the longest suite subroutine name?""" - maxlen = 0 - for spart in self.groups: - if self.is_run_group(spart): - maxlen = max(maxlen, len(spart.name)) - # end if - # end for - return maxlen - - def part_list(self): - """Return list of run phase parts (groups)""" - parts = list() - for spart in self.groups: - if self.is_run_group(spart): - parts.append(spart.name[len(self.name)+1:]) - # end if - # end for - return parts - - def phase_group(self, phase): - """Return the (non-run) group specified by """ - if phase in self._full_phases: - return self._full_phases[phase] - # end if - raise ParseInternalError("Incorrect phase, '{}'".format(phase)) - - def constituent_dictionary(self): - """Return the constituent dictionary for this suite""" - return self.parent - - def write(self, output_dir, run_env): - """Create caps for all groups in the suite and for the entire suite - (calling the group caps one after another)""" - # Set name of module and filename of cap - filename = '{module_name}.F90'.format(module_name=self.module) - if run_env.verbose: - run_env.logger.debug('Writing CCPP suite file, {}'.format(filename)) - # end if - # Retrieve the name of the constituent module for Group use statements - const_mod = self.parent.constituent_module_name() - # Init - output_file_name = os.path.join(output_dir, filename) - with FortranWriter(output_file_name, 'w', - "CCPP Suite Cap for {}".format(self.name), - self.module) as outfile: - # Write module 'use' statements here - outfile.write('use {}'.format(KINDS_MODULE), 1) - # Look for any DDT types - self.__ddt_library.write_ddt_use_statements(self.values(), - outfile, 1) - # Write out constituent module use statement(s) - const_dict = self.constituent_dictionary() - const_dict.write_suite_use(outfile, 1) - outfile.write_preamble() - outfile.write('! Suite interfaces', 1) - line = Suite.__state_machine_init - var_name = Suite.__state_machine_var_name - var_state = Suite.__state_machine_initial_state - outfile.write(line.format(css_var_name=var_name, - state=var_state), 1) - for group in self.__groups: - outfile.write('public :: {}'.format(group.name), 1) - # end for - # Declare constituent public interfaces - const_dict.declare_public_interfaces(outfile, 1) - # Declare constituent private suite interfaces and data - const_dict.declare_private_data(outfile, 1) - outfile.write('\n! Private suite variables', 1) - for svar in self.keys(): - self[svar].write_def(outfile, 1, self, allocatable=True) - # end for - outfile.end_module_header() - for group in self.__groups: - if group.name in self._beg_groups: - if group.name == self.__suite_reg_group.name: - group.write(outfile, self.__host_arg_list_noloop, - 1, const_mod, suite_vars=self) - else: - group.write(outfile, self.__host_arg_list_noloop, - 1, const_mod, suite_vars=self, allocate=True) - # end if - elif group.name in self._end_groups: - group.write(outfile, self.__host_arg_list_noloop, - 1, const_mod, suite_vars=self, deallocate=True) - else: - group.write(outfile, self.__host_arg_list_full, 1, - const_mod) - # end if - # end for - err_vars = self.find_error_variables(any_scope=True, - clone_as_out=True) - # Write the constituent properties interface - const_dict.write_constituent_routines(outfile, 1, - self.name, err_vars) - # end with - return output_file_name - -############################################################################### - -class API(VarDictionary): - """Class representing the API for the CCPP framework. - The API class organizes the suites for which CAPS will be generated""" - - __suite_fname = 'ccpp_physics_suite_list' - __part_fname = 'ccpp_physics_suite_part_list' - __vars_fname = 'ccpp_physics_suite_variables' - __schemes_fname = 'ccpp_physics_suite_schemes' - - __file_desc = "API for {host_model} calls to CCPP suites" - - __preamble = ''' -{module_use} -''' - - __sub_name_template = 'ccpp_physics' - - __subhead = 'subroutine {subname}({api_call_list})' - - __subfoot = 'end subroutine {subname}\n' - - # Note, we cannot add these vars to our dictionary as we do not want - # them showing up in group dummy arg lists - __suite_name = Var({'local_name':'suite_name', - 'standard_name':'suite_name', - 'intent':'in', 'type':'character', - 'kind':'len=*', 'units':'', - 'dimensions':'()'}, _API_SOURCE, _API_DUMMY_RUN_ENV) - - __suite_part = Var({'local_name':'suite_part', - 'standard_name':'suite_part', - 'intent':'in', 'type':'character', - 'kind':'len=*', 'units':'', - 'dimensions':'()'}, _API_SOURCE, _API_DUMMY_RUN_ENV) - - def __init__(self, sdfs, host_model, scheme_headers, run_env): - """Initialize this API. - is the list of Suite Definition Files to be parsed for - data needed by the CCPP cap. - is a HostModel object to reference for host model - variables. - is the list of parsed physics scheme metadata files. - Every scheme referenced by an SDF in MUST be in this list, - however, unused schemes are allowed. - is the CCPPFrameworkEnv object for this framework run. - """ - self.__module = 'ccpp_physics_api' - self.__host = host_model - self.__suites = list() - super().__init__(self.module, run_env, parent_dict=self.host_model) - # Create a usable library out of scheme_headers - # Structure is dictionary of dictionaries - # Top-level dictionary is keyed by function name - # Secondary level is by phase - scheme_library = {} - # First, process DDT headers - all_ddts = [d for d in scheme_headers if d.header_type == 'ddt'] - ddt_titles = [d.title for d in all_ddts] - for ddt_title in self.host_model.ddt_lib: - if ddt_title not in ddt_titles: - all_ddts.append(self.host_model.ddt_lib[ddt_title]) - # end if - # end for - self.__ddt_lib = DDTLibrary('{}_api'.format(self.host_model.name), - run_env, ddts=all_ddts) - for header in [d for d in scheme_headers if d.header_type != 'ddt']: - if header.header_type != 'scheme': - if header.header_type == 'module': - errmsg = f"{header.title} is a module metadata header type." - errmsg+=" This is not an allowed CCPP scheme header type." - else: - errmsg = f"{header.title} is an unknown CCPP API metadata header type, {header.header_type}" - # end if - raise CCPPError(errmsg) - # end if - func_id, _, match_trans = \ - CCPP_STATE_MACH.function_match(header.title) - if func_id not in scheme_library: - scheme_library[func_id] = {} - # end if - func_entry = scheme_library[func_id] - if match_trans not in func_entry: - func_entry[match_trans] = header - else: - errmsg = "Duplicate scheme entry, {}" - raise CCPPError(errmsg.format(header.title)) - # end if - # end for - - # Turn the SDF files into Suites - for sdf in sdfs: - # Load the suite definition file to determine the schema version, - # validate the file, and expand nested suites if applicable - _, xml_root = read_xml_file(sdf, run_env.logger) - # We do not have line number information for the XML file - self.__context = ParseContext(filename=sdf) - # Validate the XML file - schema_version = find_schema_version(xml_root) - _ = validate_xml_file(sdf, 'suite', schema_version, run_env.logger) - - # Write the expanded sdf to the capgen output directory. - # This file isn't used by capgen (everything is in memory - # from here onwards), but it is useful for developers/users - # (although the output can also be found in the datatable). - (sdf_path, sdf_name) = os.path.split(sdf) - sdf_expanded = os.path.join(run_env.output_dir, - sdf_name.replace(".xml", "_expanded.xml")) - if schema_version[0] in [1, 2]: - # Preprocess the sdf to expand nested suites - if schema_version[0] == 2: - expand_nested_suites(xml_root, sdf_path, logger=run_env.logger) - # For both versions 1 and 2, write the SDF (expanded for - # version 2, original for version 1) to the current directory - write_xml_file(xml_root, sdf_expanded, run_env.logger) - # Validate the expanded SDF for version 2 - if schema_version[0] == 2: - _ = validate_xml_file(sdf, 'suite', schema_version, run_env.logger) - suite = Suite(sdf, xml_root, self, run_env) - suite.analyze(self.host_model, scheme_library, - self.__ddt_lib, run_env) - self.__suites.append(suite) - else: - errmsg = f"Suite XML schema not supported: " + \ - "root={xml_root.tag}, version={schema_version}" - raise CCPPError(errmsg) - # end if - # end for - - # We will need the correct names for errmsg and errcode - evar = self.host_model.find_variable(standard_name='ccpp_error_message') - if evar is not None: - self._errmsg_var = evar - else: - raise CCPPError('Required variable, ccpp_error_message, not found') - # end if - evar = self.host_model.find_variable(standard_name='ccpp_error_code') - if evar is not None: - self._errcode_var = evar - else: - raise CCPPError('Required variable, ccpp_error_code, not found') - # end if - # We need a call list for every phase - self.__call_lists = {} - for phase in CCPP_STATE_MACH.transitions(): - self.__call_lists[phase] = CallList('API_' + phase, run_env) - self.__call_lists[phase].add_variable(self.suite_name_var, run_env) - if phase == RUN_PHASE_NAME: - self.__call_lists[phase].add_variable(self.suite_part_var, - run_env) - # end if - for suite in self.__suites: - for group in suite.groups: - if group.phase() == phase: - self.__call_lists[phase].add_vars(group.call_list, - run_env, - gen_unique=True) - # end if - # end for - # end for - # end for - - @classmethod - def interface_name(cls, phase): - 'Return the name of an API interface function' - return "{}_{}".format(cls.__sub_name_template, phase) - - def call_list(self, phase): - "Return the appropriate API call list variables" - if phase in self.__call_lists: - return self.__call_lists[phase] - # end if - raise ParseInternalError("Illegal phase, '{}'".format(phase)) - - def write(self, output_dir, run_env): - """Write CCPP API module""" - if not self.suites: - raise CCPPError("No suite specified for generating API") - # end if - api_filenames = list() - # Write out the suite files - for suite in self.suites: - out_file_name = suite.write(output_dir, run_env) - api_filenames.append(out_file_name) - # end for - return api_filenames - - @classmethod - def declare_inspection_interfaces(cls, ofile): - """Declare the API interfaces for the suite inquiry functions""" - ofile.write(f"public :: {API.__suite_fname}", 1) - ofile.write(f"public :: {API.__part_fname}", 1) - ofile.write(f"public :: {API.__vars_fname}", 1) - ofile.write(f"public :: {API.__schemes_fname}", 1) - - def get_errinfo_names(self, base_only=False): - """Return a tuple of error output local names. - If base_only==True, return only the name string of the variable. - If base_only=False, return the local name as a full reference. - If the error variables are intrinsic variables, this makes no - difference, however, for a DDT variable, the full reference is - % while the local name is just .""" - if base_only: - errmsg_name = self._errmsg_var.get_prop_value('local_name') - errcode_name = self._errcode_var.get_prop_value('local_name') - else: - errmsg_name = self._errmsg_var.call_string(self) - errcode_name = self._errcode_var.call_string(self) - # end if - return (errmsg_name, errcode_name) - - @staticmethod - def write_var_set_loop(ofile, varlist_name, var_list, indent, - add_allocate=True, start_index=1, start_var=None): - """Write code to allocate (if is True) and set - to . Elements of are set - beginning at . - """ - if add_allocate: - ofile.write(f"allocate({varlist_name}({len(var_list)}))", indent) - # end if - for ind, var in enumerate(var_list): - if start_var: - ind_str = f"{start_var} + {ind + start_index}" - else: - ind_str = f"{ind + start_index}" - # end if - ofile.write(f"{varlist_name}({ind_str}) = '{var}'", indent) - # end for - - def write_suite_part_list_sub(self, ofile, errmsg_name, errcode_name): - """Write the suite-part list subroutine""" - inargs = f"suite_name, part_list, {errmsg_name}, {errcode_name}" - ofile.write(f"subroutine {API.__part_fname}({inargs})", 1) - oline = "character(len=*), intent(in) :: suite_name" - ofile.write(oline, 2) - oline = "character(len=*), allocatable, intent(out) :: part_list(:)" - ofile.write(oline, 2) - self._errmsg_var.write_def(ofile, 2, self, dummy=True, add_intent="out", - extra_space=11) - self._errcode_var.write_def(ofile, 2, self, dummy=True, add_intent="out", - extra_space=11) - else_str = '' - ename = self._errcode_var.get_prop_value('local_name') - ofile.write(f"{ename} = 0", 2) - ename = self._errmsg_var.get_prop_value('local_name') - ofile.write(f"{ename} = ''", 2) - for suite in self.suites: - oline = "{}if(trim(suite_name) == '{}') then" - ofile.write(oline.format(else_str, suite.name), 2) - API.write_var_set_loop(ofile, 'part_list', suite.part_list(), 3) - else_str = 'else ' - # end for - ofile.write("else", 2) - emsg = f"write({errmsg_name}, '(3a)')" - emsg += "'No suite named ', trim(suite_name), ' found'" - ofile.write(emsg, 3) - ofile.write(f"{errcode_name} = 1", 3) - ofile.write("end if", 2) - ofile.write(f"end subroutine {API.__part_fname}", 1) - - def write_req_vars_sub(self, ofile, errmsg_name, errcode_name): - """Write the required variables subroutine""" - oline = "suite_name, variable_list, {errmsg}, {errcode}" - oline += ", input_vars, output_vars, struct_elements" - inargs = oline.format(errmsg=errmsg_name, errcode=errcode_name) - ofile.write("\nsubroutine {}({})".format(API.__vars_fname, inargs), 1) - ofile.write("! Dummy arguments", 2) - oline = "character(len=*), intent(in) :: suite_name" - ofile.write(oline, 2) - oline = "character(len=*), allocatable, intent(out) :: variable_list(:)" - ofile.write(oline, 2) - self._errmsg_var.write_def(ofile, 2, self, dummy=True, - add_intent="out", extra_space=11) - self._errcode_var.write_def(ofile, 2, self, dummy=True, - add_intent="out", extra_space=11) - oline = "logical, optional, intent(in) :: input_vars" - ofile.write(oline, 2) - oline = "logical, optional, intent(in) :: output_vars" - ofile.write(oline, 2) - oline = "logical, optional, intent(in) :: struct_elements" - ofile.write(oline, 2) - ofile.write("! Local variables", 2) - ofile.write("logical {}:: input_vars_use".format(' '*34), 2) - ofile.write("logical {}:: output_vars_use".format(' '*34), 2) - ofile.write("logical {}:: struct_elements_use".format(' '*34), 2) - ofile.write("integer {}:: num_vars".format(' '*34), 2) - ofile.write("", 0) - ename = self._errcode_var.get_prop_value('local_name') - ofile.write("{} = 0".format(ename), 2) - ename = self._errmsg_var.get_prop_value('local_name') - ofile.write("{} = ''".format(ename), 2) - ofile.write("if (present(input_vars)) then", 2) - ofile.write("input_vars_use = input_vars", 3) - ofile.write("else", 2) - ofile.write("input_vars_use = .true.", 3) - ofile.write("end if", 2) - ofile.write("if (present(output_vars)) then", 2) - ofile.write("output_vars_use = output_vars", 3) - ofile.write("else", 2) - ofile.write("output_vars_use = .true.", 3) - ofile.write("end if", 2) - ofile.write("if (present(struct_elements)) then", 2) - ofile.write("struct_elements_use = struct_elements", 3) - ofile.write("else", 2) - ofile.write("struct_elements_use = .true.", 3) - ofile.write("end if", 2) - else_str = '' - for suite in self.suites: - parent = suite.parent - # Collect all the suite variables - oline = "{}if(trim(suite_name) == '{}') then" - input_vars = [set(), set(), set()] # leaves, arrays, leaf elements - inout_vars = [set(), set(), set()] # leaves, arrays, leaf elements - output_vars = [set(), set(), set()] # leaves, arrays, leaf elements - const_initialized_in_physics = {} - for part in suite.groups: - for var in part.call_list.variable_list(): - phase = part.phase() - stdname = var.get_prop_value("standard_name") - intent = var.get_prop_value("intent") - protected = var.get_prop_value("protected") - constituent = var.is_constituent() - if stdname not in const_initialized_in_physics: - const_initialized_in_physics[stdname] = False - # end if - if (parent is not None) and (not protected): - pvar = parent.find_variable(standard_name=stdname) - if pvar is not None: - protected = pvar.get_prop_value("protected") - # end if - # end if - elements = var.intrinsic_elements(check_dict=self.parent, - ddt_lib=self.__ddt_lib) - if (intent == 'in') and (not protected) and (not const_initialized_in_physics[stdname]): - if isinstance(elements, list): - input_vars[1].add(stdname) - input_vars[2].update(elements) - else: - input_vars[0].add(stdname) - # end if - elif intent == 'inout' and (not const_initialized_in_physics[stdname]): - if isinstance(elements, list): - inout_vars[1].add(stdname) - inout_vars[2].update(elements) - else: - inout_vars[0].add(stdname) - # end if - elif constituent and (intent == 'out' and phase != 'initialize' and not - const_initialized_in_physics[stdname]): - # constituents HAVE to be initialized in the init phase because the dycore needs to advect them - emsg = f"constituent variable '{stdname}' cannot be initialized in the '{phase}' phase" - raise CCPPError(emsg) - elif intent == 'out' and constituent and phase == 'initialize': - const_initialized_in_physics[stdname] = True - elif intent == 'out': - if isinstance(elements, list): - output_vars[1].add(stdname) - output_vars[2].update(elements) - else: - output_vars[0].add(stdname) - # end if - # end if - # end for - # end for - # Figure out how many total variables to return and allocate - # variable_list to that size - ofile.write(oline.format(else_str, suite.name), 2) - ofile.write("if (input_vars_use .and. output_vars_use) then", 3) - have_elems = input_vars[2] or inout_vars[2] or output_vars[2] - if have_elems: - ofile.write("if (struct_elements_use) then", 4) - numvars = len(input_vars[0] | input_vars[2] | inout_vars[0] | - inout_vars[2] | output_vars[0] | output_vars[2]) - ofile.write("num_vars = {}".format(numvars), 5) - ofile.write("else", 4) - # end if - numvars = len(input_vars[0] | input_vars[1] | inout_vars[0] | - inout_vars[1] | output_vars[0] | output_vars[1]) - ofile.write("num_vars = {}".format(numvars), 5 if have_elems else 4) - if have_elems: - ofile.write("end if", 4) - # end if - ofile.write("else if (input_vars_use) then", 3) - have_elems = input_vars[2] or inout_vars[2] - if have_elems: - ofile.write("if (struct_elements_use) then", 4) - numvars = len(input_vars[0] | input_vars[2] | - inout_vars[0] | inout_vars[2]) - ofile.write("num_vars = {}".format(numvars), 5) - ofile.write("else", 4) - # end if - numvars = len(input_vars[0] | input_vars[1] | - inout_vars[0] | inout_vars[1]) - ofile.write("num_vars = {}".format(numvars), 5 if have_elems else 4) - if have_elems: - ofile.write("end if", 4) - # end if - ofile.write("else if (output_vars_use) then", 3) - have_elems = inout_vars[2] or output_vars[2] - if have_elems: - ofile.write("if (struct_elements_use) then", 4) - numvars = len(inout_vars[0] | inout_vars[2] | - output_vars[0] | output_vars[2]) - ofile.write("num_vars = {}".format(numvars), 5) - ofile.write("else", 4) - # end if - numvars = len(inout_vars[0] | inout_vars[1] | - output_vars[0] | output_vars[1]) - ofile.write("num_vars = {}".format(numvars), 5 if have_elems else 4) - if have_elems: - ofile.write("end if", 4) - # end if - ofile.write("else", 3) - ofile.write("num_vars = 0", 4) - ofile.write("end if", 3) - ofile.write("allocate(variable_list(num_vars))", 3) - # Now, fill in the variable_list array - # Start with inout variables - elem_start = 1 - leaf_start = 1 - leaf_written_set = inout_vars[0].copy() - elem_written_set = inout_vars[0].copy() - leaf_list = sorted(inout_vars[0]) - if inout_vars[0] or inout_vars[1] or inout_vars[2]: - ofile.write("if (input_vars_use .or. output_vars_use) then", 3) - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 4, - add_allocate=False, - start_index=leaf_start) - # end if - leaf_start += len(leaf_list) - elem_start += len(leaf_list) - # elements which have not been written out - elem_list = sorted(inout_vars[2] - elem_written_set) - elem_written_set = elem_written_set | inout_vars[2] - leaf_list = sorted(inout_vars[1] - leaf_written_set) - leaf_written_set = leaf_written_set | inout_vars[1] - if elem_list or leaf_list: - ofile.write("if (struct_elements_use) then", 4) - API.write_var_set_loop(ofile, 'variable_list', elem_list, 5, - add_allocate=False, - start_index=elem_start) - elem_start += len(elem_list) - ofile.write("num_vars = {}".format(elem_start - 1), 5) - ofile.write("else", 4) - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 5, - add_allocate=False, - start_index=leaf_start) - leaf_start += len(leaf_list) - ofile.write("num_vars = {}".format(leaf_start - 1), 5) - ofile.write("end if", 4) - else: - ofile.write("num_vars = {}".format(len(leaf_written_set)), - 4 if leaf_written_set else 3) - # end if - if inout_vars[0] or inout_vars[1] or inout_vars[2]: - ofile.write("end if", 3) - # end if - # Write input variables - leaf_list = sorted(input_vars[0] - leaf_written_set) - # Are there any output variables which are also input variables - # (e.g., for a different part (group) of the suite)? - # We need to collect them now in case is selected - # but not . - leaf_cross_set = output_vars[0] & input_vars[0] - simp_cross_set = (output_vars[1] & input_vars[1]) - leaf_cross_set - elem_cross_set = (output_vars[2] & input_vars[2]) - leaf_cross_set - # Subtract the variables which have already been written out - leaf_cross_list = sorted(leaf_cross_set - leaf_written_set) - simp_cross_list = sorted(simp_cross_set - leaf_written_set) - elem_cross_list = sorted(elem_cross_set - elem_written_set) - # Next move back to processing the input variables - leaf_written_set = leaf_written_set | input_vars[0] - elem_list = sorted(input_vars[2] - elem_written_set) - elem_written_set = elem_written_set | input_vars[0] | input_vars[2] - have_inputs = elem_list or leaf_list - if have_inputs: - ofile.write("if (input_vars_use) then", 3) - # elements which have not been written out - # end if - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 4, - add_allocate=False, start_var="num_vars", - start_index=1) - if leaf_list: - ofile.write("num_vars = num_vars + {}".format(len(leaf_list)), - 4) - # end if - leaf_start += len(leaf_list) - elem_start += len(leaf_list) - leaf_list = input_vars[1].difference(leaf_written_set) - leaf_written_set.union(input_vars[1]) - if elem_list or leaf_list: - ofile.write("if (struct_elements_use) then", 4) - API.write_var_set_loop(ofile, 'variable_list', elem_list, 5, - add_allocate=False, - start_index=elem_start) - elem_start += len(elem_list) - 1 - ofile.write("num_vars = {}".format(elem_start), 5) - ofile.write("else", 4) - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 5, - add_allocate=False, - start_index=leaf_start) - leaf_start += len(leaf_list) - 1 - ofile.write("num_vars = {}".format(leaf_start), 5) - ofile.write("end if", 4) - # end if - if have_inputs: - ofile.write("end if", 3) - # end if - # Write output variables - leaf_list = sorted(output_vars[0].difference(leaf_written_set)) - leaf_written_set = leaf_written_set.union(output_vars[0]) - elem_written_set = elem_written_set.union(output_vars[0]) - elem_list = sorted(output_vars[2].difference(elem_written_set)) - elem_written_set = elem_written_set.union(output_vars[2]) - have_outputs = elem_list or leaf_list - if have_outputs: - ofile.write("if (output_vars_use) then", 3) - # end if - leaf_start = 1 - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 4, - add_allocate=False, start_var="num_vars", - start_index=leaf_start) - leaf_start += len(leaf_list) - elem_start = leaf_start - leaf_list = output_vars[1].difference(leaf_written_set) - leaf_written_set.union(output_vars[1]) - if elem_list or leaf_list: - ofile.write("if (struct_elements_use) then", 4) - API.write_var_set_loop(ofile, 'variable_list', elem_list, 5, - add_allocate=False, start_var="num_vars", - start_index=elem_start) - elem_start += len(elem_list) - ofile.write("else", 4) - API.write_var_set_loop(ofile, 'variable_list', leaf_list, 5, - add_allocate=False, start_var="num_vars", - start_index=leaf_start) - leaf_start += len(leaf_list) - ofile.write("end if", 4) - # end if - if leaf_cross_list or elem_cross_list: - ofile.write("if (.not. input_vars_use) then", 4) - API.write_var_set_loop(ofile, 'variable_list', leaf_cross_list, - 5, add_allocate=False, - start_var="num_vars", - start_index=leaf_start) - leaf_start += len(leaf_cross_list) - elem_start += len(leaf_cross_list) - if elem_cross_list or simp_cross_list: - ofile.write("if (struct_elements_use) then", 5) - API.write_var_set_loop(ofile, 'variable_list', - elem_cross_list, 6, - add_allocate=False, - start_var="num_vars", - start_index=elem_start) - elem_start += len(elem_list) - ofile.write("else", 5) - API.write_var_set_loop(ofile, 'variable_list', - leaf_cross_list, 6, - add_allocate=False, - start_var="num_vars", - start_index=leaf_start) - leaf_start += len(leaf_list) - ofile.write("end if", 5) - # end if - ofile.write("end if", 4) - if have_outputs: - ofile.write("end if", 3) - # end if - else_str = 'else ' - # end for - ofile.write("else", 2) - emsg = "write({errmsg}, '(3a)')".format(errmsg=errmsg_name) - emsg += "'No suite named ', trim(suite_name), ' found'" - ofile.write(emsg, 3) - ofile.write("{errcode} = 1".format(errcode=errcode_name), 3) - ofile.write("end if", 2) - ofile.write("end subroutine {}".format(API.__vars_fname), 1) - - def write_suite_schemes_sub(self, ofile, errmsg_name, errcode_name): - """Write the suite schemes list subroutine""" - oline = "suite_name, scheme_list, {errmsg}, {errcode}" - inargs = oline.format(errmsg=errmsg_name, errcode=errcode_name) - ofile.write("\nsubroutine {}({})".format(API.__schemes_fname, - inargs), 1) - oline = "character(len=*), intent(in) :: suite_name" - ofile.write(oline, 2) - oline = "character(len=*), allocatable, intent(out) :: scheme_list(:)" - ofile.write(oline, 2) - self._errmsg_var.write_def(ofile, 2, self, dummy=True, - add_intent="out", extra_space=11) - self._errcode_var.write_def(ofile, 2, self, dummy=True, - add_intent="out", extra_space=11) - else_str = '' - ename = self._errcode_var.get_prop_value('local_name') - ofile.write("{} = 0".format(ename), 2) - ename = self._errmsg_var.get_prop_value('local_name') - ofile.write("{} = ''".format(ename), 2) - for suite in self.suites: - oline = "{}if(trim(suite_name) == '{}') then" - ofile.write(oline.format(else_str, suite.name), 2) - # Collect the list of schemes in this suite - schemes = set() - for part in suite.groups: - schemes.update([x.name for x in part.schemes()]) - # end for - # Write out the list - API.write_var_set_loop(ofile, 'scheme_list', schemes, 3) - else_str = 'else ' - # end for - ofile.write("else", 2) - emsg = "write({errmsg}, '(3a)')".format(errmsg=errmsg_name) - emsg += "'No suite named ', trim(suite_name), ' found'" - ofile.write(emsg, 3) - ofile.write("{errcode} = 1".format(errcode=errcode_name), 3) - ofile.write("end if", 2) - ofile.write("end subroutine {}".format(API.__schemes_fname), 1) - - def write_inspection_routines(self, ofile): - """Write the list_suites and list_suite_parts subroutines""" - errmsg_name, errcode_name = self.get_errinfo_names(base_only=True) - ofile.write("subroutine {}(suites)".format(API.__suite_fname), 1) - nsuites = len(self.suites) - oline = "character(len=*), allocatable, intent(out) :: suites(:)" - ofile.write(oline, 2) - ofile.write("\nallocate(suites({}))".format(nsuites), 2) - for ind, suite in enumerate(self.suites): - ofile.write("suites({}) = '{}'".format(ind+1, suite.name), 2) - # end for - ofile.write("end subroutine {}".format(API.__suite_fname), 1) - ofile.blank_line() - # Write out the suite part list subroutine - self.write_suite_part_list_sub(ofile, errmsg_name, errcode_name) - # Write out the suite required variable subroutine - self.write_req_vars_sub(ofile, errmsg_name, errcode_name) - # Write out the suite scheme list subroutine - self.write_suite_schemes_sub(ofile, errmsg_name, errcode_name) - - @property - def module(self): - """Return the module name of the API.""" - return self.__module - - @property - def host_model(self): - """Return the host model which will use this API.""" - return self.__host - - @property - def suite_name_var(self): - "Return the name of the variable specifying the suite to run" - return self.__suite_name - - @property - def suite_part_var(self): - "Return the name of the variable specifying the suite group to run" - return self.__suite_part - - @property - def suites(self): - "Return the list of this API's suites" - return self.__suites - -############################################################################### -if __name__ == "__main__": - try: - # First, run doctest - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - # Goal: Replace this test with a suite from unit tests - FRAME_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - TEMP_SUITE = os.path.join(FRAME_ROOT, 'test', 'capgen_test', - 'temp_suite.xml') - if os.path.exists(TEMP_SUITE): - _ = Suite(TEMP_SUITE, VarDictionary('temp_suite', - _API_DUMMY_RUN_ENV), - _API_DUMMY_RUN_ENV) - else: - print("Cannot find test file, '{}', skipping test".format(TEMP_SUITE)) - # end if - sys.exit(fail) - except CCPPError as suite_error: - print("{}".format(suite_error)) - sys.exit(fail) - # end try -# end if diff --git a/scripts/ccpp_track_variables.py b/scripts/ccpp_track_variables.py deleted file mode 100755 index 6d3f6d0b..00000000 --- a/scripts/ccpp_track_variables.py +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env python3 - -# Standard modules -import os -import argparse -import logging -import glob - -# CCPP framework imports -from metadata_table import find_scheme_names, parse_metadata_file -from ccpp_prebuild import import_config, gather_variable_definitions -from mkstatic import Suite -from common import lowercase_keys -from parse_checkers import registered_fortran_ddt_names -from parse_tools import init_log, set_log_level -from framework_env import CCPPFrameworkEnv - -############################################################################### -# Set up the command line argument parser and other global variables # -############################################################################### - -############################################################################### -# Functions and subroutines # -############################################################################### - -def parse_arguments(): - """Parse command line arguments.""" - parser = argparse.ArgumentParser() - parser.add_argument('-s', '--sdf', help='suite definition file to parse', required=True) - parser.add_argument('-m', '--metadata_path', - help='path to CCPP scheme metadata files', required=True) - parser.add_argument('-c', '--config', - help='path to CCPP prebuild configuration file', required=True) - parser.add_argument('-v', '--variable', help='variable to track through CCPP suite', - required=True) - parser.add_argument('--debug', action='store_true', help='enable debugging output', - default=False) - - args = parser.parse_args() - - return args - -def setup_logging(debug): - """Sets up the logging module and logging level.""" - - #Use capgen logging tools - logger = init_log('ccpp_track_variables') - - if debug: - set_log_level(logger, logging.DEBUG) - logger.info('Logging level set to DEBUG') - else: - set_log_level(logger, logging.WARNING) - return logger - -def parse_suite(sdf, run_env): - """Reads the provided sdf, parses into a Suite data structure, including the "call tree": - the ordered list of schemes for the suite specified by the provided sdf""" - run_env.logger.info(f'Reading sdf {sdf} and populating Suite object') - suite = Suite(sdf_name=sdf) - success = suite.parse(make_call_tree=True) - if not success: - raise Exception(f'Parsing suite definition file {sdf} failed.') - run_env.logger.info(f'Successfully read sdf {suite.sdf_name}') - return suite - -def create_metadata_filename_dict(metapath): - """Given a path, read all .meta files in that directory and add them to a dictionary: the keys - are the name of the scheme, and the values are the filename of the .meta file associated - with that scheme""" - - metadata_dict = {} - scheme_filenames = glob.glob(os.path.join(metapath, "*.meta"), recursive=True) - if not scheme_filenames: - raise Exception(f'No files found in {metapath} with ".meta" extension') - - for scheme_fn in scheme_filenames: - schemes = find_scheme_names(scheme_fn) - # The above returns a list of schemes in each filename, but - # we want a dictionary of schemes associated with filenames: - for scheme in schemes: - metadata_dict[scheme.lower()] = scheme_fn - - return metadata_dict - - -def create_var_graph(suite, var, config, metapath, run_env): - """Given a suite, variable name, a 'config' dictionary, and a path to .meta files: - 1. Creates a dictionary associating schemes with their .meta files - 2. Loops through the call tree of the provided suite by group - 3. For each scheme, reads .meta file for said scheme, checks for variable within that - scheme, and if it exists, adds an entry to a list of tuples for the corresponding - group, where each tuple includes the name of the scheme and the intent of the variable - within that scheme""" - - # Create a list of tuples for each group that will hold the in/out information for each scheme - var_graph={} - var_graph_empty = True - - run_env.logger.debug(f"reading .meta files in path:\n {metapath}") - metadata_dict=create_metadata_filename_dict(metapath) - - # Loop through call tree, find matching filename for scheme via dictionary schemes_in_files, - # then parse that metadata file to find variable info - partial_matches = {} - for group in suite.call_tree: - run_env.logger.debug(f"for group {group} ") - # Create list of tuples that will hold the in/out information for each scheme in this group - var_graph[group] = [] - for scheme in suite.call_tree[group]: - run_env.logger.debug(f"reading meta file for scheme {scheme} ") - - if scheme in metadata_dict: - scheme_filename = metadata_dict[scheme] - else: - raise Exception(f"Error, scheme '{scheme}' from suite '{suite.sdf_name}' " - f"not found in metadata files in {metapath}") - - run_env.logger.debug(f"reading metadata file {scheme_filename} for scheme {scheme}") - - new_metadata_headers = parse_metadata_file(scheme_filename, - known_ddts=registered_fortran_ddt_names(), - run_env=run_env) - for scheme_metadata in new_metadata_headers: - if scheme_metadata.table_name != scheme: - # Some metadata files contain information for multiple schemes, - # need to make sure we only read the relevant section - continue - for section in scheme_metadata.sections(): - found_var = [] - intent = '' - for scheme_var in section.variable_list(): - exact_match = False - if var == scheme_var.get_prop_value('standard_name'): - run_env.logger.debug(f"Found variable {var} in scheme {section.title}") - found_var=var - exact_match = True - intent = scheme_var.get_prop_value('intent') - break - scheme_var_standard_name = scheme_var.get_prop_value('standard_name') - if scheme_var_standard_name.find(var) != -1: - run_env.logger.debug(f"{var} matches {scheme_var_standard_name}") - found_var.append(scheme_var_standard_name) - if not found_var: - run_env.logger.debug(f"Did not find variable {var} in scheme {section.title}") - elif exact_match: - run_env.logger.debug(f"Exact match found for variable {var} in scheme " - f"{section.title}, intent {intent}") - var_graph[group].append((section.title,intent)) - var_graph_empty = False - else: - run_env.logger.debug(f"Found inexact matches for variable(s) {var} " - f"in scheme {section.title}:\n{found_var}") - partial_matches[section.title] = found_var - - - if not var_graph_empty: - success = True - run_env.logger.debug(f"Successfully generated variable graph for sdf {suite.sdf_name}\n") - else: - success = False - run_env.logger.error(f"Variable {var} not found in any suites for sdf {suite.sdf_name}\n") - if partial_matches: - print("Did find partial matches that may be of interest:\n") - for key in partial_matches: - print(f"In {key} found variable(s) {partial_matches[key]}") - - return (success,var_graph) - -def track_variables(sdf,metadata_path,config,variable,debug): - """Main routine that traverses a CCPP suite and outputs the list of schemes that use given - variable, broken down by group - - Args: - sdf (str) : The full path of the suite definition file to parse - metadata_path (str) : path to CCPP scheme metadata files - config (str) : path to CCPP prebuild configuration file - variable (str) : variable to track through CCPP suite - debug (bool) : Enable extra output for debugging - - Returns: - None -""" - - logger = setup_logging(debug) - - #Use new capgen class CCPPFrameworkEnv - run_env = CCPPFrameworkEnv(logger, host_files="", scheme_files="", suites="") - - suite = parse_suite(sdf,run_env) - - (success, config) = import_config(config, None) - if not success: - raise Exception('Call to import_config failed.') - - # Variables defined by the host model; this call is necessary because it converts some old - # metadata formats so they can be used later in the script - (success, _, _) = gather_variable_definitions(config['variable_definition_files'], - config['typedefs_new_metadata']) - if not success: - raise Exception('Call to gather_variable_definitions failed.') - - (success, var_graph) = create_var_graph(suite, variable, config, metadata_path, run_env) - if success: - print(f"For suite {suite.sdf_name}, the following schemes (in order for each group) " - f"use the variable {variable}:") - for group in var_graph: - if var_graph[group]: - print(f"In group {group}") - for entry in var_graph[group]: - print(f" {entry[0]} (intent {entry[1]})") - - -if __name__ == '__main__': - - args = parse_arguments() - - track_variables(args.sdf,args.metadata_path,args.config,args.variable,args.debug) diff --git a/scripts/code_block.py b/scripts/code_block.py deleted file mode 100644 index ccd3f209..00000000 --- a/scripts/code_block.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env python3 -# - -"""Class and methods to create a code block which can then be written -to a file.""" - -# Python library imports -import re -# CCPP framework imports -from parse_tools import ParseContext, ParseSource, context_string -from parse_tools import ParseInternalError - -class CodeBlock(object): - """Class to store a block of code and a method to write it to a file - >>> CodeBlock([]) #doctest: +ELLIPSIS - - >>> CodeBlock(['hi mom']) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: Each element of must contain exactly two items, a code string and a relative indent - >>> CodeBlock([('hi mom')]) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: Each element of must contain exactly two items, a code string and a relative indent - >>> CodeBlock([('hi mom', 'x')]) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: Each element of must contain exactly two items, a code string and a relative indent - >>> CodeBlock([('hi mom', 1)]) #doctest: +ELLIPSIS - - >>> from fortran_tools import FortranWriter - >>> outfile_name = "__code_block_temp.F90" - >>> outfile = FortranWriter(outfile_name, 'w', 'test file', 'test_mod') - >>> CodeBlock([('hi mom', 1)]).write(outfile, 1, {}) - - >>> CodeBlock([('hi {greet} mom', 1)]).write(outfile, 1, {}) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: 'greet' missing from - >>> CodeBlock([('hi {{greet}} mom', 1)]).write(outfile, 1, {}) - >>> CodeBlock([('{greet} there mom', 1)]).write(outfile, 1, {'greet':'hi'}) - >>> outfile.__exit__() - False - >>> import os - >>> os.remove(outfile_name) - """ - - __var_re = re.compile(r"[{][ ]*([A-Za-z][A-Za-z0-9_]*)[ ]*[}]") - - __fmt_msg = ('Each element of must contain exactly two ' - 'items, a code string and a relative indent') - - def __init__(self, code_list): - """Initialize object with a list of statements. - Capture and store all variables required for output. - Each statement is a tuple consisting of a string and an indent level. - Non-negative indents will be added to a current indent at write time - while negative indents are written with no indentation. - """ - self.__code_block = code_list - self.__write_vars = list() - for line in self.__code_block: - if len(line) != 2: - raise ParseInternalError(CodeBlock.__fmt_msg.format(code_list)) - # end if - stmt = line[0] - if not isinstance(stmt, str): - raise ParseInternalError(CodeBlock.__fmt_msg.format(code_list)) - # end if - if not isinstance(line[1], int): - raise ParseInternalError(CodeBlock.__fmt_msg.format(code_list)) - # end if - beg = 0 - end = len(stmt) - while beg < end: - # Ignore double curly braces - open_double_curly = stmt.find('{{', beg) - close_double_curly = stmt.find('}}', - max(open_double_curly, beg)) - if 0 <= open_double_curly < close_double_curly: - beg = close_double_curly + 2 - else: - match = CodeBlock.__var_re.search(stmt[beg:]) - if match: - self.__write_vars.append(match.group(1)) - beg = stmt.index('}', beg) + 1 - else: - beg = end + 1 - # end if - # end if - # end while - # end for - - def write(self, outfile, indent_level, var_dict): - """Write this object's code block to using - as a basic offset. - Format each line using the variables from . - It is an error for to not contain any variable - indicated in the code block.""" - - for line in self.__code_block: - stmt = line[0] - if indent_level >= 0: - indent = indent_level + line[1] - else: - indent = 0 - # end if - # Check that contains all required items - errmsg = '' - sep = '' - for var in self.__write_vars: - if var not in var_dict: - errmsg += "'{}' missing from ".format(sep, var) - sep = '\n' - # end if - # end for - if errmsg: - raise ParseInternalError(errmsg) - # end if - outfile.write(stmt.format(**var_dict), indent) - # end for - -############################################################################### diff --git a/scripts/common.py b/scripts/common.py deleted file mode 100755 index 234e2521..00000000 --- a/scripts/common.py +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env python3 - -from collections import OrderedDict -import keyword -import logging -import os -import re -import subprocess -import sys - -# This dictionary contains short names for the different CCPP stages, -# because Fortran does not allow subroutine names with more than 63 characters -# Important: 'timestep_init' and 'timestep_finalize' need to come first so that -# a pattern match won't pick "init" for a CCPP subroutine name "xyz_timestep_init" -CCPP_STAGES = OrderedDict() -CCPP_STAGES['timestep_init'] = 'tsinit' -CCPP_STAGES['timestep_finalize'] = 'tsfinal' -CCPP_STAGES['init'] = 'init' -CCPP_STAGES['run'] = 'run' -CCPP_STAGES['finalize'] = 'final' - -CCPP_T_INSTANCE_VARIABLE = 'ccpp_t_instance' -CCPP_CONSTANT_ONE = 'ccpp_constant_one' -CCPP_ERROR_CODE_VARIABLE = 'ccpp_error_code' -CCPP_ERROR_MSG_VARIABLE = 'ccpp_error_message' -CCPP_LOOP_COUNTER = 'ccpp_loop_counter' -CCPP_LOOP_EXTENT = 'ccpp_loop_extent' -CCPP_BLOCK_NUMBER = 'ccpp_block_number' -CCPP_BLOCK_COUNT = 'ccpp_block_count' -CCPP_BLOCK_SIZES = 'ccpp_block_sizes' -CCPP_THREAD_NUMBER = 'ccpp_thread_number' -CCPP_THREAD_COUNT = 'ccpp_thread_count' - -CCPP_CHUNK_EXTENT = 'ccpp_chunk_extent' -CCPP_HORIZONTAL_LOOP_BEGIN = 'horizontal_loop_begin' -CCPP_HORIZONTAL_LOOP_END = 'horizontal_loop_end' - -CCPP_HORIZONTAL_LOOP_EXTENT = 'horizontal_loop_extent' -CCPP_HORIZONTAL_DIMENSION = 'horizontal_dimension' - -FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '(', ')', '==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.', - '.true.', '.false.', '.lt.', '.le.', '.eq.', '.ge.', '.gt.', '.ne.', - '.not.', '.and.', '.or.', '.xor.'] -FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([word.replace('(','\(').replace(')', '\)') for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) - -CCPP_TYPE = 'ccpp_t' - -# SCRIPTDIR is the directory where ccpp_prebuild.py and its Python modules are located -SCRIPTDIR = os.path.abspath(os.path.dirname(__file__)) - -# SRCDIR is the directory where the CCPP framework source code (C, Fortran) is located -SRCDIR = os.path.abspath(os.path.join(SCRIPTDIR, os.pardir, 'src')) - -# Definition of variables (metadata tables) that are provided by CCPP -CCPP_INTERNAL_VARIABLE_DEFINITON_FILE = os.path.join(SRCDIR, 'ccpp_types.F90') - -# List of internal variables provided by the CCPP -CCPP_INTERNAL_VARIABLES = { - CCPP_ERROR_CODE_VARIABLE : 'cdata%errflg', - CCPP_ERROR_MSG_VARIABLE : 'cdata%errmsg', - CCPP_LOOP_COUNTER : 'cdata%loop_cnt', - CCPP_BLOCK_NUMBER : 'cdata%blk_no', - CCPP_THREAD_NUMBER : 'cdata%thrd_no', - CCPP_THREAD_COUNT : 'cdata%thrd_cnt', - } - -STANDARD_CHARACTER_TYPE = 'character' -STANDARD_INTEGER_TYPE = 'integer' -STANDARD_VARIABLE_TYPES = [ STANDARD_CHARACTER_TYPE, STANDARD_INTEGER_TYPE, 'logical', 'real' ] - -# For static build -CCPP_STATIC_API_MODULE = 'ccpp_static_api' -CCPP_STATIC_SUBROUTINE_NAME = 'ccpp_physics_{stage}' - -# Filename pattern for suite definition files -SUITE_DEFINITION_FILENAME_PATTERN = re.compile('^(.*)\.xml$') - -# Maximum number of concurrent CCPP instances per MPI task -CCPP_NUM_INSTANCES = 200 - -def split_var_name_and_array_reference(var_name): - """Split an expression like foo(:,a,1:ddt%ngas) - into components foo and (:,a,1:ddt%ngas).""" - actual_var_name = None - array_reference = None - # Search for first pair of parentheses from the end of the string - parentheses = 0 - i = len(var_name)-1 - while i>=0: - if var_name[i] == ')': - parentheses += 1 - elif var_name[i] == '(': - parentheses -= 1 - if parentheses == 0: - actual_var_name = var_name[:i] - array_reference = var_name[i:] - break - i -= 1 - return (actual_var_name, array_reference) - -def encode_container(*args): - """Encodes a container, i.e. the location of a metadata table for CCPP. - Currently, there are three possibilities with different numbers of input - arguments: module, module+typedef, module+scheme+subroutine. Convert all - names to lowercase to support the new case-insensitive capgen parser.""" - if len(args)==3: - container = 'MODULE_{0} SCHEME_{1} SUBROUTINE_{2}'.format(*[arg.lower() for arg in args]) - elif len(args)==2: - container = 'MODULE_{0} TYPE_{1}'.format(*[arg.lower() for arg in args]) - elif len(args)==1: - container = 'MODULE_{0}'.format(*[arg.lower() for arg in args]) - else: - raise Exception("encode_container not implemented for {0} arguments".format(len(args))) - return container - -def decode_container(container): - """Decodes a container, i.e. the description of a location of a metadata table - for CCPP. Currently, there are three possibilities with different numbers of - input arguments: module, module+typedef, module+scheme+subroutine.""" - items = container.split(' ') - if not len(items) in [1, 2, 3]: - raise Exception("decode_container not implemented for {0} items".format(len(items))) - for i in range(len(items)): - items[i] = items[i][:items[i].find('_')] + ' ' + items[i][items[i].find('_')+1:] - return ' '.join(items) - -def decode_container_as_dict(container): - """Decodes a container, i.e. the description of a location of a metadata table - for CCPP. Currently, there are three possibilities with different numbers of - input arguments: module, module+typedef, module+scheme+subroutine. Return - a dictionary with possible keys MODULE, TYPE, SCHEME, SUBROUTINE.""" - items = container.split(' ') - if not len(items) in [1, 2, 3]: - raise Exception("decode_container not implemented for {0} items".format(len(items))) - itemsdict = {} - for i in range(len(items)): - key, value = (items[i][:items[i].find('_')], items[i][items[i].find('_')+1:]) - itemsdict[key] = value - return itemsdict - -def escape_tex(text): - """Substitutes characters for generating LaTeX sources files from Python.""" - return text.replace( - '%', '\%').replace( - '_', '\_').replace( - '&', '\&') - -def isstring(s): - """Return true if a variable is a string""" - return isinstance(s, str) - -def insert_plus_sign_for_positive_exponents(string): - """Parse a string (a unit string) and insert plus (+) signs - for positive exponents where needed""" - # Break up the string by spaces - items = string.split() - # Identify units with positive exponents - # without a plus sign (m2 instead of m+2). - pattern = re.compile(r"([a-zA-Z]+)([0-9]+)") - for index, item in enumerate(items): - match = pattern.match(item) - if match: - items[index] = "+".join(match.groups()) - # Recombine items to string - return " ".join(items) - -def string_to_python_identifier(string): - """Replaces forbidden characters in strings with standard substitutions - so that the result is a valid Python object (variable, function) name. - At this point, it only converts characters found in the units attributes. - A check for allowed characters in Python v2 catches missing conversions.""" - # Replace whitespaces with underscores - string = string.replace(" ","_") - # Replace decimal points with _p_ - string = string.replace(".","_p_") - # Replace dashes and minus sign with _minus_ - string = string.replace("-","_minus_") - # Replace plus signs with _plus_ - string = string.replace("+","_plus_") - # "1" is a valid unit - if string == "1": - string = "one" - # Test that the resulting string is a valid Python identifier - if re.match("[_A-Za-z][_a-zA-Z0-9]*$", string) and not keyword.iskeyword(string): - return string - else: - raise Exception("Resulting string '{0}' is not a valid Python identifier".format(string)) - -# New utilities added 2025/07/25 for dealing with case-insensitivity changes from capgen. Used to convert XML data read with the xml library and other ccpp_prebuild dictionaries - -def lowercase_keys_and_values(d): - """Recursively convert all keys and values in a regular dictionary to lowercase""" - if isinstance(d, dict): - return { - (k.lower() if isinstance(k, str) else k): - lowercase_keys_and_values(v) - for k, v in d.items() - } - elif isinstance(d, list): - return [lowercase_keys_and_values(item) for item in d] - elif isinstance(d, str): - return d.lower() - else: - return d - -def lowercase_keys(d): - """Recursively convert all keys in an OrderedDict to lowercase""" - if isinstance(d, OrderedDict): - new_dict = OrderedDict() - for k, v in d.items(): - new_key = k.lower() if isinstance(k, str) else k - new_dict[new_key] = lowercase_keys(v) - return new_dict - elif isinstance(d, list): - return [lowercase_keys(item) for item in d] - else: - return d - -def lowercase_xml(element): - """Recursively convert XML elements to lowercase""" - # Lowercase the tag name - element.tag = element.tag.lower() - # Lowercase the text content, if it exists - if element.text: - element.text = element.text.lower() - if element.tail: - element.tail = element.tail.lower() - # Lowercase attribute keys and values - element.attrib = {k.lower(): v.lower() for k, v in element.attrib.items()} - # Recurse into child elements - for i in range(len(element)): - element[i] = lowercase_xml(element[i]) - return element diff --git a/scripts/constituents.py b/scripts/constituents.py deleted file mode 100644 index e7e182c0..00000000 --- a/scripts/constituents.py +++ /dev/null @@ -1,798 +0,0 @@ -#!/usr/bin/env python3 - -""" -Class and supporting code to hold all information on CCPP constituent -variables. A constituent variable is defined and maintained by the CCPP -Framework instead of the host model. -The ConstituentVarDict class contains methods to generate the necessary code -to implement this support. -""" - -# CCPP framework imports -from parse_tools import ParseInternalError -from metavar import VarDictionary - -######################################################################## - -CONST_DDT_NAME = "ccpp_model_constituents_t" -CONST_DDT_MOD = "ccpp_constituent_prop_mod" -CONST_PROP_TYPE = "ccpp_constituent_properties_t" -CONST_PROP_PTR_TYPE = "ccpp_constituent_prop_ptr_t" -CONST_OBJ_STDNAME = "ccpp_model_constituents_object" - -######################################################################## - -class ConstituentVarDict(VarDictionary): - """A class to hold all the constituent variables for a CCPP Suite. - Also contains methods to generate the necessary code for runtime - allocation and support for these variables. - """ - - __const_prop_array_name = "ccpp_constituents" - __const_prop_init_name = "ccpp_constituents_initialized" - __const_prop_init_consts = "ccpp_create_constituent_array" - __constituent_type = "suite" - - def __init__(self, name, parent_dict, run_env, variables=None): - """Create a specialized VarDictionary for constituents. - The main difference is functionality to allocate and support - these variables with special functions for the host model. - The main reason for a separate dictionary is that these are not - proper Suite variables but will belong to the host model at run time. - The feature of the VarDictionary class is required - because this dictionary must be connected to a host model. - """ - self.__run_env = run_env - super().__init__(name, run_env, - variables=variables, parent_dict=parent_dict) - - def find_variable(self, standard_name=None, source_var=None, - any_scope=True, clone=None, - search_call_list=False, loop_subst=False): - """Attempt to return the variable matching . - if is None, the standard name from is used. - It is an error to pass both and if - the standard name of is not the same as . - If is True, search parent scopes if not in current scope. - Note: Unlike the version of this method, the case for - CCPP_CONSTANT_VARS is not handled -- it should have been handled - by a lower level. - If the variable is not found but is a constituent variable type, - create the variable in this dictionary - Note that although the argument is accepted for consistency, - cloning is not handled at this level. - If the variable is not found and is not a constituent - variable, return None. - """ - if standard_name is None: - if source_var is None: - emsg = "One of or must be passed." - raise ParseInternalError(emsg) - # end if - standard_name = source_var.get_prop_value('standard_name') - elif source_var is not None: - stest = source_var.get_prop_value('standard_name') - if stest != standard_name: - emsg = ("Only one of or may " + - "be passed.") - raise ParseInternalError(emsg) - # end if - # end if - if standard_name in self: - var = self[standard_name] - elif any_scope and (self.parent is not None): - srch_clist = search_call_list - var = self.parent.find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, clone=None, - search_call_list=srch_clist, - loop_subst=loop_subst) - else: - var = None - # end if - if (var is None) and source_var and source_var.is_constituent(): - # If we did not find the variable and it is a constituent type, - # add a clone of to our dictionary. - # First, maybe do a loop substitution - dims = source_var.get_dimensions() - newdims = list() - for dim in dims: - dstdnames = dim.split(':') - new_dnames = list() - for dstdname in dstdnames: - if dstdname == 'horizontal_loop_extent': - new_dnames.append('horizontal_dimension') - elif dstdname == 'horizontal_loop_end': - new_dnames.append('horizontal_dimension') - elif dstdname == 'horizontal_loop_begin': - new_dnames.append('ccpp_constant_one') - else: - new_dnames.append(dstdname) - # end if - # end for - newdims.append(':'.join(new_dnames)) - # end for - var = source_var.clone({'dimensions' : newdims}, remove_intent=True, - source_type=self.__constituent_type) - self.add_variable(var, self.__run_env) - return var - - @staticmethod - def __init_err_var(evar, outfile, indent): - """If is a known error variable, generate the code to - initialize it as an output variable. - If unknown, simply ignore. - """ - stdname = evar.get_prop_value('standard_name') - if stdname == 'ccpp_error_message': - lname = evar.get_prop_value('local_name') - outfile.write(f"{lname} = ''", indent) - elif stdname == 'ccpp_error_code': - lname = evar.get_prop_value('local_name') - outfile.write(f"{lname} = 0", indent) - # end if (no else, just ignore) - - def declare_public_interfaces(self, outfile, indent): - """Declare the public constituent interfaces. - Declarations are written to at indent, .""" - outfile.write("! Public interfaces for handling constituents", indent) - outfile.write("! Return the number of constituents for this suite", - indent) - outfile.write(f"public :: {self.num_consts_funcname()}", indent) - outfile.write("! Return the name of a constituent", indent) - outfile.write(f"public :: {self.const_name_subname()}", indent) - outfile.write("! Copy the data for a constituent", indent) - outfile.write(f"public :: {self.copy_const_subname()}", indent) - - def declare_private_data(self, outfile, indent): - """Declare private suite module variables and interfaces - to with indent, .""" - outfile.write("! Private constituent module data", indent) - if self: - stmt = f"type({CONST_PROP_TYPE}), private, allocatable :: {self.constituent_prop_array_name()}(:)" - outfile.write(stmt, indent) - # end if - stmt = f"logical, private :: {self.constituent_prop_init_name()} = .false." - outfile.write(stmt, indent) - outfile.write("! Private interface for constituents", indent) - stmt = f"private :: {self.constituent_prop_init_consts()}" - outfile.write(stmt, indent) - - @classmethod - def __errcode_names(cls, err_vars): - """Return the ( ) where is the local name - for ccpp_error_code in and is the local name for - ccpp_error_message in . - if either variable is not found in , return None.""" - errcode = None - errmsg = None - for evar in err_vars: - stdname = evar.get_prop_value('standard_name') - if stdname == 'ccpp_error_code': - errcode = evar.get_prop_value('local_name') - elif stdname == 'ccpp_error_message': - errmsg = evar.get_prop_value('local_name') - else: - emsg = f"Bad errcode variable, '{stdname}'" - raise ParseInternalError(emsg) - # end if - # end for - if (not errcode) or (not errmsg): - raise ParseInternalError("Unsupported error scheme") - # end if - return errcode, errmsg - - @staticmethod - def __errcode_callstr(errcode_name, errmsg_name, suite): - """Create and return the error code calling string for . - is the calling routine's ccpp_error_code variable name. - is the calling routine's ccpp_error_message variable name. - """ - err_vars = suite.find_error_variables(any_scope=True, clone_as_out=True) - errcode, errmsg = ConstituentVarDict.__errcode_names(err_vars) - errvar_str = f"{errcode}={errcode_name}, {errmsg}={errmsg_name}" - return errvar_str - - def _write_init_check(self, outfile, indent, suite_name, - err_vars, use_errcode): - """Write a check to to make sure the constituent properties - are initialized. Write code to initialize the error variables and/or - set them to error values.""" - outfile.write('', 0) - if use_errcode: - errcode, errmsg = self.__errcode_names(err_vars) - outfile.write(f"{errcode} = 0", indent+1) - outfile.write(f"{errmsg} = ''", indent+1) - else: - raise ParseInternalError("Alternative to errcode not implemented") - # end if - outfile.write("! Make sure that our constituent array is initialized", - indent+1) - stmt = f"if (.not. {self.constituent_prop_init_name()}) then" - outfile.write(stmt, indent+1) - if use_errcode: - outfile.write(f"{errcode} = 1", indent+2) - stmt = f'{errmsg} = "constituent properties not ' - stmt += f'initialized for suite, {suite_name}"' - outfile.write(stmt, indent+2) - outfile.write("end if", indent+1) - # end if (no else until an alternative error mechanism supported) - - def _write_index_check(self, outfile, indent, suite_name, - err_vars, use_errcode): - """Write a check to to make sure the "index" input - is in bounds. Write code to set error variables if index is - out of bounds.""" - if use_errcode: - errcode, errmsg = self.__errcode_names(err_vars) - if self: - outfile.write("if (index < 1) then", indent+1) - outfile.write(f"{errcode} = 1", indent+2) - stmt = f"write({errmsg}, '(a,i0,a)') 'ERROR: index (',index,') " - stmt += "too small, must be >= 1'" - outfile.write(stmt, indent+2) - stmt = f"else if (index > SIZE({self.constituent_prop_array_name()})) then" - outfile.write(stmt, indent+1) - outfile.write(f"{errcode} = 1", indent+2) - stmt = f"write({errmsg}, '(2(a,i0))') 'ERROR: index (',index,') " - stmt += f"too large, must be <= ', SIZE({self.constituent_prop_array_name()})" - outfile.write(stmt, indent+2) - outfile.write("end if", indent+1) - else: - outfile.write(f"{errcode} = 1", indent+1) - stmt = f"write({errmsg}, '(a,i0,a)') 'ERROR: {self.name}, " - stmt += "has no constituents'" - outfile.write(stmt, indent+1) - # end if - else: - raise ParseInternalError("Alternative to errcode not implemented") - # end if - - def write_constituent_routines(self, outfile, indent, suite_name, err_vars): - """Write the subroutine that, when called allocates and defines the - suite-cap module variable describing the constituent species for - this suite. - Code is written to starting at indent, .""" - # Format our error variables - errvar_names = {x.get_prop_value('standard_name') : - x.get_prop_value('local_name') for x in err_vars} - errcode_snames = ('ccpp_error_code', 'ccpp_error_message') - use_errcode = all([x.get_prop_value('standard_name') in errcode_snames - for x in err_vars]) - errvar_alist = ", ".join([x for x in errvar_names.values()]) - errvar_alist2 = f", {errvar_alist}" if errvar_alist else "" - call_vnames = {'ccpp_error_code' : 'errcode', - 'ccpp_error_message' : 'errmsg'} - errvar_call = ", ".join([f"{call_vnames[x]}={errvar_names[x]}" for x in errcode_snames]) - errvar_call2 = f", {errvar_call}" if errvar_call else "" - local_call = ", ".join([f"{errvar_names[x]}={errvar_names[x]}" for x in errcode_snames]) - # Allocate and define constituents - stmt = f"subroutine {self.constituent_prop_init_consts()}({errvar_alist})" - outfile.write(stmt, indent) - outfile.write("! Allocate and fill the constituent property array", - indent + 1) - outfile.write("! for this suite", indent+1) - outfile.write("! Dummy arguments", indent+1) - for evar in err_vars: - evar.write_def(outfile, indent+1, self, dummy=True) - # end for - # Figure out how many unique (non-tendency) constituent variables we have - const_num = 0 - for std_name, _ in self.items(): - if not std_name.startswith('tendency_of_'): - const_num += 1 - # end if - # end for - if self: - outfile.write("! Local variables", indent+1) - outfile.write("integer :: index", indent+1) - stmt = f"allocate({self.constituent_prop_array_name()}({const_num}))" - outfile.write(stmt, indent+1) - outfile.write("index = 0", indent+1) - # end if - for evar in err_vars: - self.__init_err_var(evar, outfile, indent+1) - # end for - for std_name, var in self.items(): - if std_name.startswith('tendency_of_'): - # Skip tendency variables - continue - # end if - outfile.write("index = index + 1", indent+1) - long_name = var.get_prop_value('long_name') - diag_name = var.get_prop_value('diagnostic_name') - units = var.get_prop_value('units') - dims = var.get_dim_stdnames() - default_value = var.get_prop_value('default_value') - if 'vertical_layer_dimension' in dims: - vertical_dim = 'vertical_layer_dimension' - elif 'vertical_interface_dimension' in dims: - vertical_dim = 'vertical_interface_dimension' - else: - vertical_dim = '' - # end if - advect_str = self.TF_string(var.get_prop_value('advected')) - init_args = [f'{std_name=}', f'{long_name=}', f'{diag_name=}', - f'{units=}', f'{vertical_dim=}', - f'advected={advect_str}', - f'errcode={errvar_names["ccpp_error_code"]}', - f'errmsg={errvar_names["ccpp_error_message"]}'] - if default_value is not None and default_value != '': - init_args.append(f'default_value={default_value}') - stmt = f'call {self.constituent_prop_array_name()}(index)%instantiate({", ".join(init_args)})' - outfile.write(f'if ({errvar_names["ccpp_error_code"]} == 0) then', indent+1) - outfile.write(stmt, indent+2) - outfile.write("end if", indent+1) - # end for - outfile.write(f"{self.constituent_prop_init_name()} = .true.", indent+1) - stmt = f"end subroutine {self.constituent_prop_init_consts()}" - outfile.write(stmt, indent) - outfile.write("", 0) - border = "="*72 - outfile.write(f"\n! {border}\n", 1) - # Return number of constituents - fname = self.num_consts_funcname() - outfile.write(f"integer function {fname}({errvar_alist})", indent) - outfile.write("! Return the number of constituents for this suite", - indent+1) - outfile.write("! Dummy arguments", indent+1) - for evar in err_vars: - evar.write_def(outfile, indent+1, self, dummy=True) - # end for - for evar in err_vars: - self.__init_err_var(evar, outfile, indent+1) - # end for - outfile.write("! Make sure that our constituent array is initialized", - indent+1) - stmt = f"if (.not. {self.constituent_prop_init_name()}) then" - outfile.write(stmt, indent+1) - outfile.write(f"call {self.constituent_prop_init_consts()}({local_call})", indent+2) - outfile.write("end if", indent+1) - outfile.write(f"{fname} = {const_num}", indent+1) - outfile.write(f"end function {fname}", indent) - outfile.write(f"\n! {border}\n", 1) - # Return the name of a constituent given an index - stmt = f"subroutine {self.const_name_subname()}(index, name_out{errvar_alist2})" - outfile.write(stmt, indent) - outfile.write("! Return the name of constituent, ", indent+1) - outfile.write("! Dummy arguments", indent+1) - outfile.write("integer, intent(in) :: index", indent+1) - outfile.write("character(len=*), intent(out) :: name_out", indent+1) - for evar in err_vars: - evar.write_def(outfile, indent+1, self, dummy=True) - # end for - self._write_init_check(outfile, indent, suite_name, - err_vars, use_errcode) - self._write_index_check(outfile, indent, suite_name, - err_vars, use_errcode) - if self: - init_args = ['std_name=name_out', - f'errcode={errvar_names["ccpp_error_code"]}', - f'errmsg={errvar_names["ccpp_error_message"]}'] - stmt = f"call {self.constituent_prop_array_name()}(index)%standard_name({', '.join(init_args)})" - outfile.write(stmt, indent+1) - # end if - outfile.write(f"end subroutine {self.const_name_subname()}", indent) - outfile.write(f"\n! {border}\n", 1) - # Copy a consitituent's properties - fname = self.copy_const_subname() - stmt = f"subroutine {fname}(index, cnst_out{errvar_alist2})" - outfile.write(stmt, indent) - outfile.write("! Copy the data for a constituent", indent+1) - outfile.write("! Dummy arguments", indent+1) - outfile.write("integer, intent(in) :: index", indent+1) - stmt = f"type({CONST_PROP_TYPE}), intent(out) :: cnst_out" - outfile.write(stmt, indent+1) - for evar in err_vars: - evar.write_def(outfile, indent+1, self, dummy=True) - # end for - self._write_init_check(outfile, indent, suite_name, - err_vars, use_errcode) - self._write_index_check(outfile, indent, suite_name, - err_vars, use_errcode) - if self: - stmt = f"cnst_out = {self.constituent_prop_array_name()}(index)" - outfile.write(stmt, indent+1) - # end if - outfile.write(f"end subroutine {fname}", indent) - - def constituent_module_name(self): - """Return the name of host model constituent module""" - if not ((self.parent is not None) and - hasattr(self.parent.parent, "constituent_module")): - emsg = "ConstituentVarDict parent not HostModel?" - emsg += f"\nparent is '{type_name(self.parent.parent)}'" - raise ParseInternalError(emsg) - # end if - return self.parent.parent.constituent_module - - def num_consts_funcname(self): - """Return the name of the function which returns the number of - constituents for this suite.""" - return f"{self.name}_num_consts" - - def const_name_subname(self): - """Return the name of the routine that returns a constituent's - standard name given an index""" - return f"{self.name}_const_name" - - def copy_const_subname(self): - """Return the name of the routine that returns a copy of a - constituent's metadata given an index""" - return f"{self.name}_copy_const" - - @staticmethod - def constituent_index_name(standard_name): - """Return the index name associated with """ - return f"index_of_{standard_name}" - - @staticmethod - def write_constituent_use_statements(cap, suite_list, indent): - """Write the suite use statements needed by the constituent - initialization routines.""" - maxmod = max([len(s.module) for s in suite_list]) - smod = len(CONST_DDT_MOD) - maxmod = max(maxmod, smod) - use_str = "use {},{} only: {}" - spc = ' '*(maxmod - smod) - cap.write(use_str.format(CONST_DDT_MOD, spc, CONST_PROP_TYPE), indent) - cap.write('! Suite constituent interfaces', indent) - for suite in suite_list: - const_dict = suite.constituent_dictionary() - smod = suite.module - spc = ' '*(maxmod - len(smod)) - fname = const_dict.num_consts_funcname() - cap.write(use_str.format(smod, spc, fname), indent) - fname = const_dict.const_name_subname() - cap.write(use_str.format(smod, spc, fname), indent) - fname = const_dict.copy_const_subname() - cap.write(use_str.format(smod, spc, fname), indent) - # end for - - @staticmethod - def write_host_routines(cap, host, reg_funcname, init_funcname, num_const_funcname, - query_const_funcname, copy_in_funcname, copy_out_funcname, cleanup_funcname, - const_obj_name, dyn_const_names, const_names_name, const_indices_name, - const_array_func, advect_array_func, prop_array_func, - const_index_func, suite_list, err_vars): - """Write out the host model routine which will - instantiate constituent fields for all the constituents in . - is a list of the host model's error variables. - Also write out the following routines: - : Initialize constituent data - : Number of constituents - : Check if standard name matches existing constituent - : Collect constituent fields for host - : Update constituent fields from host - : Return a pointer to the constituent array - : Return a pointer to the advected constituent array - : Return a pointer to the constituent properties array - : Return the index of a provided constituent name - Output is written to . - """ -# XXgoldyXX: v need to generalize host model error var type support - use_errcode = [x.get_prop_value('standard_name') in - ('ccpp_error_code' 'ccpp_error_message') - for x in err_vars] - if not use_errcode: - emsg = "Error object not supported for {}" - raise ParseInternalError(emsg(host.name)) - # end if - herrcode, herrmsg = ConstituentVarDict.__errcode_names(err_vars) - err_dummy_str = f"{herrcode}, {herrmsg}" - obj_err_callstr = f"errcode={herrcode}, errmsg={herrmsg}" -# XXgoldyXX: ^ need to generalize host model error var type support - # First up, the registration routine - substmt = f"subroutine {reg_funcname}" - args = "host_constituents " - stmt = f"{substmt}({args}, {err_dummy_str})" - cap.write(stmt, 1) - cap.comment("Create constituent object for suites in ", 2) - cap.blank_line() - ConstituentVarDict.write_constituent_use_statements(cap, suite_list, 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write(f"type({CONST_PROP_TYPE}), target, intent(in) :: " + \ - "host_constituents(:)", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, - add_intent="out", extra_space=25) - # end for - cap.comment("Local variables", 2) - spc = ' '*37 - cap.write(f"integer{spc} :: num_suite_consts", 2) - cap.write(f"integer{spc} :: num_consts", 2) - cap.write(f"integer{spc} :: index, index_start", 2) - cap.write(f"integer{spc} :: field_ind", 2) - cap.write(f"type({CONST_PROP_TYPE}), pointer :: const_prop => NULL()", 2) - cap.blank_line() - cap.write(f"{herrcode} = 0", 2) - cap.write("num_consts = size(host_constituents, 1)", 2) - for suite in suite_list: - const_dict = suite.constituent_dictionary() - funcname = const_dict.num_consts_funcname() - cap.comment(f"Number of suite constants for {suite.name}", 2) - errvar_str = ConstituentVarDict.__errcode_callstr(herrcode, - herrmsg, suite) - cap.write(f"num_suite_consts = {funcname}({errvar_str})", 2) - cap.write(f"if ({herrcode} /= 0) then", 2) - cap.write("return", 3) - cap.write("end if", 2) - cap.write("num_consts = num_consts + num_suite_consts", 2) - # end for - cap.comment("Initialize constituent data and field object", 2) - stmt = f"call {const_obj_name}%initialize_table(num_consts)" - cap.write(stmt, 2) - # Register host model constituents - cap.comment("Add host model constituent metadata", 2) - cap.write("do index = 1, size(host_constituents, 1)", 2) - cap.write("const_prop => host_constituents(index)", 3) - stmt = f"call {const_obj_name}%new_field(const_prop, {obj_err_callstr})" - cap.write(stmt, 3) - cap.write("nullify(const_prop)", 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - cap.write("end do", 2) - cap.blank_line() - # Register dynamic constituents - cap.comment("Add dynamic constituent properties", 2) - for dyn_const in dyn_const_names: - cap.write(f"do index = 1, size({dyn_const}, 1)", 2) - cap.write(f"const_prop => {dyn_const}(index)", 3) - stmt = f"call {const_obj_name}%new_field(const_prop, {obj_err_callstr})" - cap.write(stmt, 3) - cap.write("nullify(const_prop)", 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - cap.write("end do", 2) - # end for - - # Register suite constituents - for suite in suite_list: - errvar_str = ConstituentVarDict.__errcode_callstr(herrcode, - herrmsg, suite) - cap.comment(f"Add {suite.name} constituent metadata", 2) - const_dict = suite.constituent_dictionary() - funcname = const_dict.num_consts_funcname() - cap.write(f"num_suite_consts = {funcname}({errvar_str})", 2) - cap.write(f"if ({herrcode} /= 0) then", 2) - cap.write("return", 3) - cap.write("end if", 2) - funcname = const_dict.copy_const_subname() - cap.write("do index = 1, num_suite_consts", 2) - cap.write(f"allocate(const_prop, stat={herrcode})", 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write(f'{herrmsg} = "ERROR allocating const_prop"', 4) - cap.write("return", 4) - cap.write("end if", 3) - stmt = f"call {funcname}(index, const_prop, {errvar_str})" - cap.write(stmt, 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - stmt = f"call {const_obj_name}%new_field(const_prop, {obj_err_callstr})" - cap.write(stmt, 3) - cap.write("nullify(const_prop)", 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - cap.write("end do", 2) - cap.blank_line() - # end for - stmt = f"call {const_obj_name}%lock_table({obj_err_callstr})" - cap.write(stmt, 2) - cap.write(f"if ({herrcode} /= 0) then", 2) - cap.write("return", 3) - cap.write("end if", 2) - cap.comment("Set the index for each active constituent", 2) - cap.write(f"do index = 1, SIZE({const_indices_name})", 2) - stmt = f"call {const_obj_name}%const_index(field_ind, {const_names_name}(index), {obj_err_callstr})" - cap.write(stmt, 3) - cap.write(f"if ({herrcode} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - cap.write("if (field_ind > 0) then", 3) - cap.write(f"{const_indices_name}(index) = field_ind", 4) - cap.write("else", 3) - cap.write(f"{herrcode} = 1", 4) - stmt = f"{herrmsg} = 'No field index for '//trim({const_names_name}(index))" - cap.write(stmt, 4) - cap.write("return", 4) - cap.write("end if", 3) - cap.write("end do", 2) - cap.write(f"end {substmt}", 1) - # Write constituent_init routine - substmt = f"subroutine {init_funcname}" - cap.blank_line() - cap.write(f"{substmt}(ncols, num_layers, {err_dummy_str})", 1) - cap.comment("Initialize constituent data", 2) - cap.blank_line() - cap.write("use ccpp_scheme_utils, only: ccpp_initialize_constituent_ptr", 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write("integer, intent(in) :: ncols", 2) - cap.write("integer, intent(in) :: num_layers", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, add_intent="out") - # end for evar - cap.blank_line() - call_str = f"call {const_obj_name}%lock_data(ncols, num_layers, {obj_err_callstr})" - cap.write(call_str, 2) - cap.write(f"call ccpp_initialize_constituent_ptr({const_obj_name})", 2) - cap.write(f"end {substmt}", 1) - # Write num_consts routine - substmt = f"subroutine {num_const_funcname}" - cap.blank_line() - cap.write(f"{substmt}(num_flds, advected, {err_dummy_str})", 1) - cap.comment("Return the number of constituent fields for this run", 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write("integer, intent(out) :: num_flds", 2) - cap.write("logical, optional, intent(in) :: advected", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, add_intent="out") - # end for - cap.blank_line() - call_str = f"call {const_obj_name}%num_constituents(num_flds, advected=advected, {obj_err_callstr})" - cap.write(call_str, 2) - cap.write(f"end {substmt}", 1) - # Write query_consts routine - substmt = f"subroutine {query_const_funcname}" - cap.blank_line() - cap.write(f"{substmt}(var_name, constituent_exists, {err_dummy_str})", 1) - cap.comment(f"Return constituent_exists = true iff var_name appears in {host.name}_model_const_stdnames", 2) - cap.blank_line() - cap.write("character(len=*), intent(in) :: var_name", 2) - cap.write("logical, intent(out) :: constituent_exists", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, add_intent="out") - # end for - cap.blank_line() - cap.write(f"{herrcode} = 0", 2) - cap.write(f"{herrmsg} = ''", 2) - cap.blank_line() - cap.write("constituent_exists = .false.", 2) - cap.write(f"if (any({host.name}_model_const_stdnames == var_name)) then", 2) - cap.write("constituent_exists = .true.", 3) - cap.write("end if", 2) - cap.blank_line() - cap.write(f"end {substmt}", 1) - # Write copy_in routine - substmt = f"subroutine {copy_in_funcname}" - cap.blank_line() - cap.write(f"{substmt}(const_array, {err_dummy_str})", 1) - cap.comment("Copy constituent field info into ", 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write("real(kind_phys), intent(out) :: const_array(:,:,:)", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, add_intent="out") - # end for - cap.blank_line() - cap.write(f"call {const_obj_name}%copy_in(const_array, {obj_err_callstr})", 2) - cap.write(f"end {substmt}", 1) - # Write copy_out routine - substmt = f"subroutine {copy_out_funcname}" - cap.blank_line() - cap.write(f"{substmt}(const_array, {err_dummy_str})", 1) - cap.comment("Update constituent field info from ", 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write("real(kind_phys), intent(in) :: const_array(:,:,:)", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, add_intent="out") - # end for - cap.blank_line() - cap.write(f"call {const_obj_name}%copy_out(const_array, {obj_err_callstr})", 2) - cap.write(f"end {substmt}", 1) - # Write cleanup routine - substmt = f"subroutine {cleanup_funcname}" - cap.blank_line() - cap.write(f"{substmt}()", 1) - cap.comment("Deallocate dynamic constituent array", 2) - cap.blank_line() - for dyn_const in dyn_const_names: - cap.write(f"if (allocated({dyn_const})) then", 2) - cap.write(f"deallocate({dyn_const})", 3) - cap.write("end if", 2) - cap.write(f"call {const_obj_name}%reset()", 2) - # end if - cap.write(f"end {substmt}", 1) - # Write constituents routine - cap.blank_line() - cap.write(f"function {const_array_func}() result(const_ptr)", 1) - cap.blank_line() - cap.comment("Return pointer to constituent array", 2) - cap.blank_line() - cap.comment("Dummy argument", 2) - cap.write("real(kind_phys), pointer :: const_ptr(:,:,:)", 2) - cap.blank_line() - cap.write(f"const_ptr => {const_obj_name}%field_data_ptr()", 2) - cap.write(f"end function {const_array_func}", 1) - # Write advected constituents routine - cap.blank_line() - cap.write(f"function {advect_array_func}() result(const_ptr)", 1) - cap.blank_line() - cap.comment("Return pointer to advected constituent array", 2) - cap.blank_line() - cap.comment("Dummy argument", 2) - cap.write("real(kind_phys), pointer :: const_ptr(:,:,:)", 2) - cap.blank_line() - cap.write(f"const_ptr => {const_obj_name}%advected_constituents_ptr()", - 2) - cap.write(f"end function {advect_array_func}", 1) - # Write the constituent property array routine - cap.blank_line() - cap.write(f"function {prop_array_func}() result(const_ptr)", 1) - cap.write(f"use {CONST_DDT_MOD}, only: {CONST_PROP_PTR_TYPE}", 2) - cap.blank_line() - cap.comment("Return pointer to array of constituent properties", 2) - cap.blank_line() - cap.comment("Dummy argument", 2) - cap.write("type(ccpp_constituent_prop_ptr_t), pointer :: const_ptr(:)", - 2) - cap.blank_line() - cap.write(f"const_ptr => {const_obj_name}%constituent_props_ptr()", - 2) - cap.write(f"end function {prop_array_func}", 1) - # Write constituent index function - substmt = f"subroutine {const_index_func}" - cap.blank_line() - cap.write(f"{substmt}(stdname, const_index, {err_dummy_str})", 1) - cap.comment("Set to the constituent array index " + \ - "for .", 2) - cap.comment("If is not found, set to -1 " + \ - "set an error condition", 2) - cap.blank_line() - cap.comment("Dummy arguments", 2) - cap.write("character(len=*), intent(in) :: stdname", 2) - cap.write("integer, intent(out) :: const_index", 2) - for evar in err_vars: - evar.write_def(cap, 2, host, dummy=True, - add_intent="out", extra_space=1) - # end for - cap.blank_line() - cap.write(f"call {const_obj_name}%const_index(const_index, " + \ - f"stdname, {obj_err_callstr})", 2) - cap.write(f"end {substmt}", 1) - - @staticmethod - def constitutent_source_type(): - """Return the source type for constituent species""" - return ConstituentVarDict.__constituent_type - - @staticmethod - def constituent_prop_array_name(): - """Return the name of the constituent properties array for this suite""" - return ConstituentVarDict.__const_prop_array_name - - @staticmethod - def constituent_prop_init_name(): - """Return the name of the array initialized flag for this suite""" - return ConstituentVarDict.__const_prop_init_name - - @staticmethod - def constituent_prop_init_consts(): - """Return the name of the routine to initialize the constituent - properties array for this suite""" - return ConstituentVarDict.__const_prop_init_consts - - @staticmethod - def write_suite_use(outfile, indent): - """Write use statements for any modules needed by the suite cap. - The statements are written to at indent, . - """ - omsg = f"use ccpp_constituent_prop_mod, only: {CONST_PROP_TYPE}" - outfile.write(omsg, indent) - - @staticmethod - def TF_string(tf_val): - """Return a string of the Fortran equivalent of """ - if tf_val: - tf_str = ".true." - else: - tf_str = ".false." - # end if - return tf_str diff --git a/scripts/conversion_tools/__init__.py b/scripts/conversion_tools/__init__.py deleted file mode 100644 index 4842889a..00000000 --- a/scripts/conversion_tools/__init__.py +++ /dev/null @@ -1,54 +0,0 @@ -"""Public API for the conversion_tools library -""" - -__all__ = [ - 'cm__to__m', - 'm__to__cm', - 'mm__to__m', - 'm__to__mm', - 'um__to__m', - 'm__to__um', - 'm__to__km', - 'km__to__m', - 'mm__to__km', - 'km__to__mm', - 's__to__min', - 'min__to__s', - 's__to__h', - 'h__to__s', - 'h__to__d', - 'd__to__h', - 's__to__d', - 'd__to__s', - 'Pa__to__hPa', - 'hPa__to__Pa', - 'm_s_minus_1__to__km_h_minus_1', - 'km_h_minus_1__to__m_s_minus_1', - 'W_m_minus_2__to__erg_cm_minus_2_s_minus_1', - 'erg_cm_minus_2_s_minus_1__to__W_m_minus_2', - ] - -from .unit_conversion import cm__to__m -from .unit_conversion import m__to__cm -from .unit_conversion import mm__to__m -from .unit_conversion import m__to__mm -from .unit_conversion import um__to__m -from .unit_conversion import m__to__um -from .unit_conversion import m__to__km -from .unit_conversion import km__to__m -from .unit_conversion import mm__to__km -from .unit_conversion import km__to__mm -from .unit_conversion import s__to__min -from .unit_conversion import min__to__s -from .unit_conversion import s__to__h -from .unit_conversion import h__to__s -from .unit_conversion import h__to__d -from .unit_conversion import d__to__h -from .unit_conversion import s__to__d -from .unit_conversion import d__to__s -from .unit_conversion import Pa__to__hPa -from .unit_conversion import hPa__to__Pa -from .unit_conversion import m_s_minus_1__to__km_h_minus_1 -from .unit_conversion import km_h_minus_1__to__m_s_minus_1 -from .unit_conversion import W_m_minus_2__to__erg_cm_minus_2_s_minus_1 -from .unit_conversion import erg_cm_minus_2_s_minus_1__to__W_m_minus_2 diff --git a/scripts/conversion_tools/unit_conversion.py b/scripts/conversion_tools/unit_conversion.py deleted file mode 100755 index 25a44cab..00000000 --- a/scripts/conversion_tools/unit_conversion.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 - -"""A pilot version to perform unit conversions. Each conversion must be representable -as a formula where {var} is substituted by the actual variable (scalar, array) to convert, -and {kind} by either _ followed by the kind of the variable, or an emptry string. -This allows having formulars such as func({var}) where func is defined as an elemental -function or as an interface to scalar and array-based functions that perform more -complex conversions than the ones listed here. It is also possible, but in some cases -less performant, to construct conversions for composed units by combining some of the -basic conversions listed here. For instance, one could create a speed conversion from -km h-1 to m s-1 by combining the formulas for km to m and h to min, which will be -slower than boiling it down to a single mathematical expression (see example below).""" - -############ -# Length # -############ - -def mm__to__m(): - """Convert millimeter to meter""" - return '1.0E-3{kind}*{var}' - -def m__to__mm(): - """Convert meter to millimeter""" - return '1.0E+3{kind}*{var}' - -def cm__to__m(): - """Convert centimeter to meter""" - return '1.0E-2{kind}*{var}' - -def m__to__cm(): - """Convert meter to centimeter""" - return '1.0E+2{kind}*{var}' - -def um__to__m(): - """Convert micrometer to meter""" - return '1.0E-6{kind}*{var}' - -def m__to__um(): - """Convert meter to micrometer""" - return '1.0E+6{kind}*{var}' - -def m__to__km(): - """Convert meter to kilometer""" - return '1.0E-3{kind}*{var}' - -def km__to__m(): - """Convert kilometer to meter""" - return '1.0E+3{kind}*{var}' - -def mm__to__km(): - """Convert millimeter to kilometer""" - return '1.0E-6{kind}*{var}' - -def km__to__mm(): - """Convert kilometer to millimeter""" - return '1.0E+6{kind}*{var}' - -############ -# Time # -############ - -def s__to__min(): - """Convert second to minute""" - return '{var}/6.0E+1{kind}' - -def min__to__s(): - """Convert minute to second""" - return '6.0E+1{kind}*{var}' - -def s__to__h(): - """Convert second to hour""" - return '{var}/3.6E+3{kind}' - -def h__to__s(): - """Convert hour to second""" - return '3.6E+3{kind}*{var}' - -def h__to__d(): - """Convert hour to day""" - return '{var}/2.4E+1{kind}' - -def d__to__h(): - """Convert day to hour""" - return '2.4E+1{kind}*{var}' - -def s__to__d(): - """Convert second to day""" - return '{var}/8.64E+4{kind}' - -def d__to__s(): - """Convert day to second""" - return '8.64E+4{kind}*{var}' - -################## -# Temperature # -################## - -def K__to__C(): - """Convert Kelvin to Celcius""" - return '{var}-273.15{kind}' - -def C__to__K(): - """Convert Celcius to Kelvin""" - return '{var}+273.15{kind}' - -################## -# Mass # -################## - -def kg_kg_minus_1__to__g_kg_minus_1(): - """Convert kilogram per kilogram to gram per kilogram""" - return '1.0E+3{kind}*{var}' - -def g_kg_minus_1__to__kg_kg_minus_1(): - """Convert gram per kilogram to kilogram per kilogram""" - return '1.0E-3{kind}*{var}' - -################## -# Plane angle # -################## - -def radian__to__degree(): - """Convert radian to degree""" - return '57.295779513{kind}*{var}' - -def degree__to__radian(): - """Convert degree to radian""" - return '{var}/57.295779513{kind}' - -def radian__to__degree_north(): - """Convert radian to degree north""" - return radian__to__degree() - -def degree_north__to__radian(): - """Convert degree north to radian""" - return degree__to__radian() - -def radian__to__degree_east(): - """Convert radian to degree east""" - return radian__to__degree() - -def degree_east__to__radian(): - """Convert degree east to radian""" - return degree__to__radian() - -################## -# Composed units # -################## - -def Pa__to__hPa(): - """Convert Pascal to Hectopascal""" - return '1.0E-2{kind}*{var}' - -def hPa__to__Pa(): - """Convert Hectopascal to Pascal""" - return '1.0E+2{kind}*{var}' - -def m_s_minus_1__to__km_h_minus_1(): - """Convert meter per second to kilometer per hour. A more expensive - and less accurate option would be to combine the above conversions - for meter to kilometer and second to hours into the following formula: - '({0})/({1})'.format(m__to__km(),s__to__h()) + '*{var}'""" - return '3.6E+0{kind}*{var}' - -def km_h_minus_1__to__m_s_minus_1(): - """Convert kilometer per hour to meter per second. A more expensive - and less accurate option would be to combine the above conversions - for kilometer to meter and hours to second into the following formula: - '({0})/({1})'.format(km__to__m(),h__to__s()) + '*{var}'""" - return '{var}/3.6E+0{kind}' - -def W_m_minus_2__to__erg_cm_minus_2_s_minus_1(): - """Convert Watt per square meter to erg per square centimeter and second.""" - return '1.0E+3{kind}*{var}' - -def erg_cm_minus_2_s_minus_1__to__W_m_minus_2(): - """Convert erg per square centimeter and second to Watt per square meter""" - return '1.0E-3{kind}*{var}' - -#################### -# Equivalent units # -#################### - -def m_plus_2_s_minus_2__to__J_kg_minus_1(): - """Equivalent units""" - return '{var}' - -def J_kg_minus_1__to__m_plus_2_s_minus_2(): - """Equivalent units""" - return '{var}' - -def V_A__to__W(): - """Equivalent units""" - return '{var}' - -def W__to__V_A(): - """Equivalent units""" - return '{var}' - -def N_m_minus_2__to__Pa(): - """Equivalent units""" - return '{var}' - -def Pa__to__N_m_minus_2(): - """Equivalent units""" - return '{var}' diff --git a/scripts/ddt_library.py b/scripts/ddt_library.py deleted file mode 100644 index 1c362108..00000000 --- a/scripts/ddt_library.py +++ /dev/null @@ -1,358 +0,0 @@ -#!/usr/bin/env python3 -# -# Class -# - -"""Module to implement DDT support in the CCPP Framework. -VarDDT is a class to hold all information on a CCPP DDT metadata variable -""" - -# Python library imports -import logging -# CCPP framework imports -from parse_tools import ParseInternalError, CCPPError, context_string -from metavar import Var -from metadata_table import MetadataSection - -############################################################################### - -class VarDDT(Var): - """A class to store a variable that is a component of a DDT (at any - DDT nesting level). - """ - - def __init__(self, new_field, var_ref, run_env, recur=False): - """Initialize a new VarDDT object. - is the DDT component. - is a Var or VarDDT whose root originates in a model - dictionary. - is the CCPPFrameworkEnv object for this framework run. - The structure of the VarDDT object is: - The super class Var object is a copy of the model root Var. - The (a Var) ends up at the end of a VarDDT chain. - """ - self.__field = None - # Grab the info from the root of - source = var_ref.source - super().__init__(var_ref, source, run_env, context=source.context) - # Find the correct place for - if isinstance(var_ref, Var): - # We are at a top level DDT var, set our field - self.__field = new_field - else: - # Recurse to find correct (tail) location for - self.__field = VarDDT(new_field, var_ref.field, run_env, recur=True) - # end if - if ((not recur) and - run_env.verbose): - run_env.logger.debug('Adding DDT field, {}'.format(self)) - # end if - - def is_ddt(self): - """Return True iff is a DDT type.""" - return True - - def get_parent_prop(self, name): - """Return the Var property value for the parent Var object. - """ - return super().get_prop_value(name) - - def get_prop_value(self, name): - """Return the Var property value for the leaf Var object. - """ - if self.field is None: - pvalue = super().get_prop_value(name) - else: - pvalue = self.field.get_prop_value(name) - # end if - return pvalue - - def intrinsic_elements(self, check_dict=None): - """Return the Var intrinsic elements for the leaf Var object. - See Var.intrinsic_elements for details - """ - if self.field is None: - pvalue = super().intrinsic_elements(check_dict=check_dict) - else: - pvalue = self.field.intrinsic_elements(check_dict=check_dict) - # end if - return pvalue - - def clone(self, subst_dict, source_name=None, source_type=None, - context=None): - """Create a clone of this VarDDT object's leaf Var with properties - from overriding this variable's properties. - may also be a string in which case only the local_name - property is changed (to the value of the string). - The optional , , and inputs - allow the clone to appear to be coming from a designated source, - by default, the source and type are the same as this Var (self). - """ - if self.field is None: - clone_var = super().clone(subst_dict, source_name=source_name, - source_type=source_type, context=context) - else: - clone_var = self.field.clone(subst_dict, - source_name=source_name, - source_type=source_type, - context=context) - # end if - return clone_var - - def call_string(self, var_dict, loop_vars=None): - """Return a legal call string of this VarDDT's local name sequence. - """ - # XXgoldyXX: Need to add dimensions to this - call_str = super().get_prop_value('local_name') - if self.field is not None: - call_str += '%' + self.field.call_string(var_dict, - loop_vars=loop_vars) - # end if - return call_str - - def write_def(self, outfile, indent, ddict, allocatable=False, target=False, - dummy=False, add_intent=None, extra_space=0, public=False): - """Write the definition line for this DDT. - The type of this declaration is the type of the Var at the - end of the chain of references.""" - if self.field is None: - super().write_def(outfile, indent, ddict, - allocatable=allocatable, target=target, dummy=dummy, - add_intent=add_intent, extra_space=extra_space, - public=public) - else: - self.field.write_def(outfile, indent, ddict, - allocatable=allocatable, target=target, - dummy=dummy, add_intent=add_intent, - extra_space=extra_space, public=public) - # end if - - @staticmethod - def __var_rep(var, prefix=""): - """Internal helper function for creating VarDDT representations - Create a call string from the local_name and dimensions of . - Optionally, prepend %. - """ - lname = var.get_prop_value('local_name') - ldims = var.get_prop_value('dimensions') - if ldims: - if prefix: - lstr = '{}%{}({})'.format(prefix, lname, ', '.join(ldims)) - else: - lstr = '{}({})'.format(lname, ', '.join(ldims)) - # end if - else: - if prefix: - lstr = '{}%{}'.format(prefix, lname) - else: - lstr = '{}'.format(lname) - # end if - # end if - return lstr - - def __repr__(self): - """Print representation for VarDDT objects""" - # Note, recursion would be messy because of formatting issues - lstr = "" - sep = "" - field = self - while field is not None: - if isinstance(field, VarDDT): - lstr += sep + self.__var_rep(field.var) - field = field.field - elif isinstance(field, Var): - lstr = self.__var_rep(field, prefix=lstr) - field = None - # end if - sep = '%' - # end while - return "".format(lstr) - - def __str__(self): - """Print string for VarDDT objects""" - return self.__repr__() - - @property - def var(self): - "Return this VarDDT's Var object" - return super() - - @property - def field(self): - "Return this objects field object, or None" - return self.__field - -############################################################################### -class DDTLibrary(dict): - """DDTLibrary is a collection of DDT definitions, broken down into - individual fields with metadata. It provides efficient ways to find - the field corresponding to any standard-named field contained in - any of the (potentially nested) included DDT definitions. - The dictionary holds known standard names. - """ - - def __init__(self, name, run_env, ddts=None): - "Our dict is DDT definition headers, key is type" - self.__name = '{}_ddt_lib'.format(name) -# XXgoldyXX: v remove? -# self.__ddt_fields = {} # DDT field to DDT access map -# XXgoldyXX: ^ remove? - self.__max_mod_name_len = 0 - self.__run_env = run_env - super().__init__() - if ddts is None: - ddts = list() - elif not isinstance(ddts, list): - ddts = [ddts] - # end if - # Add all the DDT headers, then process - for ddt in ddts: - if not isinstance(ddt, MetadataSection): - errmsg = 'Invalid DDT metadata type, {}' - raise ParseInternalError(errmsg.format(type(ddt).__name__)) - # end if - if not ddt.header_type == 'ddt': - errmsg = 'Metadata table header is for a {}, should be DDT' - raise ParseInternalError(errmsg.format(ddt.header_type)) - # end if - if ddt.title in self: - errmsg = "Duplicate DDT, {}, found{}, original{}" - ctx = context_string(ddt.context) - octx = context_string(self[ddt.title].context) - raise CCPPError(errmsg.format(ddt.title, ctx, octx)) - # end if - if run_env.verbose: - lmsg = f"Adding DDT {ddt.title} to {self.name}" - run_env.logger.debug(lmsg) - # end if - self[ddt.title] = ddt - dlen = len(ddt.module) - if dlen > self.__max_mod_name_len: - self.__max_mod_name_len = dlen - # end if - # end for - - def check_ddt_type(self, var, header, lname=None): - """If is a DDT, check to make sure it is in this DDT library. - If not, raise an exception. - """ - if var.is_ddt(): - # Make sure we know this DDT type - vtype = var.get_prop_value('type') - if vtype not in self: - if lname is None: - lname = var.get_prop_value('local_name') - # end if - errmsg = 'Variable {} is of unknown type ({}) in {}' - ctx = context_string(var.context) - raise CCPPError(errmsg.format(lname, vtype, header.title, ctx)) - # end if - # end if (no else needed) - - def collect_ddt_fields(self, var_dict, var, run_env, - ddt=None, skip_duplicates=False, parent=None): - """Add all the reachable fields from DDT variable of type, - to . Each field is added as a VarDDT. - If , add VarDDT recursively using parent. - Note: By default, it is an error to try to add a duplicate - field to (i.e., the field already exists in - or one of its parents). To simply skip duplicate - fields, set to True. - """ - if ddt is None: - vtype = var.get_prop_value('type') - if vtype in self: - ddt = self[vtype] - else: - lname = var.get_prop_value('local_name') - ctx = context_string(var.context) - errmsg = "Variable, {}, is not a known DDT{}" - raise ParseInternalError(errmsg.format(lname, ctx)) - # end if - # end if - for dvar in ddt.variable_list(): - if parent is None: - subvar = VarDDT(dvar, var, self.run_env) - else: - subvar = VarDDT(VarDDT(dvar, var, self.run_env), parent, self.run_env) - # end if - dvtype = dvar.get_prop_value('type') - if (dvar.is_ddt()) and (dvtype in self): - # If DDT in our library, we need to add sub-fields recursively. - subddt = self[dvtype] - self.collect_ddt_fields(var_dict, dvar, run_env, parent=var, ddt=subddt) - # end if - # add_variable only checks the current dictionary. By default, - # for a DDT, the variable also cannot be in our parent - # dictionaries. - stdname = dvar.get_prop_value('standard_name') - pvar = var_dict.find_variable(standard_name=stdname, any_scope=True) - if pvar and (not skip_duplicates): - ntx = context_string(dvar.context) - ctx = context_string(pvar.context) - emsg = f"Attempt to add duplicate DDT sub-variable, {stdname}{ntx}." - emsg += f"\nVariable originally defined{ctx}" - raise CCPPError(emsg.format(stdname, ntx, ctx)) - # end if - # Add this intrinsic to - if not pvar: - var_dict.add_variable(subvar, run_env) - # end if - # end for - - def ddt_modules(self, variable_list, ddt_mods=None): - """Collect information for module use statements. - Add module use information (module name, DDT name) for any variable - in which is a DDT in this library. - """ - if ddt_mods is None: - ddt_mods = set() # Need a new set for every call - # end if - for var in variable_list: - vtype = var.get_prop_value('type') - if vtype in self: - module = self[vtype].module - ddt_mods.add((module, vtype)) - # end if - # end for - return ddt_mods - - def write_ddt_use_statements(self, variable_list, outfile, indent, pad=0): - """Write the use statements for all ddt modules needed by - """ - pad = max(pad, self.__max_mod_name_len) - ddt_mods = self.ddt_modules(variable_list) - for ddt_mod in ddt_mods: - dmod = ddt_mod[0] - dtype = ddt_mod[1] - slen = ' '*(pad - len(dmod)) - ustring = 'use {},{} only: {}' - outfile.write(ustring.format(dmod, slen, dtype), indent) - # end for - - @property - def name(self): - """Return the name of this DDT library""" - return self.__name - - @property - def run_env(self): - """Return the CCPPFrameworkEnv object for this DDT library""" - return self.__run_env - - @property - def max_mod_name_len(self): - """Return the maximum module name length of this DDT library's modules""" - return self.__max_mod_name_len - -############################################################################### -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/file_utils.py b/scripts/file_utils.py deleted file mode 100644 index 1cad8338..00000000 --- a/scripts/file_utils.py +++ /dev/null @@ -1,303 +0,0 @@ -#!/usr/bin/env python3 - -""" -Utilities for checking and manipulating file status -""" - -# Python library imports -import filecmp -import glob -import os -# CCPP framework imports -from parse_tools import CCPPError, ParseInternalError - -# Standardize name of generated kinds file and module -KINDS_MODULE = 'ccpp_kinds' -KINDS_FILENAME = '{}.F90'.format(KINDS_MODULE) - -############################################################################### -def check_for_existing_file(filename, description, readable=True): -############################################################################### - """Check for file existence and access. - Return a list of error strings in case - does not exist or does not have read access and is True""" - errors = list() - if os.path.exists(filename): - if readable: - if not os.access(filename, os.R_OK): - errmsg = "No read access to {}, '{}'" - errors.append(errmsg.format(description, filename)) - # end if (no else, everything is fine) - # end if (no else, everything is fine) - else: - errors.append("{}, '{}', must exist".format(description, filename)) - # end if - return errors - -############################################################################### -def check_for_writeable_file(filename, description): -############################################################################### - """If exists but not writable, raise an error. - If does not exist and its directory is not writable, raise - an error. is a description of .""" - if os.path.exists(filename) and not os.access(filename, os.W_OK): - raise CCPPError("Cannot write {}, '{}'".format(description, filename)) - # end if - if not os.access(os.path.dirname(filename), os.W_OK): - raise CCPPError("Cannot write {}, '{}'".format(description, filename)) - # end if (else just return) - -############################################################################### -def add_unique_files(filepath, pdesc, master_list, logger): -############################################################################### - """Add any new files indicated by to . - Check each file for readability. - Log duplicate files - Return a list of errors found - Wildcards in are expanded""" - errors = list() - for file in glob.glob(filepath): - errs = check_for_existing_file(file, pdesc) - if errs: - errors.extend(errs) - elif file in master_list: - lmsg = "WARNING: Ignoring duplicate file, {}" - logger.warning(lmsg.format(file)) - else: - master_list.append(file) - # end if - # end for - return errors - -############################################################################### -def read_pathnames_from_file(pathsfile, file_type): -############################################################################### - """Read and return path names from . - Convert relative pathnames to use 's directory as root. - Also return a list of any errors encountered - """ - # We want to end up with absolute paths, treat as root location - root_path = os.path.dirname(os.path.abspath(pathsfile)) - file_list = list() - pdesc = '{} pathsnames file'.format(file_type) - errors = check_for_existing_file(pathsfile, pdesc) - pdesc = '{} pathname in {}'.format(file_type, pathsfile) - if not errors: - with open(pathsfile, 'r') as infile: - for line in infile.readlines(): - path = line.strip() - # Skip blank lines & lines which appear to start with a comment. - if path and (path[0] not in ['#', '!']): - # Check for an absolute path - if not os.path.isabs(path): - path = os.path.normpath(os.path.join(root_path, path)) - # end if - file_list.append(path) - # end if (else skip blank or comment line) - # end for - # end with open - # end if (no else, we already have the errors) - return file_list, errors - -############################################################################### -def _create_file_list_int(files, suffices, file_type, logger, - txt_files, pathname, root_path, master_list): -############################################################################### - """Create and return a master list of files from . - is a list of pathnames which may include wildcards. - is a list of allowed file types. Filenames in - with an allowed suffix will be added to the master list. - Filenames with a '.txt' suffix will be parsed to look for allowed - filenames. - is a description of the allowed file types. - is a logger used to print warnings (unrecognized filename types) - and debug messages. - is a list of previously-encountered text files (to prevent - infinite recursion). - is the text file name from which was read (if any). - is the list of files which have already been collected - A list of error strings is also returned - """ - errors = list() - if pathname: - pdesc = '{} pathname file, found in {}'.format(file_type, pathname) - else: - pdesc = '{} pathnames file'.format(file_type) - # end if - if not isinstance(files, list): - raise ParseInternalError("'{}' is not a list".format(files)) - # end if - for filename in files: - # suff is filename's extension - suff = os.path.splitext(filename)[1] - if suff: - suff = suff[1:] - # end if - if not os.path.isabs(filename): - filename = os.path.normpath(os.path.join(root_path, filename)) - # end if - if os.path.isdir(filename): - for suff_type in suffices: - file_type = os.path.join(filename, '*.{}'.format(suff_type)) - errs = add_unique_files(file_type, pdesc, master_list, logger) - errors.extend(errs) - # end for - elif suff in suffices: - errs = add_unique_files(filename, pdesc, master_list, logger) - errors.extend(errs) - elif suff == 'txt': - tfiles = glob.glob(filename) - if tfiles: - for file in tfiles: - if file in txt_files: - lmsg = "WARNING: Ignoring duplicate '.txt' file, {}" - logger.warning(lmsg.format(filename)) - else: - lmsg = 'Reading .{} filenames from {}' - logger.debug(lmsg.format(', .'.join(suffices), - file)) - flist, errs = read_pathnames_from_file(file, file_type) - errors.extend(errs) - txt_files.append(file) - root = os.path.dirname(file) - _, errs = _create_file_list_int(flist, suffices, - file_type, logger, - txt_files, file, - root, master_list) - errors.extend(errs) - # end if - # end for - else: - emsg = "{} pathnames file, '{}', does not exist" - errors.append(emsg.format(file_type, filename)) - # end if - else: - lmsg = 'WARNING: Not reading {}, only reading .{} or .txt files' - logger.warning(lmsg.format(filename, ', .'.join(suffices))) - # end if - # end for - - return master_list, errors - -############################################################################### -def create_file_list(files, suffices, file_type, logger, root_path=None): -############################################################################### - """Create and return a master list of files from . - is either a comma-separated string of pathnames or a list. - If a pathname is a directory, all files with extensions in - are included. - Wildcards in a pathname are expanded. - is a list of allowed file types. Filenames in - with an allowed suffix will be added to the master list. - Filenames with a '.txt' suffix will be parsed to look for allowed - filenames. - is a description of the allowed file types. - is a logger used to print warnings (unrecognized filename types) - and debug messages. - If is not None, it is used to create absolute paths for - , otherwise, the current working directory is used. - """ - master_list = list() - txt_files = list() # Already processed txt files - pathname = None - if isinstance(files, str): - file_list = [x.strip() for x in files.split(',') if x.strip()] - elif isinstance(files, (list, tuple)): - file_list = files - else: - raise ParseInternalError("Bad input, = {}".format(files)) - # end if - if root_path is None: - root_path = os.getcwd() - # end if - master_list, errors = _create_file_list_int(file_list, suffices, file_type, - logger, txt_files, pathname, - root_path, master_list) - if errors: - emsg = 'Error processing list of {} files:\n {}' - raise CCPPError(emsg.format(file_type, '\n '.join(errors))) - # end if - return master_list - -############################################################################### -def replace_paths(dir_list, src_dir, dest_dir): -############################################################################### - """For every path in , replace instances of with - """ - for index, path in enumerate(dir_list): - dir_list[index] = path.replace(src_dir, dest_dir) - # end for - -############################################################################### -def remove_dir(src_dir, force=False): -############################################################################### - """Remove and its children. This operation can only succeed if - contains no files or if is True.""" - currdir = os.getcwd() - src_parent = os.path.split(src_dir)[0] - src_rel = os.path.relpath(src_dir, src_parent) - os.chdir(src_parent) # Prevent removing the parent of src_dir - if force: - leaf_dirs = set() - for root, dirs, files in os.walk(src_rel): - for file in files: - os.remove(os.path.join(root, file)) - # end for - if not dirs: - leaf_dirs.add(root) - # end if - # end for - for ldir in leaf_dirs: - os.removedirs(ldir) - # end for - # end if (no else, always try to remove top level - try: - os.removedirs(src_rel) - except OSError: - pass # Ignore error, fail silently - # end try - os.chdir(currdir) - -############################################################################### -def move_modified_files(src_dir, dest_dir, overwrite=False, remove_src=False): -############################################################################### - """For each file in , move it to if that file is - different in the two locations. - if is True, move all files to , even if unchanged. - If is True, remove when complete.""" - src_files = {} # All files in - if os.path.normpath(src_dir) != os.path.normpath(dest_dir): - for root, _, files in os.walk(src_dir): - for file in files: - src_path = os.path.join(root, file) - if file in src_files: - # We do not allow two files with the same name - emsg = "Duplicate CCPP file found, '{}', original is '{}'" - raise CCPPError(emsg.format(src_path, src_files[file])) - # end if - src_files[file] = src_path - # end for - # end for - for file in src_files: - src_path = src_files[file] - src_file = os.path.relpath(src_path, start=src_dir) - dest_path = os.path.join(dest_dir, src_file) - if os.path.exists(dest_path): - if overwrite: - fmove = True - else: - fmove = filecmp.cmp(src_path, dest_path, shallow=False) - # end if - else: - fmove = True - # end if - if fmove: - os.replace(src_path, dest_path) - else: - os.remove(src_path) - # end if - # end for - if remove_src: - remove_dir(src_dir, force=True) - # end if - # end if (no else, take no action if the directories are identical) diff --git a/scripts/fortran_tools/__init__.py b/scripts/fortran_tools/__init__.py deleted file mode 100644 index 8626a9b1..00000000 --- a/scripts/fortran_tools/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Public API for the fortran_parser library -""" -import sys -import os.path -sys.path.insert(0, os.path.dirname(__file__)) - -# pylint: disable=wrong-import-position -from parse_fortran_file import parse_fortran_file -from parse_fortran import parse_fortran_var_decl, fortran_type_definition -from fortran_write import FortranWriter -# pylint: enable=wrong-import-position - -__all__ = [ - 'fortran_type_definition', - 'parse_fortran_file', - 'parse_fortran_var_decl', - 'FortranWriter' -] diff --git a/scripts/fortran_tools/fortran_write.py b/scripts/fortran_tools/fortran_write.py deleted file mode 100644 index 35d403e0..00000000 --- a/scripts/fortran_tools/fortran_write.py +++ /dev/null @@ -1,436 +0,0 @@ -#!/usr/bin/env python3 -# - -"""Code to write Fortran code -""" - -import math - -class FortranWriter: - """Class to turn output into properly continued and indented Fortran code - >>> FortranWriter("foo.F90", 'r', 'test', 'mod_name') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ValueError: Read mode not allowed in FortranWriter object - >>> FortranWriter("foo.F90", 'wb', 'test', 'mod_name') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ValueError: Binary mode not allowed in FortranWriter object - """ - - ########################################################################### - # Class variables - ########################################################################### - __INDENT = 2 # Spaces per indent level - - __CONTINUE_INDENT = 4 # Extra spaces on continuation line - - __LINE_FILL = 97 # Target line length - - __LINE_MAX = 120 # Max line length (for Codee) - - __BREAK_CHARS = [',', '+', '*', '/', '(', ')'] - - # CCPP copyright statement to be included in all generated Fortran files - __COPYRIGHT = '''! -! This work (Common Community Physics Package Framework), identified by -! NOAA, NCAR, CU/CIRES, is free of known copyright restrictions and is -! placed in the public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.''' - - __MOD_HEADER = ''' -!> -!! @brief Auto-generated {file_desc} -!! -! -module {module} -''' - - __MOD_PREAMBLE = ["implicit none", "private"] - - __CONTAINS = ''' -contains''' - - __MOD_FOOTER = ''' -end module {module}''' - - ########################################################################### - - def indent(self, level=0, continue_line=False): - 'Return an indent string for any level' - indent = self.indent_size * level - if continue_line: - indent = indent + self.__continue_indent - # End if - return indent*' ' - - ########################################################################### - - def find_best_break(self, choices, last=None): - """Find the best line break point given . - If is present, use it as a target line length.""" - if last is None: - last = self.__line_fill - # End if - # Find largest good break - possible = [x for x in choices if 0 < x < last] - if not possible: - best = self.__line_max + 1 - else: - best = max(possible) - # End if - if (best > self.__line_max) and (last < self.__line_max): - best = self.find_best_break(choices, last=self.__line_max) - # End if - return best - - ########################################################################### - - @staticmethod - def _in_quote(test_str): - """Return True if ends in a character context. - >>> FortranWriter._in_quote("hi'mom") - True - >>> FortranWriter._in_quote("hi mom") - False - >>> FortranWriter._in_quote("'hi mom'") - False - >>> FortranWriter._in_quote("'hi"" mom'") - False - """ - in_single_char = False - in_double_char = False - for char in test_str: - if in_single_char: - if char == "'": - in_single_char = False - # end if - elif in_double_char: - if char == '"': - in_double_char = False - # end if - elif char == "'": - in_single_char = True - elif char == '"': - in_double_char = True - # end if - # end for - return in_single_char or in_double_char - - ########################################################################### - - def write(self, statement, indent_level, continue_line=False): - """Write to the open file, indenting to - (see self.indent). - If is True, treat this line as a continuation of - a previous statement.""" - if isinstance(statement, list): - for stmt in statement: - self.write(stmt, indent_level, continue_line) - # End for - elif '\n' in statement: - for stmt in statement.split('\n'): - self.write(stmt, indent_level, continue_line) - # End for - else: - istr = self.indent(indent_level, continue_line) - ostmt = statement.strip() - is_comment_stmt = ostmt and (ostmt[0] == '!') - in_comment = "" - outstr = istr + ostmt - line_len = len(outstr) - if line_len > self.__line_fill: - # Collect pretty break points - spaces = [] - break_chars = [] - sptr = len(istr) - in_single_char = False - in_double_char = False - while sptr < line_len: - if in_single_char: - if outstr[sptr] == "'": - in_single_char = False - # End if (no else, just copy stuff in string) - elif in_double_char: - if outstr[sptr] == '"': - in_double_char = False - # End if (no else, just copy stuff in string) - elif outstr[sptr] == "'": - in_single_char = True - elif outstr[sptr] == '"': - in_double_char = True - elif outstr[sptr] == '!': - # Comment in non-character context - spaces.append(sptr-1) - in_comment = "! " # No continue for comment - if ((not is_comment_stmt) and - (sptr >= self.__max_comment_start)): - # suck in rest of line - sptr = line_len - 1 - # end if - elif outstr[sptr] == ' ': - # Non-quote spaces are where we can break - spaces.append(sptr) - elif outstr[sptr:sptr+2] == '//': - # Non-quote syntax are where we can break - break_chars.append(sptr + 1) - elif outstr[sptr] in FortranWriter.__BREAK_CHARS: - # Non-quote syntax are where we can break - break_chars.append(sptr) - # End if (no else, other characters will be ignored) - sptr = sptr + 1 - # End while - # Before looking for best space, reject any that are on a - # comment line but before any significant characters - if outstr.lstrip().startswith('!'): - first_space = outstr.index('!') + 1 - while ((outstr[first_space] == '!' or - outstr[first_space] == ' ') and - (first_space < line_len)): - first_space += 1 - # end while - if min(spaces) < first_space: - spaces = [x for x in spaces if x >= first_space] - # end if - best = self.find_best_break(spaces) - if best >= self.__line_fill: - best = min(best, self.find_best_break(break_chars)) - # End if - line_continue = False - if best >= self.__line_max: - # This is probably a bad situation so we have to break - # in an ugly spot - best = self.__line_max - 1 - if len(outstr) > best: - line_continue = '&' - # end if - # end if - if len(outstr) > best: - if self._in_quote(outstr[0:best+1]): - if best >= FortranWriter.__LINE_MAX - 1: - best = FortranWriter.__LINE_MAX - 2 - # end if - line_continue = '&' - elif not outstr[best+1:].lstrip(): - # If the next line is empty, the current line is done - # and is equal to the max line length. Do not use - # continue and set best to line_max (best+1) - line_continue = False - best = best+1 - else: - # If next line is just comment, do not use continue - line_continue = outstr[best+1:].lstrip()[0] != '!' - # end if - elif not line_continue: - line_continue = len(outstr) > best - # End if - if in_comment or is_comment_stmt: - line_continue = False - # end if - if line_continue == '&': - fill = '&' - elif line_continue: - fill = ' &' - else: - fill = "" - # End if - outline = f"{outstr[0:best+1].rstrip()}{fill}" - self.__file.write(f"{outline}\n") - if best <= 0: - imsg = "Internal ERROR: Unable to break line" - raise ValueError(f"{imsg}, '{statement}'") - # end if - statement = in_comment + outstr[best+1:] - if isinstance(line_continue, str) and statement.strip(): - statement = line_continue + statement - # end if - if statement.strip(): - self.write(statement, indent_level, continue_line=line_continue) - else: - self.__file.write(f"{outstr}\n") - # End if - # End if - - ########################################################################### - - def __init__(self, filename, mode, file_description, module_name, - indent=None, continue_indent=None, - line_fill=None, line_max=None): - """Initialize thie FortranWriter object. - Some boilerplate is written automatically.""" - self.__file_desc = file_description.replace('\n', '\n!! ') - self.__module = module_name - # We only handle writing situations (for now) and only text - if 'r' in mode: - raise ValueError('Read mode not allowed in FortranWriter object') - # end if - if 'b' in mode: - raise ValueError('Binary mode not allowed in FortranWriter object') - # End if - self.__file = open(filename, mode) - if indent is None: - self.__indent = FortranWriter.__INDENT - else: - self.__indent = indent - # End if - if continue_indent is None: - self.__continue_indent = FortranWriter.__CONTINUE_INDENT - else: - self.__continue_indent = continue_indent - # End if - if line_fill is None: - self.__line_fill = FortranWriter.__LINE_FILL - else: - self.__line_fill = line_fill - # End if - self.__max_comment_start = math.ceil(self.__line_fill * 3 / 4) - if line_max is None: - self.__line_max = FortranWriter.__LINE_MAX - else: - self.__line_max = line_max - # End if - - ########################################################################### - - def write_preamble(self): - """Write the module boilerplate that goes between use statements - and module declarations.""" - self.write("", 0) - for stmt in FortranWriter.__MOD_PREAMBLE: - self.write(stmt, 1) - # end for - self.write("", 0) - - ########################################################################### - - def end_module_header(self): - """Write the module contains statement.""" - self.write(FortranWriter.__CONTAINS, 0) - - ########################################################################### - - def __enter__(self, *args): - self.write(FortranWriter.__COPYRIGHT, 0) - self.write(self.module_header(), 0) - return self - - ########################################################################### - - def __exit__(self, *args): - self.write(FortranWriter.__MOD_FOOTER.format(module=self.__module), 0) - self.__file.close() - return False - - ########################################################################### - - def module_header(self): - """Return the standard Fortran module header for and - """ - return FortranWriter.__MOD_HEADER.format(file_desc=self.__file_desc, - module=self.__module) - - ########################################################################### - - def comment(self, comment, indent): - """Write a Fortran comment with contents, """ - mlcomment = comment.replace('\n', '\n! ') # No backslash in f string - self.write(f"! {mlcomment}", indent) - - ########################################################################### - - def blank_line(self): - """Write a blank line""" - self.write("", 0) - - ########################################################################### - - def include(self, filename): - """Insert the contents of verbatim.""" - with open(filename, 'r') as infile: - for line in infile: - self.__file.write(line) - # end for - # end with - - ########################################################################### - - @property - def line_fill(self): - """Return the target line length for this Fortran file""" - return self.__line_fill - - ########################################################################### - - @property - def indent_size(self): - """Return the number of spaces for each indent level for this - Fortran file - """ - return self.__indent - - ########################################################################### - - @classmethod - def copyright(cls): - """Return the standard Fortran file copyright string""" - return cls.__COPYRIGHT - -############################################################################### -if __name__ == "__main__": - # First, run doctest - # pylint: disable=ungrouped-imports - import doctest - import os - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - # Make sure we can write a file - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - NAME = 'foo' - while os.path.exists(NAME+'.F90'): - NAME = NAME + 'xo' - # end while - NAME = NAME + '.F90' - if os.access(os.getcwd(), os.W_OK): - _CHECK = FortranWriter.copyright().split('\n') - with FortranWriter(NAME, 'w', 'doctest', 'foo') as foo: - foo.write_preamble() - foo.end_module_header() - foo.write(("subroutine foo(long_argument1, long_argument2, " - "long_argument3, long_argument4, long_argument5)"), 2) - foo.write("end subroutine foo", 2) - _CHECK.extend(foo.module_header().rstrip().split('\n')) - # End with - _CHECK.extend(["", "", " implicit none", " private", - "", "", "CONTAINS"]) - _CHECK.extend([(' subroutine foo(long_argument1, long_argument2, ' - 'long_argument3, long_argument4, &'), - ' long_argument5)', - ' end subroutine foo', '', - 'end module foo']) - # Check file - with open(NAME, 'r') as foo: - _STATEMENTS = foo.readlines() - if len(_STATEMENTS) != len(_CHECK): - EMSG = "ERROR: File has {} statements, should have {}" - print(EMSG.format(len(_STATEMENTS), len(_CHECK))) - else: - for _line_num, _statement in enumerate(_STATEMENTS): - if _statement.rstrip() != _CHECK[_line_num]: - EMSG = "ERROR: Line {} does not match" - print(EMSG.format(_line_num+1)) - print("{}".format(_statement.rstrip())) - print("{}".format(_CHECK[_line_num])) - # end if - # end for - # end with - os.remove(NAME) - else: - print("WARNING: Unable to write test file, '{}'".format(NAME)) - # end if - sys.exit(fail) -# end if diff --git a/scripts/fortran_tools/offline_check_fortran_vs_metadata.py b/scripts/fortran_tools/offline_check_fortran_vs_metadata.py deleted file mode 100755 index ef50ced7..00000000 --- a/scripts/fortran_tools/offline_check_fortran_vs_metadata.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/env python3 - -""" -Recursively compare all fortran and metadata files in user-supplied directory, and report any problems -USAGE: - ./offline_check_fortran_vs_metadata.py --directory (--debug) -""" - - -import sys -import os -import glob -import logging -import argparse -import site -# Enable imports from parent directory -site.addsitedir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -# CCPP framework imports -from framework_env import CCPPFrameworkEnv -from fortran_tools import parse_fortran_file -from metadata_table import parse_metadata_file -from ccpp_capgen import find_associated_fortran_file -from ccpp_capgen import parse_scheme_files -from parse_tools import init_log, set_log_level -from parse_tools import register_fortran_ddt_name -from parse_tools import CCPPError, ParseInternalError - -_LOGGER = init_log(os.path.basename(__file__)) -_DUMMY_RUN_ENV = CCPPFrameworkEnv(_LOGGER, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -def find_files_to_compare(directory): - metadata_files = [] - for file in glob.glob(os.path.join(directory,'**','*.meta'), recursive=True): - metadata_files.append(file) - # end for - return metadata_files - -def compare_fortran_and_metadata(scheme_directory, run_env): - ## Check for files - metadata_files = find_files_to_compare(scheme_directory) - # Perform checks - parse_scheme_files(metadata_files, run_env, skip_ddt_check=True, relative_source_path=True) - -def parse_command_line(arguments, description): - """Parse command-line arguments""" - parser = argparse.ArgumentParser(description=description, - formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument("--directory", type=str, required=True, - metavar='top-level directory to analyze - REQUIRED', - help="""Full path to scheme directory""") - parser.add_argument("--debug", action='store_true', default=False, - help="""turn on debug mode for additional verbosity""") - pargs = parser.parse_args(arguments) - return pargs - -def _main_func(): - """Parse command line, then parse indicated host, scheme, and suite files. - Finally, generate code to allow host model to run indicated CCPP suites.""" - pargs = parse_command_line(sys.argv[1:], __doc__) - logger = _LOGGER - if pargs.debug: - set_log_level(logger, logging.DEBUG) - else: - set_log_level(logger, logging.INFO) - # end if - compare_fortran_and_metadata(pargs.directory, _DUMMY_RUN_ENV) - print('All checks passed!') - -############################################################################### - -if __name__ == "__main__": - try: - _main_func() - sys.exit(0) - except ParseInternalError as pie: - _LOGGER.exception(pie) - sys.exit(-1) - except CCPPError as ccpp_err: - if _LOGGER.getEffectiveLevel() <= logging.DEBUG: - _LOGGER.exception(ccpp_err) - else: - _LOGGER.error(ccpp_err) - # end if - sys.exit(1) - finally: - logging.shutdown() - # end try - diff --git a/scripts/fortran_tools/parse_fortran.py b/scripts/fortran_tools/parse_fortran.py deleted file mode 100644 index ab072afd..00000000 --- a/scripts/fortran_tools/parse_fortran.py +++ /dev/null @@ -1,910 +0,0 @@ -#!/usr/bin/env python3 - -"""Types and code for parsing Fortran source code. -""" - -# pylint: disable=wrong-import-position -if __name__ == '__main__' and __package__ is None: - import sys - import os.path - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import re -from parse_tools import ParseSyntaxError, ParseInternalError -from parse_tools import ParseContext, context_string -from parse_tools import check_fortran_intrinsic -from parse_tools import check_balanced_paren, unique_standard_name -#pylint: disable=unused-import -from parse_tools import ParseSource # Used in doctest -#pylint: enable=unused-import -from metavar import FortranVar -# pylint: enable=wrong-import-position - -# A collection of types and tools for parsing Fortran code to support -# CCPP metadata parsing. The purpose of this code is limited to type -# checking of routines with CCPP metadata caps, therefore full routines are -# not parsed and a full Fortran syntax tree is not warranted. - -######################################################################## - -# Fortran ID specifier (do not want a group like FORTRAN_ID from parse_tools) -_FORTRAN_ID = r"(?:[A-Za-z][A-Za-z0-9_]*)" -# Regular expression for a dimension specifier -_DIMID = r"(?:"+_FORTRAN_ID+r"|[0-9]+)" -_DIMCOLON = r"(?:\s*:\s*"+_DIMID+r"?\s*)" -_DIMCOLONS = r"(?:"+_DIMID+r"?"+_DIMCOLON+_DIMCOLON+r"?)" -_DIMSPEC = r"(?:"+_DIMID+r"|"+_DIMCOLONS+r")" -_dims_list_ = _DIMSPEC+r"(?:\s*,\s*"+_DIMSPEC+r"){0,6}" -# Regular expression for a variable name with optional dimensions -_VAR_ID_RE = re.compile(r"("+_FORTRAN_ID+r")\s*(\(\s*"+_dims_list_+r"\s*\))?$") - -######################################################################## - -class Ftype(object): - """Ftype is the base class for all Fortran types - It is also the final type for intrinsic types except for character - >>> Ftype('integer').typestr - 'integer' - >>> Ftype('integer', kind_in='( kind= I8').__str__() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid kind_selector, '( kind= I8', at :1 - >>> Ftype('integer', kind_in='(kind=I8)').__str__() - 'integer(kind=I8)' - >>> Ftype('integer', kind_in='(I8)').__str__() - 'integer(kind=I8)' - >>> Ftype('real', kind_in='(len=*,R8)').__str__() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid kind_selector, '(len=*,R8)', at :1 - >>> Ftype(typestr_in='real', line_in='real(kind=kind_phys)') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: typestr_in and line_in cannot both be used in a single call, at :1 - >>> Ftype(typestr_in='real', line_in='real(kind=kind_phys)', context=ParseContext(linenum=37, filename="foo.F90")) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: typestr_in and line_in cannot both be used in a single call, at foo.F90:37 - >>> Ftype(line_in='real(kind=kind_phys)').__str__() - 'real(kind=kind_phys)' - >>> Ftype(line_in="integer").__str__() - 'integer' - >>> Ftype(line_in="INTEGER").__str__() - 'INTEGER' - """ - - # Note that "character" is not in intrinsic_types even though it is a - # Fortran intrinsic. This is because character has its own type. - __intrinsic_types__ = [r"integer", r"real", r"logical", - r"double\s*precision", r"complex"] - - __itype_re = re.compile(r"(?i)({})\s*(\([A-Za-z0-9,=_\s]+\))?".format(r"|".join(__intrinsic_types__))) - __kind_re = re.compile(r"(?i)kind\s*(\()?\s*([\'\"])?(.+?)([\'\"])?\s*(\))?") - - __attr_spec = ['allocatable', 'asynchronous', 'dimension', 'external', - 'intent', 'intrinsic', 'bind', 'optional', 'parameter', - 'pointer', 'private', 'protected', 'public', 'save', - 'target', 'value', 'volatile'] - - def __init__(self, typestr_in=None, kind_in=None, match_len_in=None, - line_in=None, context=None): - """Initialize this FType object, either using and - , OR using line_in.""" - if context is None: - self.__context = ParseContext() - else: - self.__context = ParseContext(context=context) - # end if - # We have to distinguish which type of initialization we have - self.__typestr = typestr_in - if typestr_in is not None: - if line_in is not None: - emsg = "Cannot pass both typestr_in and line_in as arguments" - raise ParseInternalError(emsg, self.__context) - # end if - self.__default_kind = kind_in is None - if kind_in is None: - self.__kind = None - elif kind_in[0] == '(': - # Parse an explicit kind declaration - self.__kind = self.parse_kind_selector(kind_in) - else: - # The kind has already been parsed for us (e.g., by character) - self.__kind = kind_in - # end if - if match_len_in is not None: - self.__match_len = match_len_in - else: - self.__match_len = len(self.typestr) - if kind_in is not None: - self.__match_len += len(self.__kind) + 2 - # end if - # end if - elif kind_in is not None: - emsg = "kind_in cannot be passed without typestr_in" - raise ParseInternalError(emsg, self.__context) - elif line_in is not None: - match = Ftype.type_match(line_in) - if match is None: - emsg = "type declaration" - raise ParseSyntaxError(emsg, token=line_in, - context=self.__context) - # end if - if match_len_in is not None: - self.__match_len = match_len_in - else: - self.__match_len = len(match.group(0)) - # end if - if check_fortran_intrinsic(match.group(1)): - self.__typestr = match.group(1) - if match.group(2) is not None: - # Parse kind section - kmatch = match.group(2).strip() - self.__kind = self.parse_kind_selector(kmatch) - else: - self.__kind = None - # end if - self.__default_kind = self.__kind is None - else: - raise ParseSyntaxError("type declaration", - token=line_in, context=self.__context) - # end if - else: - emsg = "At least one of typestr_in or line_in must be passed" - raise ParseInternalError(emsg, self.__context) - # end if - - def parse_kind_selector(self, kind_selector, context=None): - """Find and return the 'kind' value from - '(foo)' and '(kind=foo)' both return 'foo'""" - if context is None: - if hasattr(self, 'context'): - context = self.__context - else: - context = ParseContext() - # end if - kind = None - if (kind_selector[0] == '(') and (kind_selector[-1] == ')'): - args = kind_selector[1:-1].split('=') - else: - args = kind_selector.split('=') - # end if - if (len(args) > 2) or (len(args) < 1): - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - if len(args) == 1: - kind = args[0].strip() - elif args[0].strip().lower() != 'kind': - # We have two args, the first better be kind - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - if kind is None: - # We have two args and the second is our kind string - kind = args[1].strip() - # end if - # One last check for missing right paren - match = Ftype.__kind_re.search(kind) - if match is not None: - if match.group(2) is not None: - if match.group(2) != match.group(4): - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - if (match.group(1) is None) and (match.group(5) is not None): - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - if (match.group(1) is not None) and (match.group(5) is None): - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - else: - pass - elif kind[0:4].lower() == "kind": - # Got 'something' == 'kind'?? - raise ParseSyntaxError("kind_selector", - token=kind_selector, context=context) - # end if - return kind - - @classmethod - def type_match(cls, line): - """Return an RE match if represents an Ftype declaration""" - match = Ftype.__itype_re.match(line.strip()) - return match - - @classmethod - def reassemble_parens(cls, propstr, errstr, context, splitstr=','): - """Return list of split by top-level instances of . - Occurrences of in character contexts or in parentheses are - ignored. - >>> Ftype.reassemble_parens("a(b, c),d,e()", 'spec', ParseContext()) - ['a(b, c)', 'd', 'e()'] - >>> Ftype.reassemble_parens("dimension(size(Grid%xlon,1),NSPC1), intent(in)", 'spec', ParseContext()) - ['dimension(size(Grid%xlon,1),NSPC1)', 'intent(in)'] - """ - var_list = list() - proplist = propstr.split(splitstr) - while len(proplist) > 0: - var = proplist.pop(0) - while var.count('(') != var.count(')'): - if len(proplist) == 0: - raise ParseSyntaxError(errstr, token=propstr, context=context) - # end if - var = var + ',' + proplist.pop(0) - # end while - var = var.strip() - if len(var) > 0: - var_list.append(var) - # end if - # end while - return var_list - - @classmethod - def parse_attr_specs(cls, propstring, context): - """Return a list of variable properties""" - properties = list() - # Remove leading comma - propstring = propstring.strip() - if propstring and (propstring[0] == ','): - propstring = propstring[1:].lstrip() - # end if - proplist = cls.reassemble_parens(propstring, 'attr_spec', context) - for prop in proplist: - prop = prop.strip().lower() - if '(' in prop: - # Strip out value from dimensions, bind, or intent - pval = prop[0:prop.index('(')].strip() - else: - pval = prop - # end if - if pval not in cls.__attr_spec: - raise ParseSyntaxError('attr_spec', token=prop, context=context) - # end if - properties.append(prop) - # end for - return properties - - @property - def typestr(self): - """ Return this FType object's type string""" - return self.__typestr - - @property - def default_kind(self): - """Return True iff this FType object is of default kind.""" - return self.__default_kind - - def kind(self): - """ Return this FType's kind string""" - return self.__kind - - @property - def type_len(self): - """ Return the length of this FType's kind string""" - return self.__match_len - - def __str__(self): - """Return a string of the declaration of the type""" - if self.default_kind: - return self.typestr - # end if - if check_fortran_intrinsic(self.typestr): - return "{}(kind={})".format(self.typestr, self.__kind) - # end if - # Derived type - return "{}({})".format(self.typestr, self.__kind) - -######################################################################## - -class FtypeCharacter(Ftype): - """FtypeCharacter is a type that represents character types - >>> FtypeCharacter.type_match('character') #doctest: +ELLIPSIS - - >>> FtypeCharacter.type_match('CHARACTER') #doctest: +ELLIPSIS - - >>> FtypeCharacter.type_match('chaRActer (len=*)') #doctest: +ELLIPSIS - - >>> FtypeCharacter.type_match('integer') - - >>> FtypeCharacter('character', ParseContext(169, 'foo.F90')).__str__() - Traceback (most recent call last): - parse_source.ParseSyntaxError: Invalid character declaration, 'character', at foo.F90:170 - >>> FtypeCharacter('character ::', ParseContext(171, 'foo.F90')).__str__() - 'character(len=1)' - >>> FtypeCharacter('CHARACTER(len=*)', ParseContext(174, 'foo.F90')).__str__() - 'CHARACTER(len=*)' - >>> FtypeCharacter('CHARACTER(len=:)', None).__str__() - 'CHARACTER(len=:)' - >>> FtypeCharacter('Character(len=512)', None).__str__() - 'Character(len=512)' - >>> FtypeCharacter('character(*)', None).__str__() - 'character(len=*)' - >>> FtypeCharacter('character*7', None).__str__() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid character declaration, 'character*7', at :1 - >>> FtypeCharacter('character*7,', None).__str__() - 'character(len=7)' - >>> FtypeCharacter("character (kind=kind('a')", None).__str__() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid kind_selector, 'kind=kind('a'', at :1 - >>> FtypeCharacter("character (kind=kind('a'))", None).__str__() - "character(len=1, kind=kind('a'))" - >>> FtypeCharacter("character (13, kind=kind('a'))", None).__str__() - "character(len=13, kind=kind('a'))" - >>> FtypeCharacter("character (len=13, kind=kind('a'))", None).__str__() - "character(len=13, kind=kind('a'))" - >>> FtypeCharacter("character (kind=kind('b'), len=15)", None).__str__() - "character(len=15, kind=kind('b'))" - """ - - char_re = re.compile(r"(?i)(character)\s*(\([A-Za-z0-9,=*:\s\'\"()]+\))?") - chartrail_re = re.compile(r"\s*[,:]|\s+[A-Z]") - oldchar_re = re.compile(r"(?i)(character)\s*(\*)\s*([0-9]+\s*)") - oldchartrail_re = re.compile(r"\s*[,]|\s+[A-Z]") - len_token_re = re.compile(r"(?i)([:]|[*]|[0-9]+|[A-Z][A-Z0-9_]*)$") - - @classmethod - def type_match(cls, line): - """Return an RE match if represents an FtypeCharacter - declaration""" - # Try old style first to eliminate as a possibility - match = FtypeCharacter.oldchar_re.match(line.strip()) - if match is None: - match = FtypeCharacter.char_re.match(line.strip()) - # end if - return match - - def __init__(self, line, context): - """Initialize a character type from a declaration line""" - - clen = None - kind = None # This will be interpreted as default kind - match = FtypeCharacter.type_match(line) - if match is None: - raise ParseSyntaxError("character declaration", token=line, context=context) - # end if - match_len = len(match.group(0)) - if len(match.groups()) == 3: - # We have an old style character declaration - if match.group(2) != '*': - raise ParseSyntaxError("character declaration", token=line, context=context) - # end if - if FtypeCharacter.oldchartrail_re.match(line.strip()[len(match.group(0)):]) is None: - raise ParseSyntaxError("character declaration", - token=line, context=context) - # end if - clen = match.group(3) - elif match.group(2) is not None: - # Parse attributes (strip off parentheses) - attrs = [x.strip() for x in match.group(2)[1:-1].split(',')] - if not attrs: - # Empty parentheses is not allowed - raise ParseSyntaxError("char_selector", - token=match.group(2), context=context) - # end if - if len(attrs) > 2: - # Too many attributes! - raise ParseSyntaxError("char_selector", - token=match.group(2), context=context) - # end if - if attrs[0][0:4].lower() == "kind": - # The first arg is kind, try to parse it - kind = self.parse_kind_selector(attrs[0], context=context) - # If there is a second arg, it must be of form len= - if len(attrs) == 2: - clen = self.parse_len_select(attrs[1], - context, len_optional=False) - elif len(attrs) == 2: - # We have both a len and a kind, len first - clen = self.parse_len_select(attrs[0], - context, len_optional=True) - kind = self.parse_kind_selector(attrs[1], context) - else: - # We just a len argument - clen = self.parse_len_select(attrs[0], - context, len_optional=True) - # end if - else: - # We had better check the training characters - if FtypeCharacter.chartrail_re.match(line.strip()[len(match.group(0)):]) is None: - raise ParseSyntaxError("character declaration", - token=line, context=context) - # end if - # end if - if clen is None: - clen = 1 - # end if - self.lenstr = "{}".format(clen) - super(FtypeCharacter, self).__init__(typestr_in=match.group(1), - kind_in=kind, - match_len_in=match_len, - context=context) - - def parse_len_token(self, token, context): - """Check to make sure token is a valid length identifier""" - match = FtypeCharacter.len_token_re.match(token) - if match is not None: - return match.group(1) - # end if - raise ParseSyntaxError("length type-param-value", - token=token, context=context) - # end if - - def parse_len_select(self, lenselect, context, len_optional=True): - """Parse a character type length_selector""" - largs = [x.strip() for x in lenselect.split('=')] - if len(largs) > 2: - raise ParseSyntaxError("length_selector", token=lenselect, context=context) - # end if - if (not len_optional) and ((len(largs) != 2) or (largs[0].lower() != 'len')): - raise ParseSyntaxError("length_selector when len= is required", token=lenselect, context=context) - # end if - if len(largs) == 2: - if largs[0].lower() != 'len': - raise ParseSyntaxError("length_selector", token=lenselect, context=context) - # end if - return self.parse_len_token(largs[1], context) - elif len_optional: - return self.parse_len_token(largs[0], context) - else: - raise ParseSyntaxError("length_selector when len= is required", token=lenselect, context=context) - # end if - - def kind(self): - """Return a kind metadata declaration if this Ftype object is of - a non-default kind. - Otherwise, return an empty string.""" - if self.default_kind: - kind_str = "" - else: - kind_str = ", kind={}".format(super(FtypeCharacter, self).kind()) - # end if - return "len={}{}".format(self.lenstr, kind_str) - - def __str__(self): - """Return a string of the declaration of the type - For characters, we will always print an explicit len modifier - """ - return "{}({})".format(self.typestr, self.kind()) - -######################################################################## - -class FtypeTypeDecl(Ftype): - """FtypeTypeDecl is a type that represents derived Fortran type - declarations. - >>> FtypeTypeDecl.type_match('character') - - >>> FtypeTypeDecl.type_match('type(foo)') #doctest: +ELLIPSIS - - >>> FtypeTypeDecl.type_match('class(foo)') #doctest: +ELLIPSIS - - >>> FtypeTypeDecl.class_match('class(foo)') #doctest: +ELLIPSIS - - >>> FtypeTypeDecl.type_def_line('type GFS_statein_type') - ['GFS_statein_type', None, None] - >>> FtypeTypeDecl.type_def_line('type GFS_statein_type (n, m) ') - ['GFS_statein_type', None, '(n, m)'] - >>> FtypeTypeDecl.type_def_line('type, public, extends(foo) :: GFS_statein_type') - ['GFS_statein_type', ['public', 'extends(foo)'], None] - >>> FtypeTypeDecl.type_def_line('type(foo) :: bar') - - >>> FtypeTypeDecl.type_def_line('type foo ! This is a comment') - ['foo', None, None] - """ - - __type_decl_re__ = re.compile(r"(?i)(type)\s*\(\s*([A-Z][A-Z0-9_]*)\s*\)?") - - __type_attr_spec__ = ['abstract', 'bind', 'extends', 'private', 'public'] - - __class_decl_re__ = re.compile(r"(?i)(class)\s*\(\s*([A-Z][A-Z0-9_]*)\s*\)") - - def __init__(self, line, context): - """Initialize an extended type from a declaration line""" - match = FtypeTypeDecl.type_match(line) - if match is None: - match = FtypeTypeDecl.class_match(line) - # end if - if match is None: - raise ParseSyntaxError("type declaration", - token=line, context=context) - # end if - super(FtypeTypeDecl, self).__init__(typestr_in=match.group(2), - kind_in=match.group(2), - match_len_in=len(match.group(0)), - context=context) - self.__class = match.group(1) - - @classmethod - def type_match(cls, line): - """Return an RE match if represents an FtypeTypeDecl declaration - """ - match = FtypeTypeDecl.__type_decl_re__.match(line.strip()) - # end if - return match - - @classmethod - def class_match(cls, line): - """Return an RE match if represents an FtypeTypeDecl declaration - representing the declaration of a polymorphic variable - """ - match = FtypeTypeDecl.__class_decl_re__.match(line.strip()) - # end if - return match - - @classmethod - def type_def_line(cls, line): - """Return a type information if represents the start - of a type definition. - Otherwise, return None""" - type_def = None - if not cls.type_match(line): - if '!' in line: - sline = line[0:line.index('!')].strip() - else: - sline = line.strip() - # end if - if sline.lower()[0:4] == 'type': - if '::' in sline: - elements = sline.split('::') - type_name = elements[1].strip() - type_props = [x.strip() for x in elements[0].split(',')[1:]] - else: - # Plain type decl - type_name = sline.split(' ', 1)[1].strip() - type_props = None - # end if - if '(' in type_name: - tnstr = type_name.split('(') - type_name = tnstr[0].strip() - type_params = '(' + tnstr[1].rstrip() - else: - type_params = None - # end if - type_def = [type_name, type_props, type_params] - # end if - # end if - return type_def - - def __str__(self): - """Return a printable string for this Ftype object""" - return '{}({})'.format(self.__class, self.typestr) - -######################################################################## -def ftype_factory(line, context): -######################################################################## - "Return an appropriate type object if there is a match, otherwise None" - # We have to cut off the line at the end of any possible type info - # Strip comments first (might have an = character) - if '!' in line: - line = line[0:line.index('!')].rstrip() - # end if - ppos = line.find('(') - cpos = line.find(',') - if ppos >= 0: - if 0 <= cpos < ppos: - # Whatever parentheses there are, they are not part of type - line = line[0:cpos] - else: - # Find matching right parenthesis - depth = 1 - epos = len(line) - pepos = ppos + 1 - while (depth > 0) and (pepos < epos): - if line[pepos] == '(': - depth = depth + 1 - elif line[pepos] == ')': - depth = depth - 1 - # end if - pepos = pepos + 1 - # end while - line = line[0:pepos+1] - # end if - elif cpos >= 0: - line = line[0:cpos] - # end if - tmatch = Ftype.type_match(line) - if tmatch is None: - tobj = None - else: - tobj = Ftype(line_in=line, context=context) - # end if - if tmatch is None: - tmatch = FtypeCharacter.type_match(line) - if tmatch is not None: - tobj = FtypeCharacter(line, context) - # end if - # end if - if tmatch is None: - tmatch = FtypeTypeDecl.type_match(line) - if tmatch is not None: - tobj = FtypeTypeDecl(line, context) - # end if - # end if - if tmatch is None: - tmatch = FtypeTypeDecl.class_match(line) - if tmatch is not None: - tobj = FtypeTypeDecl(line, context) - # end if - # end if - return tobj - -######################################################################## -def fortran_type_definition(line): -######################################################################## - """Return a type information if represents the start - of a type definition. - Otherwise, return None.""" - return FtypeTypeDecl.type_def_line(line) - -######################################################################## -def parse_fortran_var_decl(line, source, run_env, imports=None): -######################################################################## - """Parse a Fortran variable declaration line and return a list of - Var objects representing the variables declared on . - >>> _VAR_ID_RE.match('foo') #doctest: +ELLIPSIS - - >>> _VAR_ID_RE.match("foo()") - - >>> _VAR_ID_RE.match('foo').group(1) - 'foo' - >>> _VAR_ID_RE.match('foo').group(2) - - >>> _VAR_ID_RE.match("foo(bar)").group(1) - 'foo' - >>> _VAR_ID_RE.match("foo(bar)").group(2) - '(bar)' - >>> _VAR_ID_RE.match("foo(bar)").group(2) - '(bar)' - >>> _VAR_ID_RE.match("foo(bar, baz)").group(2) - '(bar, baz)' - >>> _VAR_ID_RE.match("foo(bar : baz)").group(2) - '(bar : baz)' - >>> _VAR_ID_RE.match("foo(bar:)").group(2) - '(bar:)' - >>> _VAR_ID_RE.match("foo(: baz)").group(2) - '(: baz)' - >>> _VAR_ID_RE.match("foo(:, :,:)").group(2) - '(:, :,:)' - >>> _VAR_ID_RE.match("foo(8)").group(2) - '(8)' - >>> _VAR_ID_RE.match("foo(::,a:b,a:,:b)").group(2) - '(::,a:b,a:,:b)' - >>> from framework_env import CCPPFrameworkEnv - >>> _DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}) - >>> parse_fortran_var_decl("integer :: foo", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('local_name') - 'foo' - >>> parse_fortran_var_decl("integer :: foo = 0", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('local_name') - 'foo' - >>> parse_fortran_var_decl("integer :: foo", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('optional') - False - >>> parse_fortran_var_decl("integer, optional :: foo", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('optional') - 'True' - >>> parse_fortran_var_decl("integer, dimension(:) :: foo", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(:)' - >>> parse_fortran_var_decl("integer, dimension(:) :: foo(bar)", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(bar)' - >>> parse_fortran_var_decl("integer, dimension(:) :: foo(:,:), baz", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(:,:)' - >>> parse_fortran_var_decl("integer, dimension(:) :: foo(:,:), baz", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][1].get_prop_value('dimensions') - '(:)' - >>> parse_fortran_var_decl("real (kind=kind_phys), pointer :: phii (:,:) => null() !< interface geopotential height", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(:,:)' - >>> parse_fortran_var_decl("real(kind=kind_phys), dimension(im, levs, ntrac), intent(in) :: qgrs", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(im, levs, ntrac)' - >>> parse_fortran_var_decl("character(len=*), intent(out) :: errmsg", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('local_name') - 'errmsg' - >>> parse_fortran_var_decl("character(len=512), intent(out) :: errmsg", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('kind') - 'len=512' - >>> parse_fortran_var_decl("real(kind_phys), intent(out) :: foo(8)", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_prop_value('dimensions') - '(8)' - >>> parse_fortran_var_decl("real(kind_phys), intent(out) :: foo(8)", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0][0].get_dimensions() - ['8'] - >>> parse_fortran_var_decl("character(len=*), intent(out) :: errmsg", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[1][0] - 'Syntax error: Invalid variable declaration, character(len=*), intent(out) :: errmsg, intent not allowed in module variable, in ' - >>> parse_fortran_var_decl("type(banana_t) :: bananas(0:N_FRUITS)", ParseSource('foo.F90', 'module', ParseContext()), _DUMMY_RUN_ENV)[1][0] - "bananas: '0:N_FRUITS' is an invalid dimension name; integer dimension indices not supported, in " - - ## NB: Expressions (including function calls) not currently supported here - #>>> parse_fortran_var_decl("real(kind_phys), intent(out) :: foo(size(bar))", ParseSource('foo.F90', 'scheme', ParseContext()), _DUMMY_RUN_ENV)[0].get_prop_value('dimensions') - #'(size(bar))' - """ - context = source.context - sline = line.strip() - # Strip comments first - if '!' in sline: - sline = sline[0:sline.index('!')].rstrip() - # end if - tobject = ftype_factory(sline, context) - newvars = [] - errors = [] - errtyp = "Syntax error" - if tobject is not None: - varprops = sline[tobject.type_len:].strip() - def_dims = None # Default dimensions - intent = None - dimensions = None - if '::' in varprops: - elements = varprops.split('::') - varlist = elements[1].strip() - varprops = Ftype.parse_attr_specs(elements[0].strip(), context) - for prop in varprops: - if prop[0:6] == 'intent': - if source.ptype != 'scheme': - typ = source.ptype - ctx = context_string(context) - emsg1 = f"Invalid variable declaration, {sline}, " - emsg2 = f"intent not allowed in {typ} variable" - errmsg = f"{errtyp}: {emsg1}{emsg2}{ctx}" - if run_env.logger is not None: - run_env.logger.warning(errmsg) - # end if - errors.append(errmsg) - else: - intent = prop[6:].strip()[1:-1].strip() - # end if - elif prop[0:9:] == 'dimension': - dimensions = prop[9:].strip() - # end if - # end for - else: - # No attr_specs - varlist = varprops - varprops = list() - # end if - # Create Vars from these pieces - # We may need to reassemble multi-dimensional specs - var_list = Ftype.reassemble_parens(varlist, 'variable_list', context) - for var in var_list: - prop_dict = {} - if '=' in var: - # We do not care about initializers - var = var[0:var.rindex('=')].rstrip() - # end if - # Scan and gather variable pieces - inchar = None # Character context - var_len = len(var) - ploc = var.find('(') - if ploc < 0: - varname = var.strip() - dimspec = None - else: - varname = var[0:ploc].strip() - begin, end = check_balanced_paren(var) - if (begin < 0) or (end < 0): - ctx = context_string(context) - errmsg = f"{errtyp}: Invalid variable declaration, {var}{ctx}" - if run_env.logger is not None: - run_env.logger.warning(errmsg) - # end if - errors.append(errmsg) - else: - dimspec = var[begin:end+1] - # end if - # end if - prop_dict['local_name'] = varname - prop_dict['standard_name'] = unique_standard_name() - prop_dict['units'] = '' - if isinstance(tobject, FtypeTypeDecl): - prop_dict['ddt_type'] = tobject.typestr - else: - prop_dict['type'] = tobject.typestr - # end if - if tobject.kind() is not None: - prop_dict['kind'] = tobject.kind() - # end if - if 'optional' in varprops: - prop_dict['optional'] = 'True' - # end if - if 'allocatable' in varprops: - prop_dict['allocatable'] = 'True' - # end if - if intent is not None: - prop_dict['intent'] = intent - # end if - if dimspec is not None: - prop_dict['dimensions'] = dimspec - elif dimensions is not None: - prop_dict['dimensions'] = dimensions - else: - prop_dict['dimensions'] = '()' - # end if - # XXgoldyXX: I am nervous about allowing invalid Var objects here - # Also, this tends to cause an exception that ends up back here - # which is not a good idea. - try: - var = FortranVar(prop_dict, source, run_env, - fortran_imports=imports) - newvars.append(var) - except ParseSyntaxError as perr: - errors.append(str(perr)) - # end try - # end for - # No else (not a variable declaration) - # end if - return newvars, errors - -######################################################################## - -class UseStatement(object): - """Class to parse and capture information from a Fortran use statement - >>> UseStatement("use foo, only: bar").valid - True - >>> UseStatement("use foo, only: bar").module - 'foo' - >>> UseStatement("use foo, only: bar").imports - ['bar'] - >>> UseStatement("USE foo, only: bar, baz, qux").imports - ['bar', 'baz', 'qux'] - >>> UseStatement("use foo, only: bar, baz").imports - ['bar', 'baz'] - >>> UseStatement("use foo, only: bar, baz !, qux").imports - ['bar', 'baz'] - >>> UseStatement("use foo!, only: bar, baz").valid - False - >>> UseStatement("use foo!, only: bar, baz").module - 'foo' - >>> UseStatement("use foo!, only: bar, baz").imports - - """ - - __modmatch = r"use\s*("+_FORTRAN_ID+r")\s*" - __imports = r"("+_FORTRAN_ID+r"(\s*,\s*"+_FORTRAN_ID+")*)" - - __use_stmt_re = re.compile(r"(?i)"+__modmatch+r",\s*only:\s*"+__imports) - __naked_use_re = re.compile(r"(?i)use\s*("+_FORTRAN_ID+")") - - def __init__(self, line): - """Initialize a UseStatement object from .""" - match = UseStatement.__use_stmt_re.match(line.strip()) - self.__valid = match is not None - self.__module_name = None - self.__imports = None - if self.valid: - self.__module_name = match.group(1) - self.__imports = [x.strip() for x in match.group(2).split(',')] - else: - match = UseStatement.__naked_use_re.match(line.strip()) - if match: - self.__module_name = match.group(1) - # end if - # end if - - @property - def valid(self): - """Return True if this object represents a valid Fortran use statment""" - return self.__valid - - @property - def module(self): - """Return the module name if valid, otherwise, None""" - return self.__module_name - - @property - def imports(self): - """Return a list of the module's imports if valid, otherwise, None""" - return self.__imports - - @classmethod - def use_stmt_line(cls, line): - """Return True if is a Fortran use statement. - >>> UseStatement.use_stmt_line("use foo, only: bar") - True - >>> UseStatement.use_stmt_line("USE foo, only: bar, baz, qux") - True - >>> UseStatement.use_stmt_line("! use foo, only: bar") - False - """ - return UseStatement.__use_stmt_re.match(line.strip()) is not None - -######################################################################## -# Future classes -#class Ftype_type_def(FtypeTypeDecl) # Not sure about that super class -#class Fmodule_spec(object) # vars and types from a module specification part -# Fmodule_spec will contain a list of documented variables and a list of -# documented type definitions -#class Fmodule_subprog(object) # routines from a module subprogram part -#class Fmodule(object) # Info about and parsing for a Fortran module -#Fmodule will contain an Fmodule_spec and a Fmodule_subprog -######################################################################## - -######################################################################## diff --git a/scripts/fortran_tools/parse_fortran_file.py b/scripts/fortran_tools/parse_fortran_file.py deleted file mode 100644 index 9808f6c8..00000000 --- a/scripts/fortran_tools/parse_fortran_file.py +++ /dev/null @@ -1,1095 +0,0 @@ -#! /usr/bin/env python3 -""" -Tool to parse a Fortran file and return signature information -from metadata tables. -At the file level, we allow only PROGRAM blocks and MODULE blocks. -Subroutines, functions, or data are not supported outside a MODULE. -""" - -# Python library imports -import os.path -if __name__ == '__main__' and __package__ is None: - import sys - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -# end if -# pylint: disable=wrong-import-position -import re -from collections import OrderedDict -import logging -# CCPP framework imports -from parse_tools import CCPPError, ParseInternalError, ParseSyntaxError -from parse_tools import ParseContext, ParseObject, ParseSource, PreprocStack -from parse_tools import FORTRAN_ID, context_string -from metadata_table import MetadataTable -try: - from parse_fortran import parse_fortran_var_decl, fortran_type_definition - from parse_fortran import UseStatement -except ModuleNotFoundError: - from .parse_fortran import parse_fortran_var_decl, fortran_type_definition - from .parse_fortran import UseStatement -# end try -from metavar import VarDictionary -# pylint: enable=wrong-import-position - -_COMMENT_RE = re.compile(r"!.*$") -_FIXED_COMMENT_RE = re.compile(r"(?i)([C*]|(?:[ ]{0,4}!))") -_PROGRAM_RE = re.compile(r"(?i)\s*program\s+"+FORTRAN_ID) -_ENDPROGRAM_RE = re.compile(r"(?i)\s*end\s*program\s+"+FORTRAN_ID+r"?") -_MODULE_RE = re.compile(r"(?i)\s*module\s+"+FORTRAN_ID) -_ENDMODULE_RE = re.compile(r"(?i)\s*end\s*module\s+"+FORTRAN_ID+r"?") -_CONTAINS_RE = re.compile(r"(?i)\s*contains") -_CONTINUE_RE = re.compile(r"(?i)&\s*(!.*)?$") -_FIXED_CONTINUE_RE = re.compile(r"(?i) [^0 ]") -_BLANK_RE = re.compile(r"\s+") -_ARG_TABLE_START_RE = re.compile(r"(?i)\s*![!>]\s*(?:\\section)?\s*arg_table_"+FORTRAN_ID) -_PREFIX_SPECS = [r"(?:recursive)", r"(?:pure)", r"(?:elemental)"] -_PREFIX_SPEC = r"(?:{})?\s*".format('|'.join(_PREFIX_SPECS)) -_SUBNAME_SPEC = r"subroutine\s*" -_ARGLIST_SPEC = r"\s*(?:[(]\s*([^)]*)[)])?" -_SUBROUTINE_SPEC = r"(?i)\s*"+_PREFIX_SPEC+_SUBNAME_SPEC+FORTRAN_ID+_ARGLIST_SPEC -_SUBROUTINE_RE = re.compile(_SUBROUTINE_SPEC) -_END_SUBROUTINE_RE = re.compile(r"(?i)\s*end\s*"+_SUBNAME_SPEC+FORTRAN_ID+r"?") -_USE_RE = re.compile(r"(?i)\s*use\s(?:,\s*intrinsic\s*::)?\s*only\s*:([^!]+)") -_END_TYPE_RE = re.compile(r"(?i)\s*end\s*type(?:\s+"+FORTRAN_ID+r")?") -_INTENT_STMT_RE = re.compile(r"(?i),\s*intent\s*[(]") - -######################################################################## - -def line_statements(line): - """Break up line into a list of component Fortran statements - Note, because this is a simple script, we can cheat on the - interpretation of two consecutive quote marks. - >>> line_statements('integer :: i, j') - ['integer :: i, j'] - >>> line_statements('integer :: i; real :: j') - ['integer :: i', ' real :: j'] - >>> line_statements('integer :: i ! Do not break; here') - ['integer :: i ! Do not break; here'] - >>> line_statements("write(6, *) 'This is all one statement; y''all;'") - ["write(6, *) 'This is all one statement; y''all;'"] - >>> line_statements('write(6, *) "This is all one statement; y""all;"') - ['write(6, *) "This is all one statement; y""all;"'] - >>> line_statements(" ! This is a comment statement; y'all;") - [" ! This is a comment statement; y'all;"] - >>> line_statements("!! ") - ['!! '] - >>> line_statements("real(kind_phys), intent(in) :: good_arr2(:,:)") - ['real(kind_phys), intent(in) :: good_arr2(:,:)'] - >>> line_statements("real(kind_phys), intent(in) :: bad_arr1(:,;)") - ['real(kind_phys), intent(in) :: bad_arr1(:,;)'] - >>> line_statements("real(kind_phys), intent(in), dimension(;,:) :: bad_arr2") - ['real(kind_phys), intent(in), dimension(;,:) :: bad_arr2'] - >>> line_statements("real(kind_phys), intent(in), dimension(:,;) :: bad_arr3") - ['real(kind_phys), intent(in), dimension(:,;) :: bad_arr3'] - """ - statements = list() - ind_start = 0 - ind_end = 0 - line_len = len(line) - in_single_char = False - in_double_char = False - in_paren = 0 - while ind_end < line_len: - if in_single_char: - if line[ind_end] == "'": - in_single_char = False - # end if (no else, just copy stuff in string) - elif in_double_char: - if line[ind_end] == '"': - in_double_char = False - # end if (no else, just copy stuff in string) - elif line[ind_end] == "'": - in_single_char = True - elif line[ind_end] == '"': - in_double_char = True - elif line[ind_end] == '!': - # Comment in non-character context, suck in rest of line - ind_end = line_len - 1 - elif line[ind_end] == '(': - in_paren += 1 - elif line[ind_end] == ')': - in_paren = max(in_paren - 1, 0) - elif (line[ind_end] == ';') and (in_paren < 1): - # The whole reason for this routine, the statement separator - if ind_end > ind_start: - statements.append(line[ind_start:ind_end]) - # end if - ind_start = ind_end + 1 - ind_end = ind_start - 1 - # end if (no else, other characters will be copied) - ind_end = ind_end + 1 - # end while - # Cleanup - if ind_end > ind_start: - statements.append(line[ind_start:ind_end]) - # end if - return statements - -######################################################################## - -def read_statements(pobj, statements=None): - """Retrieve the next line and break it into statements""" - while (statements is None) or (sum([len(x) for x in statements]) == 0): - nline, _ = pobj.next_line() - if nline is None: - statements = None - break - # end if - statements = line_statements(nline) - # end while - return statements - -######################################################################## -def scan_fixed_line(line, in_single_char, in_double_char, context): - """Scan a fixed-format FORTRAN line for continue indicators, continued - quotes, and comments - Return continue_in_col, in_single_char, in_double_char, - comment_col - >>> scan_fixed_line(' & line continued', False, False, ParseContext()) - (5, False, False, -1) - >>> scan_fixed_line(' & line continued"', False, True, ParseContext()) - (5, False, False, -1) - >>> scan_fixed_line(' * line continued', False, False, ParseContext()) - (5, False, False, -1) - >>> scan_fixed_line(' 1 line continued', False, False, ParseContext()) - (5, False, False, -1) - >>> scan_fixed_line('C comment line', False, False, ParseContext()) - (-1, False, False, 0) - >>> scan_fixed_line('* comment line', False, False, ParseContext()) - (-1, False, False, 0) - >>> scan_fixed_line('! comment line', False, False, ParseContext()) - (-1, False, False, 0) - >>> scan_fixed_line(' ! comment line', False, False, ParseContext()) - (-1, False, False, 1) - >>> scan_fixed_line(' ! comment line', False, False, ParseContext()) - (-1, False, False, 4) - >>> scan_fixed_line(' ! not comment line', False, False, ParseContext()) - (5, False, False, -1) - >>> scan_fixed_line('!...................................', False, False, ParseContext()) - (-1, False, False, 0) - >>> scan_fixed_line('123 x = x + 1', False, False, ParseContext()) - (-1, False, False, -1) - """ - - # Check if comment or continue statement - cmatch = _FIXED_COMMENT_RE.match(line) - is_comment = cmatch is not None - is_continue = _FIXED_CONTINUE_RE.match(line) is not None - # A few sanity checks - if (in_single_char or in_double_char) and (not is_continue): - raise ParseSyntaxError("Cannot start line in character context if not a continued line", context=context) - # endif - if in_single_char and in_double_char: - raise ParseSyntaxError("Cannot be both in an apostrophe character context and a quote character context", context=context) - - if is_continue: - continue_in_col = 5 - comment_col = -1 - index = 6 - elif is_comment: - comment_col = len(cmatch.group(1)) - 1 - continue_in_col = -1 - index = len(line.rstrip()) - else: - continue_in_col = -1 - comment_col = -1 - index = 0 - # end if - - last_ind = len(line.rstrip()) - 1 - # Process the line - while index <= last_ind: - blank = _BLANK_RE.match(line[index:]) - if blank is not None: - index = index + len(blank.group(0)) - 1 # +1 at end of loop - elif in_single_char: - if line[index:min(index+1, last_ind)] == "''": - # Embedded single quote - index = index + 1 # +1 and end of loop - elif line[index] == "'": - in_single_char = False - # end if - # end if (just ignore any other character) - elif in_double_char: - if line[index:min(index+1, last_ind)] == '""': - # Embedded double quote - index = index + 1 # +1 and end of loop - elif line[index] == '"': - in_double_char = False - # end if - # end if (just ignore any other character) - elif line[index] == "'": - # If we got here, we are not in a character context, start single - in_single_char = True - elif line[index] == '"': - # If we got here, we are not in a character context, start double - in_double_char = True - elif line[index] == '!': - # If we got here, we are not in a character context, done with line - comment_col = index - index = last_ind - # end if - index = index + 1 - # end while - - return continue_in_col, in_single_char, in_double_char, comment_col - -######################################################################## - -def scan_free_line(line, in_continue, in_single_char, in_double_char, context): - """Scan a Fortran line for continue indicators, continued quotes, and - comments - Return continue_in_col, continue_out_col, in_single_char, in_double_char, - comment_col - >>> scan_free_line("! Comment line", False, False, False, ParseContext()) - (-1, -1, False, False, 0) - >>> scan_free_line("!! ", False, False, False, ParseContext()) - (-1, -1, False, False, 0) - >>> scan_free_line("int :: index", False, False, False, ParseContext()) - (-1, -1, False, False, -1) - >>> scan_free_line("int :: inde& ! oops", False, False, False, ParseContext()) - (-1, 11, False, False, 13) - >>> scan_free_line("int :: inde&", False, False, False, ParseContext()) - (-1, 11, False, False, -1) - >>> scan_free_line("character(len=*), parameter :: foo = 'This line & not continued'", False, False, False, ParseContext()) - (-1, -1, False, False, -1) - >>> scan_free_line("character(len=*), parameter :: foo = 'This is continue line& ", False, False, False, ParseContext()) - (-1, 59, True, False, -1) - >>> scan_free_line('character(len=*), parameter :: foo = "This line & not continued"', False, False, False, ParseContext()) - (-1, -1, False, False, -1) - >>> scan_free_line('character(len=*), parameter :: foo = "This is continue line& ', False, False, False, ParseContext()) - (-1, 59, False, True, -1) - >>> scan_free_line(' & line continued"', True, False, True, ParseContext()) - (2, -1, False, False, -1) - >>> scan_free_line(' & line continued"', True, True, False, ParseContext()) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Cannot end non-continued line in a character context, in - >>> scan_free_line(" & line continued'", True, False, True, ParseContext()) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Cannot end non-continued line in a character context, in - >>> scan_free_line("int :: inde&", False, True, False, ParseContext()) - Traceback (most recent call last): - parse_source.ParseSyntaxError: Cannot start line in character context if not a continued line, in - >>> scan_free_line("int :: inde&", True, True, True, ParseContext()) - Traceback (most recent call last): - parse_source.ParseSyntaxError: Cannot be both in an apostrophe character context and a quote character context, in - """ - - # A few sanity checks - if (in_single_char or in_double_char) and (not in_continue): - raise ParseSyntaxError("Cannot start line in character context if not a continued line", context=context) - # endif - if in_single_char and in_double_char: - raise ParseSyntaxError("Cannot be both in an apostrophe character context and a quote character context", context=context) - - continue_in_col = -1 - continue_out_col = -1 - comment_col = -1 - - index = 0 - last_ind = len(line.rstrip()) - 1 - # Is first non-blank character a continue character? - if line.lstrip()[0] == '&': - if not in_continue: - raise ParseSyntaxError("Cannot begin line with continue character (&), not on continued line", context=context) - # end if - continue_in_col = line.find('&') - index = continue_in_col + 1 - # Process rest of line - while index <= last_ind: - blank = _BLANK_RE.match(line[index:]) - if blank is not None: - index = index + len(blank.group(0)) - 1 # +1 at end of loop - elif in_single_char: - if line[index:min(index+1, last_ind)] == "''": - # Embedded single quote - index = index + 1 # +1 and end of loop - elif line[index] == "'": - in_single_char = False - elif line[index] == '&': - if index == last_ind: - continue_out_col = index - # end if - # end if (just ignore any other character) - elif in_double_char: - if line[index:min(index+1, last_ind)] == '""': - # Embedded double quote - index = index + 1 # +1 and end of loop - elif line[index] == '"': - in_double_char = False - elif line[index] == '&': - if index == last_ind: - continue_out_col = index - # end if - # end if (just ignore any other character) - elif line[index] == "'": - # If we got here, we are not in a character context, start single - in_single_char = True - elif line[index] == '"': - # If we got here, we are not in a character context, start double - in_double_char = True - elif line[index] == '!': - # If we got here, we are not in a character context, done with line - comment_col = index - index = last_ind - elif line[index] == '&': - # If we got here, we are not in a character context, note continue - # First make sure this is a valid continue - match = _CONTINUE_RE.match(line[index:]) - if match is not None: - continue_out_col = index - else: - errmsg = ("Invalid continue, ampersand not followed by " - "comment character") - raise ParseSyntaxError(errmsg, context=context) - # end if - # end if - index = index + 1 - # end while - # A final check - if (in_single_char or in_double_char) and (continue_out_col < 0): - errmsg = "Cannot end non-continued line in a character context" - raise ParseSyntaxError(errmsg, context=context) - - return continue_in_col, continue_out_col, in_single_char, in_double_char, comment_col - -######################################################################## - -def read_file(filename, preproc_defs=None, logger=None): - """Read a file into an array of lines. - Preprocess lines to consolidate continuation lines. - Remove preprocessor directives and code eliminated by #if statements - Remvoved code results in blank lines, not removed lines - """ - preproc_status = PreprocStack() - if not os.path.exists(filename): - raise IOError("read_file: file, '{}', does not exist".format(filename)) - # end if - # We need special rules for fixed-form source - fixed_form = filename[-2:].lower() == '.f' - # Read all lines of the file at once - with open(filename, 'r') as file: - file_lines = file.readlines() - for index, line in enumerate(file_lines): - file_lines[index] = line.rstrip('\n').rstrip() - # end for - # end with - # create a parse object and context for this file - pobj = ParseObject(filename, file_lines) - continue_col = -1 # Active continue column - in_schar = False # Single quote character context - in_dchar = False # Double quote character context - prev_line = None - prev_line_num = -1 - curr_line, curr_line_num = pobj.curr_line() - while curr_line is not None: - # Skip empty lines and comment-only lines - skip_line = False - if len(curr_line.strip()) == 0: - skip_line = True - elif (fixed_form and - (_FIXED_COMMENT_RE.match(curr_line) is not None)): - skip_line = True - elif curr_line.lstrip()[0] == '!': - skip_line = True - # end if - if skip_line: - curr_line, curr_line_num = pobj.next_line() - continue - # end if - # Handle preproc issues - if preproc_status.process_line(curr_line, preproc_defs, pobj, logger): - pobj.write_line(curr_line_num, "") - curr_line, curr_line_num = pobj.next_line() - continue - # end if - if not preproc_status.in_true_region(): - # Special case to allow CCPP comment statements in False - # regions to find DDT and module table code - if (curr_line[0:2] != '!!') and (curr_line[0:2] != '!>'): - pobj.write_line(curr_line_num, "") - curr_line, curr_line_num = pobj.next_line() - continue - # end if - # end if - # scan the line for properties - if fixed_form: - res = scan_fixed_line(curr_line, in_schar, in_dchar, pobj) - cont_in_col, in_schar, in_dchar, comment_col = res - continue_col = cont_in_col # No warning in fixed form - cont_out_col = -1 - if (comment_col < 0) and (continue_col < 0): - # Real statement, grab the line # in case is continued - prev_line_num = curr_line_num - prev_line = None - # end if - else: - res = scan_free_line(curr_line, (continue_col >= 0), - in_schar, in_dchar, pobj) - cont_in_col, cont_out_col, in_schar, in_dchar, comment_col = res - # end if - # If in a continuation context, move this line to previous - if continue_col >= 0: - if fixed_form and (prev_line is None): - prev_line = pobj.peek_line(prev_line_num)[0:72] - # end if - if prev_line is None: - raise ParseInternalError("No prev_line to continue", - context=pobj) - # end if - sindex = max(cont_in_col+1, 0) - if fixed_form: - sindex = 6 - eindex = 72 - elif cont_out_col > 0: - eindex = cont_out_col - else: - eindex = len(curr_line) - # end if - prev_line = prev_line + curr_line[sindex:eindex] - if fixed_form: - prev_line = prev_line.rstrip() - # end if - # Rewrite the file's lines - pobj.write_line(prev_line_num, prev_line) - pobj.write_line(curr_line_num, "") - if (not fixed_form) and (cont_out_col < 0): - # We are done with this line, reset prev_line - prev_line = None - prev_line_num = -1 - # end if - # end if - continue_col = cont_out_col - if (continue_col >= 0) and (prev_line is None): - # We need to set up prev_line as it is continued - prev_line = curr_line[0:continue_col] - if not (in_schar or in_dchar): - prev_line = prev_line.rstrip() - # end if - prev_line_num = curr_line_num - # end if - curr_line, curr_line_num = pobj.next_line() - # end while - return pobj - -######################################################################## - -def parse_use_statement(statement, logger): - """Return True iff is a use statement""" - umatch = _USE_RE.match(statement) - if umatch is None: - return False - # end if - if logger: - logger.debug("use = {}".format(umatch.group(1))) - # end if - return True - -######################################################################## - -def is_dummy_argument_statement(statement): - """Return True iff is a dummy argument declaration""" - return _INTENT_STMT_RE.search(statement) is not None - -######################################################################## - -def is_comment_statement(statement): - """Return True iff is a Fortran comment""" - return statement.lstrip()[0] == '!' - -######################################################################## - -def parse_type_def(statements, type_def, mod_name, pobj, run_env, imports=None): - """Parse a type definition from and return the - remaining statements along with a MetadataTable object representing - the type's variables.""" - psrc = ParseSource(mod_name, 'ddt', pobj) - seen_contains = False - mheader = None - var_dict = VarDictionary(type_def[0], run_env) - inspec = True - errors = [] - while inspec and (statements is not None): - while len(statements) > 0: - statement = statements.pop(0) - # end program or module - pmatch = _END_TYPE_RE.match(statement) - if pmatch is not None: - # We hit the end of the type, make a header - mheader = MetadataTable(run_env, table_name_in=type_def[0], - table_type_in='ddt', - module=mod_name, var_dict=var_dict) - inspec = False - elif is_contains_statement(statement, inspec): - seen_contains = True - elif not seen_contains: - # Comment of variable - if ((not is_comment_statement(statement)) and - (not parse_use_statement(statement, run_env.logger))): - dvars, errs = parse_fortran_var_decl(statement, psrc, - run_env, - imports=imports) - errors.extend(errs) - for var in dvars: - var_dict.add_variable(var, run_env) - # end for - # end if - else: - # We are just skipping lines until the end type - pass - # end if - # end while - if inspec and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - return statements, mheader, errors - -######################################################################## - -def parse_preamble_data(statements, pobj, spec_name, endmatch, imports, run_env): - """Parse module variables or DDT definitions from a module preamble - or parse program variables from the beginning of a program. - Returns remaining statements, parsed metadata headers, and - any accumulated errors - """ - inspec = True - mheaders = [] - errors = [] - var_dict = VarDictionary(spec_name, run_env) - psrc = ParseSource(spec_name, 'MODULE', pobj) - active_table = None - if run_env.logger is not None: - ctx = context_string(pobj, nodir=True) - msg = "Parsing preamble variables of {}{}" - run_env.logger.debug(msg.format(spec_name, ctx)) - # end if - while inspec and (statements is not None): - while len(statements) > 0: - statement = statements.pop(0) - # end program or module - pmatch = endmatch.match(statement) - asmatch = _ARG_TABLE_START_RE.match(statement) - type_def = fortran_type_definition(statement) - if asmatch is not None: - active_table = asmatch.group(1) - elif (pmatch is not None) or is_contains_statement(statement, - inspec): - # We are done with the specification - inspec = False - # Put statement back so caller knows where we are - statements.insert(0, statement) - # Add the header (even if we found no variables) - mheader = MetadataTable(run_env, table_name_in=spec_name, - table_type_in='module', - module=spec_name, - var_dict=var_dict) - mheaders.append(mheader) - if run_env.verbose: - ctx = context_string(pobj, nodir=True) - msg = 'Adding header {}{}' - run_env.logger.debug(msg.format(mheader.table_name, ctx)) - # end if - break - elif type_def is not None: - # Put statement back so caller knows where we are - statements.insert(0, statement) - if ((active_table is not None) and - (type_def[0].lower() == active_table.lower())): - statements, ddt, errors = parse_type_def(statements, - type_def, spec_name, - pobj, run_env, - imports=imports) - if ddt is None: - ctx = context_string(pobj, nodir=True) - msg = "No DDT found at '{}'{}" - raise CCPPError(msg.format(statement, ctx)) - # end if - mheaders.append(ddt) - if run_env.verbose: - ctx = context_string(pobj, nodir=True) - msg = 'Adding DDT {}{}' - run_env.logger.debug(msg.format(ddt.table_name, ctx)) - # end if - active_table = None - else: - # We found a type definition but it is not one with - # metadata. Just parse it and throw away what is found. - _ = parse_type_def(statements, type_def, - spec_name, pobj, run_env) - # end if - elif active_table is not None: - # We should have a variable definition to add - if ((not is_comment_statement(statement)) and - (not parse_use_statement(statement, run_env.logger)) and - (active_table.lower() == spec_name.lower())): - dvars, errs = parse_fortran_var_decl(statement, - psrc, run_env) - errors.extend(errs) - for var in dvars: - var_dict.add_variable(var, run_env) - # end for - # end if - # end if (else we are not in an active table so just skip) - # end while - if inspec and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - return statements, mheaders, errors - -######################################################################## - -def parse_scheme_metadata(statements, pobj, spec_name, table_name, run_env): - "Parse dummy argument information from a subroutine" - psrc = None - mheader = None - var_dict = None - scheme_name = None - errors = [] - etyp = "Syntax error" - # Find the subroutine line, should be first executable statement - inpreamble = False - insub = True - seen_contains = False - if run_env.verbose: - ctx = context_string(pobj, nodir=True) - msg = "Parsing specification of {}{}" - run_env.logger.debug(msg.format(table_name, ctx)) - # end if - ctx = context_string(pobj) # Save initial context with directory - vdict = None # Initialized when we parse the subroutine arguments - while insub and (statements is not None): - while statements: - statement = statements.pop(0) - smatch = _SUBROUTINE_RE.match(statement) - esmatch = _END_SUBROUTINE_RE.match(statement) - pmatch = _ENDMODULE_RE.match(statement) - asmatch = _ARG_TABLE_START_RE.match(statement) - seen_contains = seen_contains or is_contains_statement(statement, insub) - if seen_contains: - inpreamble = False - # end if - if asmatch is not None: - # We have run off the end of something, hope that is okay - # Put this statement back for the caller to deal with - statements.insert(0, statement) - insub = False - break - # end if - if pmatch is not None: - # We have run off the end of the module, hope that is okay - pobj.leave_region('MODULE', region_name=spec_name) - insub = False - break - # end if - if smatch is not None and not seen_contains: - scheme_name = smatch.group(1) - inpreamble = scheme_name.lower() == table_name.lower() - if inpreamble: - if smatch.group(2) is not None: - smstr = smatch.group(2).strip() - if len(smstr) > 0: - smlist = smstr.strip().split(',') - else: - smlist = list() - # end if - scheme_args = [x.strip().lower() for x in smlist] - else: - scheme_args = list() - # end if - # Create a dict template with all the scheme's arguments - # in the correct order - vdict = OrderedDict() - for arg in scheme_args: - if len(arg) == 0: - errmsg = 'Empty argument{}' - raise ParseInternalError(errmsg.format(pobj)) - # end if - if arg in vdict: - ctx = context_string(pobj) - errors.append(f"Duplicate dummy argument, {arg}{ctx}") - else: - vdict[arg] = None - # end if - vdict[arg] = None - # end for - psrc = ParseSource(scheme_name, 'scheme', pobj) - # end if - elif inpreamble or seen_contains: - # Process a preamble statement (use or argument declaration) - if esmatch is not None: - inpreamble = False - seen_contains = False - insub = False - elif (inpreamble and - ((not is_comment_statement(statement)) and - (not parse_use_statement(statement, run_env)) and - is_dummy_argument_statement(statement))): - dvars, errs = parse_fortran_var_decl(statement, - psrc, run_env) - for err in errs: - # err might be an Exception instead of a string - errors.append(str(err)) - # end for - for var in dvars: - lname = var.get_prop_value('local_name').lower() - if lname in vdict: - if vdict[lname] is not None: - ctx = context_string(pobj) - errors.append(f"ERROR: Duplicate dummy argument, {lname}{ctx}") - else: - vdict[lname] = var - # end if - else: - ctx = context_string(pobj) - emsg = f"{etyp}: Invalid dummy argument, '{lname}'{ctx}" - errors.append(emsg) - # end if - # end for - # end if - # end if - # end while - if insub and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - # Check for missing declarations - missing = [] - if vdict is None: - errmsg = 'Subroutine, {}, not found{}' - raise CCPPError(errmsg.format(scheme_name, ctx)) - # end if - for lname in vdict.keys(): - if vdict[lname] is None: - missing.append(lname) - # end if - # end for - for lname in missing: - del vdict[lname] - # end for - if len(missing) > 0: - errmsg = f"Missing local_variables, {missing} in {scheme_name}" - errors.append(errmsg) - # end if - var_dict = VarDictionary(scheme_name, run_env, variables=vdict) - if (scheme_name is not None) and (var_dict is not None): - mheader = MetadataTable(run_env, table_name_in=scheme_name, - table_type_in='scheme', module=spec_name, - var_dict=var_dict) - # end if - return statements, mheader, errors - -######################################################################## - -def is_contains_statement(statement, in_module): - "Return True iff is an executable Fortran statement" - # Fill this in when we need to parse programs or subroutines - return in_module and (_CONTAINS_RE.match(statement.strip()) is not None) - -######################################################################## - -def duplicate_header(header, duplicate): - """Create and return an 'Duplicate header' error string""" - ctx = duplicate.start_context() - octx = header.start_context() - errmsg = 'Duplicate header, {}{}'.format(header.name, ctx) - if len(octx) > 0: - errmsg = errmsg + ', original{}'.format(octx) - # end if - return errmsg - -######################################################################## - -def parse_specification(pobj, statements, imports, run_env, mod_name=None, - prog_name=None): - """Parse specification part of a module or (sub)program. - Return the unparsed statements and a list of the parsed MetadataTable.""" - if (mod_name is not None) and (prog_name is not None): - raise ParseInternalError(" and cannot both be used") - # end if - if mod_name is not None: - spec_name = mod_name - endmatch = _ENDMODULE_RE - inmod = True - elif prog_name is not None: - spec_name = prog_name - endmatch = _ENDPROGRAM_RE - inmod = False - else: - raise ParseInternalError("One of or must be used") - # end if - if run_env.logger is not None: - ctx = context_string(pobj, nodir=True) - msg = "Parsing specification of {}{}" - run_env.logger.debug(msg.format(spec_name, ctx)) - # end if - - inspec = True - mtables = [] - errors = [] - while inspec and (statements is not None): - while len(statements) > 0: - statement = statements.pop(0) - # end program or module - pmatch = endmatch.match(statement) - asmatch = _ARG_TABLE_START_RE.match(statement) - type_def = fortran_type_definition(statement) - use_stmt = UseStatement.use_stmt_line(statement) - if pmatch is not None: - # We never found a contains statement - inspec = False - break - elif asmatch is not None: - # Put table statement back to re-read - statements.insert(0, statement) - statements, new_tbls, errors = parse_preamble_data(statements, - pobj, - spec_name, - endmatch, - imports, - run_env) - for tbl in new_tbls: - title = tbl.table_name - if title in mtables: - errors.append(duplicate_header(mtables[title], tbl)) - else: - if run_env.verbose: - ctx = tbl.start_context() - mtype = tbl.table_type - msg = "Adding metadata from {}, {}{}" - run_env.logger.debug(msg.format(mtype, title, ctx)) - # End if - mtables.append(tbl) - # end if - # end for - inspec = pobj.in_region('MODULE', region_name=mod_name) - break - elif type_def: - # We have a type definition without metadata - # Just parse it and throw away what is found. - # Put statement back so caller knows where we are - statements.insert(0, statement) - _ = parse_type_def(statements, type_def, - spec_name, pobj, run_env) - elif use_stmt: - # We have a use statement, add its imports to our set - use_obj = UseStatement(statement) - if use_obj.valid: - imports.update(use_obj.imports) - # end if - # end if - elif is_contains_statement(statement, inmod): - inspec = False - break - # end if - # end while - if inspec and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - return statements, mtables, errors - -######################################################################## - -def parse_program(pobj, statements, run_env): - """Parse a Fortran PROGRAM and return any leftover statements - and metadata tables encountered in the PROGRAM.""" - # The first statement should be a program statement, grab the name - pmatch = _PROGRAM_RE.match(statements[0]) - if pmatch is None: - raise ParseSyntaxError('PROGRAM statement', statements[0]) - # end if - prog_name = pmatch.group(1) - pobj.enter_region('PROGRAM', region_name=prog_name, nested_ok=False) - if run_env.logger is not None: - ctx = context_string(pobj, nodir=True) - msg = "Parsing Fortran program, {}{}" - run_env.logger.debug(msg.format(prog_name, ctx)) - # end if - # After the program name is the specification part - imports = set() - statements, mtables, errors = parse_specification(pobj, statements[1:], - imports, run_env, - prog_name=prog_name) - if errors: - raise CCPPError('\n'.join(errors)) - # end if - # We really cannot have tables inside a program's executable section - # Just read until end - statements = read_statements(pobj, statements) - inprogram = True - while inprogram and (statements is not None): - while len(statements) > 0: - statement = statements.pop(0) - # end program - pmatch = _ENDPROGRAM_RE.match(statement) - if pmatch is not None: - prog_name = pmatch.group(1) - pobj.leave_region('PROGRAM', region_name=prog_name) - inprogram = False - # end if - # end while - if inprogram and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - return statements, mtables - -######################################################################## - -def parse_module(pobj, statements, run_env): - """Parse a Fortran MODULE and return any leftover statements - and metadata tables encountered in the MODULE.""" - # Collect errors for more efficient error reporting - errors = [] - # Collect any imported typedef (and other) names - imports = set() - # The first statement should be a module statement, grab the name - pmatch = _MODULE_RE.match(statements[0]) - if pmatch is None: - raise ParseSyntaxError('MODULE statement', statements[0]) - # end if - mod_name = pmatch.group(1) - pobj.enter_region('MODULE', region_name=mod_name, nested_ok=False) - if run_env.verbose: - ctx = context_string(pobj, nodir=True) - msg = "Parsing Fortran module, {}{}" - run_env.logger.debug(msg.format(mod_name, ctx)) - # end if - # After the module name is the specification part - statements, mtables, errs = parse_specification(pobj, statements[1:], - imports, run_env, - mod_name=mod_name) - if errs: - errors.extend(errs) - # end if - # Look for metadata tables - statements = read_statements(pobj, statements) - inmodule = pobj.in_region('MODULE', region_name=mod_name) - active_table = None - additional_subroutines = [] - seen_contains = False - insub = False - while inmodule and (statements is not None): - while statements: - statement = statements.pop(0) - # end module - pmatch = _ENDMODULE_RE.match(statement) - asmatch = _ARG_TABLE_START_RE.match(statement) - smatch = _SUBROUTINE_RE.match(statement) - esmatch = _END_SUBROUTINE_RE.match(statement) - seen_contains = seen_contains or is_contains_statement(statement, insub) - use_stmt = UseStatement.use_stmt_line(statement) - if asmatch is not None: - active_table = asmatch.group(1) - elif pmatch is not None: - mod_name = pmatch.group(1) - pobj.leave_region('MODULE', region_name=mod_name) - inmodule = False - break - elif active_table is not None: - statements, mheader, errs = parse_scheme_metadata(statements, - pobj, - mod_name, - active_table, - run_env) - errors.extend(errs) - if mheader is not None: - title = mheader.table_name - if title in mtables: - errmsg = duplicate_header(mtables[title], mheader) - raise CCPPError(errmsg) - # end if - if run_env.verbose: - mtype = mheader.table_type - ctx = mheader.start_context() - msg = "Adding metadata from {}, {}{}" - run_env.logger.debug(msg.format(mtype, title, ctx)) - # end if - mtables.append(mheader) - # end if - active_table = None - inmodule = pobj.in_region('MODULE', region_name=mod_name) - break - elif smatch is not None and not seen_contains: - routine_name = smatch.group(1).strip() - additional_subroutines.append(routine_name) - insub = True - elif esmatch is not None and not seen_contains: - insub = False - elif esmatch is not None: - seen_contains = False - elif use_stmt: - # We have a use statement, add its imports to our set - use_obj = UseStatement(statement) - if use_obj.valid: - imports.update(use_obj.imports) - # end if - # end if - # end while - if inmodule and (statements is not None) and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - if errors: - raise CCPPError('\n'.join(errors)) - # end if - return statements, mtables, additional_subroutines - -######################################################################## - -def parse_fortran_file(filename, run_env): - """Parse a Fortran file and return all metadata tables found.""" - mtables = list() - pobj = read_file(filename, preproc_defs=run_env.preproc_defs, - logger=run_env.logger) - pobj.reset_pos() - curr_line, _ = pobj.curr_line() - statements = line_statements(curr_line) - while statements is not None: - if not statements: - statements = read_statements(pobj) - # end if - statement = statements.pop(0) - if _PROGRAM_RE.match(statement) is not None: - # push statement back so parse_program can use it - statements.insert(0, statement) - statements, ptables = parse_program(pobj, statements, run_env) - mtables.extend(ptables) - elif _MODULE_RE.match(statement) is not None: - # push statement back so parse_module can use it - statements.insert(0, statement) - statements, ptables, additional_routines = parse_module(pobj, statements, run_env) - mtables.extend(ptables) - # end if - if (statements is not None) and (len(statements) == 0): - statements = read_statements(pobj) - # end if - # end while - return mtables, additional_routines - -######################################################################## - -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - import doctest - fail, _ = doctest.testmod() - from parse_tools import register_fortran_ddt_name - # pylint: enable=ungrouped-imports - _FPATH = '/Users/goldy/scratch/foo' - _FNAMES = ['GFS_PBL_generic.F90', 'GFS_rad_time_vary.fv3.F90', - 'GFS_typedefs.F90'] - register_fortran_ddt_name('GFS_control_type') - register_fortran_ddt_name('GFS_data_type') - for fname in _FNAMES: - fpathname = os.path.join(_FPATH, fname) - if os.path.exists(fpathname): - mh = parse_fortran_file(fpathname, preproc_defs={'CCPP':1}) - for header in mheader: - print('{}: {}'.format(fname, h)) - # end for - # end if - # end for - sys.exit(fail) -# end if diff --git a/scripts/framework_env.py b/scripts/framework_env.py deleted file mode 100644 index 2237db3e..00000000 --- a/scripts/framework_env.py +++ /dev/null @@ -1,476 +0,0 @@ -#!/usr/bin/env python3 - -""" -Module to contain the runtime options for the CCPP Framework. -Function to parse arguments to the CCPP Framework and store them in an -object which allows various framework functions to access CCPP -Framework runtime information and parameter values. -""" - -# Python library imports -import argparse -import os -from parse_tools import verbose -_EPILOG = ''' -''' - -## List of kinds in ISO_FORTRAN_ENV (that are useful in CCPP) -## Note: this is defined here instead of in fortran_tools to prevent a -## circular dependency -ISO_FORTRAN_KINDS = ['int8', 'int16', 'int32', 'int64', 'real32', 'real64', 'real128'] - -############################################################################### -class CCPPFrameworkEnv: -############################################################################### - """Object and methods to hold the runtime environment and parameter - options for the CCPP Framework""" - - def __init__(self, logger, ndict=None, verbose=0, clean=False, - host_files=None, scheme_files=None, suites=None, - preproc_directives=[], generate_docfiles=False, host_name='', - kind_types=[], use_error_obj=False, force_overwrite=False, - output_root=os.getcwd(), ccpp_datafile="datatable.xml"): - """Initialize a new CCPPFrameworkEnv object from the input arguments. - is a dict with the parsed command-line arguments (or a - dictionary created with the necessary arguments). - is a logger to be used by users of this object. - is a list defining the Fortran kind types which will be - public in ccpp_kinds.F90. - It has entries of the form: - kind_type=kind_specification[:kind_module] - where is a string defining the kind type name, - is the Fortran kind parameter name - (e.g., 'REAL64'), and is the (optional) Fortran - module that contains . If is - not specified, then must be a type defined in - ISO_FORTRAN_ENV. - It is allowed to have a duplicate entry for as long as - it does not specify a different type or module. - will be made available as a kind in ccpp_kinds.F90 - """ - emsg = '' - esep = '' - if ndict and ('verbose' in ndict): - self.__verbosity = ndict['verbose'] - del ndict['verbose'] - else: - self.__verbosity = verbose - # end if - if ndict and ('clean' in ndict): - self.__clean = ndict['clean'] - del ndict['clean'] - else: - self.__clean = clean - # end if - if ndict and ('host_files' in ndict): - self.__host_files = ndict['host_files'] - del ndict['host_files'] - if host_files and logger: - wmsg = "CCPPFrameworkEnv: Using ndict, ignoring 'host_files'" - logger.warning(wmsg) - # end if - elif host_files is None: - emsg += esep + "Error: 'host_files' list required" - esep = '\n' - else: - self.__host_files = host_files - # end if - if ndict and ('scheme_files' in ndict): - self.__scheme_files = ndict['scheme_files'] - del ndict['scheme_files'] - if scheme_files and logger: - wmsg = "CCPPFrameworkEnv: Using ndict, ignoring 'scheme_files'" - logger.warning(wmsg) - # end if - elif scheme_files is None: - emsg += esep + "Error: 'scheme_files' list required" - esep = '\n' - else: - self.__scheme_files = scheme_files - # end if - if ndict and ('suites' in ndict): - self.__suites = ndict['suites'] - del ndict['suites'] - if suites and logger: - wmsg = "CCPPFrameworkEnv: Using ndict, ignoring 'suites'" - logger.warning(wmsg) - # end if - elif suites is None: - emsg += esep + "Error: 'suites' list required" - esep = '\n' - else: - self.__suites = suites - # end if - if ndict and ('preproc_directives' in ndict): - preproc_defs = ndict['preproc_directives'] - del ndict['preproc_directives'] - else: - preproc_defs = preproc_directives - # end if - # Turn preproc_defs into a dictionary, start with a list to process - if isinstance(preproc_defs, list): - # Someone already handed us a list - preproc_list = preproc_defs - elif (not preproc_defs) or (preproc_defs == 'UNSET'): - # No preprocessor definitions - preproc_list = list() - elif ',' in preproc_defs: - # String of definitions, separated by commas - preproc_list = [x.strip() for x in preproc_defs.split(',')] - elif isinstance(preproc_defs, str): - # String of definitions, separated by spaces - preproc_list = [x.strip() for x in preproc_defs.split(' ') if x] - else: - wmsg = f"Error: Bad preproc list type, '{type_name(preproc_defs)}'" - emsg += esep + wmsg - esep = '\n' - # end if - # Turn the list into a dictionary - self.__preproc_defs = {} - for item in preproc_list: - tokens = [x.strip() for x in item.split('=', 1)] - if len(tokens) > 2: - emsg += esep + "Error: Bad preproc def, '{}'".format(item) - esep = '\n' - else: - key = tokens[0] - if key[0:2] == '-D': - key = key[2:] - # end if - if len(tokens) > 1: - value = tokens[1] - else: - value = None - # end if - self.__preproc_defs[key] = value - # end if - # end for - if ndict and ('generate_docfiles' in ndict): - self.__generate_docfiles = ndict['generate_docfiles'] - del ndict['generate_docfiles'] - else: - self.__generate_docfiles = generate_docfiles - # end if - if ndict and ('host_name' in ndict): - self.__host_name = ndict['host_name'] - del ndict['host_name'] - else: - self.__host_name = host_name - # end if - self.__generate_host_cap = self.host_name != '' - self.__kind_dict = {} - if ndict and ("kind_types" in ndict): - kind_list = ndict["kind_types"] - del ndict["kind_types"] - else: - kind_list = kind_types - # end if - # Note that the command line uses repeated calls to 'kind_type' - for kind in kind_list: - kargs = [x.strip() for x in kind.strip().split('=')] - errstr = "" - if len(kargs) != 2: - emsg += (f"{esep}Error: '{kind}' is not a valid kind specification " - "(should be of the form =)") - esep = '\n' - else: - kind_name, kind_spec = kargs - kind_specs = kind_spec.split(':') - if len(kind_specs) == 1: - errstr = self.add_kind_type(kind_name, kind_specs[0]) - elif len(kind_specs) > 2: - emsg += (f"{esep}Error: Invalid format for '{kind_name}' " - "should be [] or [, ]") - esep = '\n' - else: - errstr = self.add_kind_type(kind_name, kind_specs[0], kind_specs[1]) - # end if - if errstr: - emsg += f"{esep}{errstr}" - esep = '\n' - # end if - # end if - # end for - - # We always need a kind_phys so add a default if necessary - if "kind_phys" not in self.__kind_dict: - # Use ISO-Fortran 64-bit real - # definition for default physics kind: - self.__kind_dict['kind_phys'] = ['REAL64', 'ISO_FORTRAN_ENV'] - # end if - if ndict and ('use_error_obj' in ndict): - self.__use_error_obj = ndict['use_error_obj'] - del ndict['use_error_obj'] - else: - self.__use_error_obj = use_error_obj - # end if - if ndict and ('force_overwrite' in ndict): - self.__force_overwrite = ndict['force_overwrite'] - del ndict['force_overwrite'] - else: - self.__force_overwrite = force_overwrite - # end if - # Make sure we know where output is going - if ndict and ('output_root' in ndict): - self.__output_root = ndict['output_root'] - del ndict['output_root'] - else: - self.__output_root = output_root - # end if - self.__output_dir = os.path.abspath(self.output_root) - # Make sure we can create output database - if ndict and ('ccpp_datafile' in ndict): - self.__datatable_file = os.path.normpath(ndict['ccpp_datafile']) - del ndict['ccpp_datafile'] - else: - self.__datatable_file = ccpp_datafile - # end if - if not os.path.isabs(self.datatable_file): - self.__datatable_file = os.path.join(self.output_dir, - self.datatable_file) - # end if - self.__logger = logger - ## Check to see if anything is left in dictionary - if ndict: - for key in ndict: - emsg += esep + "Error: Unknown key in , '{}'".format(key) - esep = '\n' - # end for - # end if - # Raise an exception if any errors were found - if emsg: - raise ValueError(emsg) - # end if - - @property - def verbosity(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__verbosity - - @property - def clean(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__clean - - @property - def host_files(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__host_files - - @property - def scheme_files(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__scheme_files - - @property - def suites(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__suites - - @property - def preproc_defs(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__preproc_defs - - @property - def generate_docfiles(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__generate_docfiles - - @property - def host_name(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__host_name - - @property - def generate_host_cap(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__generate_host_cap - - def kind_module(self, kind_type): - """Return the Fortran module that - contains the kind specification - for kind type, , - for this CCPPFrameworkEnv object. - If there is no entry for , - return None.""" - kind_mod = None - if kind_type in self.__kind_dict: - # The kind module should always be - # the second element in the list: - kind_mod = self.__kind_dict[kind_type][1] - # end if - return kind_mod - - def kind_spec(self, kind_type): - """Return the kind specification for kind type, - for this CCPPFrameworkEnv object. - If there is no entry for , return None.""" - kind_spec = None - if kind_type in self.__kind_dict: - # The kind specification should always be - # the first element in the list: - kind_spec = self.__kind_dict[kind_type][0] - # end if - return kind_spec - - def add_kind_type(self, new_ccpp_kind, new_kind, new_module=None): - """Add to our kind dictionary. - is the name of the Fortran module that defined - is the kind name as published in ccpp_kinds.f90 - This method assumes the inputs have been parsed. - Returns None or an error string if is already in the - kinds dictionary. - """ - emsg = "" - esep = "" - # Make sure we have a valid module - if new_module == None: - if new_kind.lower() in ISO_FORTRAN_KINDS: - new_module = 'ISO_FORTRAN_ENV' - else: - emsg += (f"{esep}Error: unknown kind, '{new_kind}' " - "and no Fortran module name specified") - esep = '\n' - # end if - # end if - # Check for incompatible duplicates - if ((new_ccpp_kind in self.__kind_dict) and - ((self.kind_spec(new_ccpp_kind) != new_kind) or - (self.kind_module(new_ccpp_kind) != new_module))): - emsg += (f"{esep}Error: '{new_ccpp_kind} = [{new_kind}, {new_module}]'" - f"is an invalid duplicate. {new_ccpp_kind} " - f"is already '{str(self.__kind_dict[new_ccpp_kind])}") - esep = '\n' - else: - if new_module: - self.__kind_dict[new_ccpp_kind] = [new_kind, new_module] - # end if - # end if - return emsg - - def kind_types(self): - """Return a list of all kind types defined in this - CCPPFrameworkEnv object.""" - return self.__kind_dict.keys() - - @property - def verbose(self): - """Return true if verbose enabled for the CCPPFrameworkEnv's - logger object.""" - return (self.logger and verbose(self.logger)) - - @property - def use_error_obj(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__use_error_obj - - @property - def force_overwrite(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__force_overwrite - - @property - def output_root(self): - """Return the property for this -CCPPFrameworkEnv object.""" - return self.__output_root - - @property - def output_dir(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__output_dir - - @property - def datatable_file(self): - """Return the property for this - CCPPFrameworkEnv object.""" - return self.__datatable_file - - @property - def logger(self): - """Return the property for this CCPPFrameworkEnv object.""" - return self.__logger - -############################################################################### -def parse_command_line(args, description, logger=None): -############################################################################### - """Create an ArgumentParser to parse and return a CCPPFrameworkEnv - object containing the command-line arguments and related quantities.""" - ap_format = argparse.RawTextHelpFormatter - parser = argparse.ArgumentParser(description=description, - formatter_class=ap_format, epilog=_EPILOG) - - parser.add_argument("--host-files", metavar='', - type=str, required=True, - help="""Comma separated list of host filenames to process -Filenames with a '.meta' suffix are treated as host model metadata files -Filenames with a '.txt' suffix are treated as containing a list of .meta -filenames""") - - parser.add_argument("--scheme-files", metavar='', - type=str, required=True, - help="""Comma separated list of scheme filenames to process -Filenames with a '.meta' suffix are treated as scheme metadata files -Filenames with a '.txt' suffix are treated as containing a list of .meta -filenames""") - - parser.add_argument("--suites", metavar='', - type=str, required=True, - help="""Comma separated list of suite definition filenames to process -Filenames with a '.xml' suffix are treated as suite definition XML files -Other filenames are treated as containing a list of .xml filenames""") - - parser.add_argument("--preproc-directives", - metavar='VARDEF1[,VARDEF2 ...]', type=str, default='', - help="Proprocessor directives used to correctly parse source files") - - parser.add_argument("--ccpp-datafile", type=str, - metavar='', - default="datatable.xml", - help="Filename for information on content generated by the CCPP Framework") - - parser.add_argument("--output-root", type=str, - metavar='', - default=os.getcwd(), - help="directory for generated files") - - parser.add_argument("--host-name", type=str, default='', - help='''Name of host model to use in CCPP API -If this option is passed, a host model cap is generated''') - - parser.add_argument("--clean", action='store_true', default=False, - help='Remove files created by this script, then exit') - - parser.add_argument("--kind-type", type=str, action='append', - metavar="kind_spec", dest="kind_types", default=list(), - help="""Data size for data (e.g., real()). -Entry in the form of = -e.g., --kind-type "kind_phys=REAL64" -Enter more than one --kind-type entry to define multiple CCPP kinds. - MUST be a valid ISO_FORTRAN_ENV type""") - - parser.add_argument("--generate-docfiles", - metavar='HTML | Latex | HTML,Latex', type=str, - help="Generate LaTeX and/or HTML documentation") - - parser.add_argument("--use-error-obj", action='store_true', default=False, - help="""Host model and caps use an error object -instead of ccpp_error_message and ccpp_error_code.""") - - parser.add_argument("--force-overwrite", action='store_true', default=False, - help="""Overwrite all CCPP-generated files, even -if unmodified""") - - parser.add_argument("--verbose", action='count', default=0, - help="Log more activity, repeat for increased output") - - pargs = parser.parse_args(args) - return CCPPFrameworkEnv(logger, vars(pargs)) diff --git a/scripts/host_cap.py b/scripts/host_cap.py deleted file mode 100644 index fb2e7012..00000000 --- a/scripts/host_cap.py +++ /dev/null @@ -1,821 +0,0 @@ -#!/usr/bin/env python3 - -""" -Parse a host-model registry XML file and return the captured variables. -""" - -# Python library imports -import logging -import os -# CCPP framework imports -from ccpp_suite import API, API_SOURCE_NAME -from ccpp_state_machine import CCPP_STATE_MACH -from constituents import ConstituentVarDict, CONST_DDT_NAME, CONST_DDT_MOD -from constituents import CONST_OBJ_STDNAME, CONST_PROP_TYPE -from ddt_library import DDTLibrary -from file_utils import KINDS_MODULE -from framework_env import CCPPFrameworkEnv -from metadata_table import MetadataTable -from metavar import Var, VarDictionary, CCPP_CONSTANT_VARS -from metavar import CCPP_LOOP_VAR_STDNAMES -from fortran_tools import FortranWriter -from parse_tools import CCPPError -from parse_tools import ParseObject, ParseSource, ParseContext, ParseSyntaxError - -############################################################################### -_HEADER = "cap for {host_model} calls to CCPP API" - -_SUBHEAD = ''' - subroutine {host_model}_ccpp_physics_{stage}({api_vars}) -''' - -_SUBFOOT = ''' - end subroutine {host_model}_ccpp_physics_{stage} -''' - -_API_SOURCE = ParseSource(API_SOURCE_NAME, "MODULE", - ParseContext(filename="host_cap.F90")) - -_API_DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -_SUITE_NAME_VAR = Var({'local_name':'suite_name', - 'standard_name':'suite_name', - 'intent':'in', 'type':'character', - 'kind':'len=*', 'units':'', 'protected':'True', - 'dimensions':'()'}, _API_SOURCE, _API_DUMMY_RUN_ENV) - -_SUITE_PART_VAR = Var({'local_name':'suite_part', - 'standard_name':'suite_part', - 'intent':'in', 'type':'character', - 'kind':'len=*', 'units':'', 'protected':'True', - 'dimensions':'()'}, _API_SOURCE, _API_DUMMY_RUN_ENV) - -############################################################################### -# Used for creating blank dictionary -_MVAR_DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -# Used to prevent loop substitution lookups -_BLANK_DICT = VarDictionary(API_SOURCE_NAME, _MVAR_DUMMY_RUN_ENV) - -############################################################################### -def suite_part_list(suite, stage): -############################################################################### - """Return a list of all the suite parts for this stage""" - run_stage = stage == 'run' - if run_stage: - spart_list = list() - for spart in suite.groups: - if suite.is_run_group(spart): - spart_list.append(spart) - # End if - # End for - else: - spart_list = [suite.phase_group(stage)] - # End if - return spart_list - -############################################################################### -def constituent_num_suite_subname(host_model): -############################################################################### - """Return the name of the number of suite constituents for this run - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_num_suite_constituents" - -############################################################################### -def constituent_register_subname(host_model): -############################################################################### - """Return the name of the subroutine used to register the constituent - properties for this run. - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_register_constituents" - -############################################################################### -def constituent_initialize_subname(host_model): -############################################################################### - """Return the name of the subroutine used to initialize the - constituents for this run. - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_initialize_constituents" - -############################################################################### -def constituent_num_consts_funcname(host_model): -############################################################################### - """Return the name of the function to return the number of - constituents for this run. - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_number_constituents" - -############################################################################### -def query_scheme_constituents_funcname(host_model): -############################################################################### - """Return the name of the function to return True if the standard name - passed in matches an existing constituent - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_is_scheme_constituent" - -############################################################################### -def constituent_copyin_subname(host_model): -############################################################################### - """Return the name of the subroutine to copy constituent fields to the - host model. - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_gather_constituents" - -############################################################################### -def constituent_copyout_subname(host_model): -############################################################################### - """Return the name of the subroutine to update constituent fields from - the host model. - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_update_constituents" - -############################################################################### -def constituent_cleanup_subname(host_model): -############################################################################### - """Return the name of the subroutine to deallocate dynamic constituent - arrays - Because this is a user interface API function, the name is fixed.""" - return f"{host_model.name}_ccpp_deallocate_dynamic_constituents" - -############################################################################### -def unique_local_name(loc_name, host_model): -############################################################################### - """Create a unique local name based on the local_name property, - for a variable with standard name, . - If is an unique local name (not in ), - simply return that. If not, create one and return that.""" - new_name = host_model.find_local_name(loc_name) is not None - if new_name: - new_lname = host_model.new_internal_variable_name(prefix=loc_name) - else: - new_lname = loc_name - # end if - return new_lname - -############################################################################### -def constituent_model_object_name(host_model): -############################################################################### - """Return the variable name of the object which holds the constituent - metadata and field information.""" - hvar = host_model.find_variable(CONST_OBJ_STDNAME) - if not hvar: - raise CCPPError(f"Host model does not contain Var, {CONST_OBJ_STDNAME}") - # end if - return hvar.get_prop_value('local_name') - -############################################################################### -def suite_dynamic_constituent_array_name(host_model, suite): -############################################################################### - """Return the name of the allocatable dynamic constituent properites array""" - hstr = f"{suite}_dynamic_constituents" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_stdnames(host_model): -############################################################################### - """Return the name of the array of constituent standard names""" - hstr = f"{host_model.name}_model_const_stdnames" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_indices(host_model): -############################################################################### - """Return the name of the array of constituent field array indices""" - hstr = f"{host_model.name}_model_const_indices" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_consts(host_model): -############################################################################### - """Return the name of the function that will return a pointer to the - array of all constituents""" - hstr = f"{host_model.name}_constituents_array" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_advected_consts(host_model): -############################################################################### - """Return the name of the function that will return a pointer to the - array of advected constituents""" - hstr = f"{host_model.name}_advected_constituents_array" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_props(host_model): -############################################################################### - """Return the name of the array of constituent property object pointers""" - hstr = f"{host_model.name}_model_const_properties" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_index(host_model): -############################################################################### - """Return the name of the interface that returns the array index of - a constituent array given its standard name""" - hstr = f"{host_model.name}_const_get_index" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_consts(host_model): -############################################################################### - """Return the name of the function that will return a pointer to the - array of all constituents""" - hstr = f"{host_model.name}_constituents_array" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_advected_consts(host_model): -############################################################################### - """Return the name of the function that will return a pointer to the - array of advected constituents""" - hstr = f"{host_model.name}_advected_constituents_array" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_props(host_model): -############################################################################### - """Return the name of the array of constituent property object pointers""" - hstr = f"{host_model.name}_model_const_properties" - return unique_local_name(hstr, host_model) - -############################################################################### -def constituent_model_const_index(host_model): -############################################################################### - """Return the name of the interface that returns the array index of - a constituent array given its standard name""" - hstr = f"{host_model.name}_const_get_index" - return unique_local_name(hstr, host_model) - -############################################################################### -def add_constituent_vars(cap, host_model, suite_list, run_env): -############################################################################### - """Create a DDT library containing array reference variables - for each constituent field for all suites in . - Create and return a dictionary containing an index variable for each of the - constituents as well as the variables from the DDT object. - Also, write declarations for these variables to . - Since the constituents are in a DDT (ccpp_constituent_properties_t), - create a metadata table with the required information, then parse it - to create the dictionary. - """ - # First create a MetadataTable for the constituents DDT - stdname_layer = "number_of_ccpp_constituents" - horiz_dim = "horizontal_dimension" - vert_layer_dim = "vertical_layer_dimension" - vert_interface_dim = "vertical_interface_dimension" - array_layer = "vars_layer" - tend_layer = "vars_layer_tend" - # Table preamble (leave off ccpp-table-properties header) - ddt_mdata = [ - #"[ccpp-table-properties]", - f" name = {CONST_DDT_NAME}", " type = ddt", - "[ccpp-arg-table]", - f" name = {CONST_DDT_NAME}", " type = ddt", - "[ num_layer_vars ]", - f" standard_name = {stdname_layer}", - " units = count", " dimensions = ()", " type = integer", - f"[ {array_layer} ]", - " standard_name = ccpp_constituents", - " units = none", - f" dimensions = ({horiz_dim}, {vert_layer_dim}, {stdname_layer})", - " type = real", " kind = kind_phys"] - # Add entries for each constituent (once per standard name) - const_stdnames = set() - tend_stdnames = set() - const_vars = set() - tend_vars = set() - for suite in suite_list: - if run_env.verbose: - lmsg = "Adding constituents from {} to {}" - run_env.logger.debug(lmsg.format(suite.name, host_model.name)) - # end if - scdict = suite.constituent_dictionary() - for cvar in scdict.variable_list(): - std_name = cvar.get_prop_value('standard_name') - if std_name not in const_stdnames and std_name not in tend_stdnames: - # Add a metadata entry for this constituent - # Check dimensions and figure vertical dimension - # Currently, we only support variables with first dimension, - # horizontal_dimension, and second (optional) dimension, - # vertical_layer_dimension or vertical_interface_dimension - is_tend_var = 'tendency_of' in std_name - dims = cvar.get_dimensions() - if (len(dims) < 1) or (len(dims) > 2): - emsg = "Unsupported constituent dimensions, '{}'" - dimstr = "({})".format(", ".join(dims)) - raise CCPPError(emsg.format(dimstr)) - # end if - hdim = dims[0].split(':')[-1] - if hdim != 'horizontal_dimension': - emsg = "Unsupported first constituent dimension, '{}', " - emsg += "must be 'horizontal_dimension'" - raise CCPPError(emsg.format(hdim)) - # end if - if len(dims) > 1: - vdim = dims[1].split(':')[-1] - if vdim == vert_layer_dim: - if is_tend_var: - cvar_array_name = tend_layer - else: - cvar_array_name = array_layer - # end if - else: - emsg = "Unsupported vertical constituent dimension, " - emsg += "'{}', must be '{}' or '{}'" - raise CCPPError(emsg.format(vdim, vert_layer_dim, - vert_interface_dim)) - # end if - else: - emsg = f"Unsupported 2-D variable, '{std_name}'" - raise CCPPError(emsg) - # end if - # Create an index variable for - if is_tend_var: - const_std_name = std_name.split("tendency_of_")[1] - else: - const_std_name = std_name - # end if - ind_std_name = f"index_of_{const_std_name}" - loc_name = f"{cvar_array_name}(:,:,{ind_std_name})" - ddt_mdata.append(f"[ {loc_name} ]") - ddt_mdata.append(f" standard_name = {std_name}") - units = cvar.get_prop_value('units') - ddt_mdata.append(f" units = {units}") - dimstr = f"({', '.join(dims)})" - ddt_mdata.append(f" dimensions = {dimstr}") - vtype = cvar.get_prop_value('type') - vkind = cvar.get_prop_value('kind') - ddt_mdata.append(f" type = {vtype} | kind = {vkind}") - if is_tend_var: - tend_vars.add(cvar) - tend_stdnames.add(std_name) - else: - const_vars.add(cvar) - const_stdnames.add(std_name) - # end if - - # end if - # end for - # end for - # Check that all tendency variables are valid - for tendency_variable in tend_vars: - tend_stdname = tendency_variable.get_prop_value('standard_name') - tend_const_name = tend_stdname.split('tendency_of_')[1] - found = False - # Find the corresponding constituent variable - for const_variable in const_vars: - const_stdname = const_variable.get_prop_value('standard_name') - if const_stdname == tend_const_name: - found = True - compat = tendency_variable.compatible(const_variable, run_env, is_tend=True) - if not compat: - errstr = f"Tendency variable, '{tend_stdname}'" - errstr += f", incompatible with associated state variable '{tend_const_name}'" - errstr += f". Reason: '{compat.incompat_reason}'" - raise ParseSyntaxError(errstr, token=tend_stdname, - context=tendency_variable.context) - # end if - # end if - # end for - if not found: - # error because we couldn't find the associated constituent - errstr = f"No associated state variable for tendency variable, '{tend_stdname}'" - raise ParseSyntaxError(errstr, token=tend_stdname, - context=tendency_variable.context) - # end if - # end for - # Parse this table using a fake filename - parse_obj = ParseObject(f"{host_model.name}_constituent_mod.meta", - ddt_mdata) - ddt_table = MetadataTable(run_env, parse_object=parse_obj) - ddt_lib = DDTLibrary(f"{host_model.name}_constituent_ddtlib", - run_env, ddts=ddt_table.sections()) - # A bit of cleanup - del parse_obj - del ddt_mdata - # Now, create the "host constituent module" dictionary - const_dict = VarDictionary(f"{host_model.name}_constituents", - run_env, parent_dict=host_model) - # Add the constituents object to const_dict and write its declaration - const_var = host_model.find_variable(CONST_OBJ_STDNAME) - if const_var: - const_dict.add_variable(const_var, run_env) - const_var.write_def(cap, 1, const_dict) - else: - raise CCPPError(f"Missing Var, {CONST_OBJ_STDNAME}, in host model") - # end if - ddt_lib.collect_ddt_fields(const_dict, const_var, run_env, - skip_duplicates=True) - # Declare the allocatable dynamic constituents array(s) - # One per suite - for suite in suite_list: - dyn_const_name = suite_dynamic_constituent_array_name(host_model, suite.name) - cap.write(f"type({CONST_PROP_TYPE}), allocatable, target :: {dyn_const_name}(:)", 1) - # end if - # Declare variable for the constituent standard names array - max_csname = max([len(x) for x in const_stdnames]) if const_stdnames else 0 - num_const_fields = len(const_stdnames) - cs_stdname = constituent_model_const_stdnames(host_model) - const_list = sorted(const_stdnames) - if const_list: - const_strs = ['"{}{}"'.format(x, ' '*(max_csname - len(x))) - for x in const_list] - cs_stdame_initstr = " = (/ " + ", ".join(const_strs) + " /)" - else: - cs_stdame_initstr = "" - # end if - cap.write("character(len={}) :: {}({}){}".format(max_csname, cs_stdname, - num_const_fields, - cs_stdame_initstr), 1) - # Declare variable for the constituent standard names array - array_name = constituent_model_const_indices(host_model) - cap.write("integer :: {}({}) = -1".format(array_name, num_const_fields), 1) - # Add individual variables for each index var to the const_dict - for index, std_name in enumerate(const_list): - ind_std_name = "index_of_{}".format(std_name) - ind_loc_name = "{}({})".format(array_name, index + 1) - prop_dict = {'standard_name' : ind_std_name, - 'local_name' : ind_loc_name, 'dimensions' : '()', - 'units' : 'index', 'protected' : "True", - 'type' : 'integer', 'kind' : ''} - ind_var = Var(prop_dict, _API_SOURCE, run_env) - const_dict.add_variable(ind_var, run_env) - # end for - # Add vertical dimensions for DDT call strings - pver = host_model.find_variable(standard_name=vert_layer_dim, - any_scope=False) - if pver is not None: - prop_dict = {'standard_name' : vert_layer_dim, - 'local_name' : pver.get_prop_value('local_name'), - 'units' : 'count', 'type' : 'integer', - 'protected' : 'True', 'dimensions' : '()'} - if const_dict.find_variable(standard_name=vert_layer_dim, - any_scope=False) is None: - ind_var = Var(prop_dict, _API_SOURCE, _API_DUMMY_RUN_ENV) - const_dict.add_variable(ind_var, run_env) - # end if - # end if - pver = host_model.find_variable(standard_name=vert_interface_dim, - any_scope=False) - if pver is not None: - prop_dict = {'standard_name' : vert_interface_dim, - 'local_name' : pver.get_prop_value('local_name'), - 'units' : 'count', 'type' : 'integer', - 'protected' : 'True', 'dimensions' : '()'} - if const_dict.find_variable(standard_name=vert_interface_dim, - any_scope=False) is None: - ind_var = Var(prop_dict, _API_SOURCE, run_env) - const_dict.add_variable(ind_var, run_env) - # end if - # end if - - return const_dict - -############################################################################### -def suite_part_call_list(host_model, const_dict, suite_part, subst_loop_vars, - dyn_const=False): -############################################################################### - """Return the controlled call list for . - is the constituent dictionary""" - spart_args = suite_part.call_list.variable_list(loop_vars=subst_loop_vars) - hmvars = list() # Host model to spart dummy args - if subst_loop_vars: - loop_vars = host_model.loop_vars - else: - loop_vars = None - # end if - for sp_var in spart_args: - stdname = sp_var.get_prop_value('standard_name') - sp_lname = sp_var.get_prop_value('local_name') - if sp_var.get_prop_value('type') == 'ccpp_constituent_properties_t': - if dyn_const: - hmvars.append(f"{sp_lname}={sp_lname}") - # end if - continue - # end if - var_dicts = [host_model, const_dict] - # Figure out which dictionary has the variable - for vdict in var_dicts: - hvar = vdict.find_variable(standard_name=stdname, any_scope=False) - if hvar is not None: - var_dict = vdict - break - # end if - # end for - if hvar is None: - errmsg = f"No host model variable for {stdname} in {suite_part.name}" - raise CCPPError(errmsg) - # End if - if stdname not in CCPP_CONSTANT_VARS: - lname = var_dict.var_call_string(hvar, loop_vars=loop_vars) - hmvars.append(f"{sp_lname}={lname}") - # End if - # End for - return ', '.join(hmvars) - -############################################################################### -def write_host_cap(host_model, api, module_name, output_dir, run_env): -############################################################################### - """Write an API to allow to call any configured CCPP suite""" - cap_filename = os.path.join(output_dir, '{}.F90'.format(module_name)) - if run_env.logger is not None: - msg = 'Writing CCPP Host Model Cap for {} to {}' - run_env.logger.info(msg.format(host_model.name, cap_filename)) - # End if - header = _HEADER.format(host_model=host_model.name) - with FortranWriter(cap_filename, 'w', header, module_name) as cap: - # Write module use statements - maxmod = len(KINDS_MODULE) - cap.write(' use {kinds}'.format(kinds=KINDS_MODULE), 1) - modules = host_model.variable_locations() - if modules: - mlen = max([len(x[0]) for x in modules]) - maxmod = max(maxmod, mlen) - # End if - mlen = max([len(x.module) for x in api.suites]) - maxmod = max(maxmod, mlen) - maxmod = max(maxmod, len(CONST_DDT_MOD)) - for mod in sorted(modules): - mspc = (maxmod - len(mod[0]))*' ' - cap.write("use {}, {}only: {}".format(mod[0], mspc, mod[1]), 1) - # End for - mspc = ' '*(maxmod - len(CONST_DDT_MOD)) - cap.write(f"use {CONST_DDT_MOD}, {mspc}only: {CONST_DDT_NAME}", 1) - cap.write(f"use {CONST_DDT_MOD}, {mspc}only: {CONST_PROP_TYPE}", 1) - cap.write_preamble() - max_suite_len = host_model.ddt_lib.max_mod_name_len - for suite in api.suites: - max_suite_len = max(max_suite_len, len(suite.module)) - # End for - cap.comment("Public Interfaces", 1) - # CCPP_STATE_MACH.transitions represents the host CCPP interface - for stage in CCPP_STATE_MACH.transitions(): - stmt = "public :: {host_model}_ccpp_physics_{stage}" - cap.write(stmt.format(host_model=host_model.name, stage=stage), 1) - # End for - API.declare_inspection_interfaces(cap) - # Write the host-model interfaces for constituents - reg_name = constituent_register_subname(host_model) - cap.write(f"public :: {reg_name}", 1) - init_name = constituent_initialize_subname(host_model) - cap.write(f"public :: {init_name}", 1) - numconsts_name = constituent_num_consts_funcname(host_model) - cap.write(f"public :: {numconsts_name}", 1) - queryconsts_name = query_scheme_constituents_funcname(host_model) - cap.write(f"public :: {queryconsts_name}", 1) - copyin_name = constituent_copyin_subname(host_model) - cap.write(f"public :: {copyin_name}", 1) - copyout_name = constituent_copyout_subname(host_model) - cap.write(f"public :: {copyout_name}", 1) - cleanup_name = constituent_cleanup_subname(host_model) - cap.write(f"public :: {cleanup_name}", 1) - const_array_func = constituent_model_consts(host_model) - cap.write(f"public :: {const_array_func}", 1) - advect_array_func = constituent_model_advected_consts(host_model) - cap.write(f"public :: {advect_array_func}", 1) - prop_array_func = constituent_model_const_props(host_model) - cap.write(f"public :: {prop_array_func}", 1) - const_index_func = constituent_model_const_index(host_model) - cap.write(f"public :: {const_index_func}", 1) - cap.write("", 0) - cap.write("! Private module variables", 1) - const_dict = add_constituent_vars(cap, host_model, api.suites, run_env) - cap.end_module_header() - for stage in CCPP_STATE_MACH.transitions(): - # Create a dict of local variables for stage - host_local_vars = VarDictionary(f"{host_model.name}_{stage}", - run_env) - has_dyn_consts = False - # Create part call lists - # Look for any loop-variable mismatch - for suite in api.suites: - spart_list = suite_part_list(suite, stage) - for spart in spart_list: - spart_args = spart.call_list.variable_list() - for sp_var in spart_args: - stdname = sp_var.get_prop_value('standard_name') - # Special handling for run-time constituents in register phase - if sp_var.get_prop_value('type') == 'ccpp_constituent_properties_t': - if spart.phase() == 'register': - prop_dict = {'standard_name' : sp_var.get_prop_value('standard_name'), - 'local_name' : sp_var.get_prop_value('local_name'), - 'dimensions' : '(:)', 'units' : 'none', - 'allocatable' : True, 'ddt_type' : 'ccpp_constituent_properties_t'} - newvar = Var(prop_dict, _API_SOURCE, run_env) - host_local_vars.add_variable(newvar, run_env) - has_dyn_consts = True - continue - else: - errmsg = f'ccpp_constituent_properties_t object "{stdname}" not allowed in "{spart.phase()}" phase' - raise CCPPError(errmsg) - # end if - # end if - hvar = const_dict.find_variable(standard_name=stdname, - any_scope=True) - if hvar is None: - errmsg = 'No host model variable for {} in {}' - raise CCPPError(errmsg.format(stdname, spart.name)) - # End if - # End for (loop over part variables) - # End for (loop of suite parts) - # End for (loop over suites) - if has_dyn_consts: - prop_dict = {'standard_name' : 'unused_count', - 'local_name' : 'num_dyn_consts', - 'dimensions' : '()', 'units' : 'count', - 'type' : 'integer'} - newvar = Var(prop_dict, _API_SOURCE, run_env) - host_local_vars.add_variable(newvar, run_env) - prop_dict = {'standard_name' : 'unused_index', - 'local_name' : 'const_index', - 'dimensions' : '()', 'units' : 'none', - 'type': 'integer'} - newvar = Var(prop_dict, _API_SOURCE, run_env) - host_local_vars.add_variable(newvar, run_env) - # end if - run_stage = stage == 'run' - # All interfaces need the suite name - apivars = [_SUITE_NAME_VAR] - if run_stage: - # Only the run phase needs a suite part name - apivars.append(_SUITE_PART_VAR) - # End if - # Create a list of dummy arguments with correct intent settings - callvars = host_model.call_list(stage) # Host interface dummy args - hdvars = list() - subst_dict = {} - for hvar in callvars: - protected = hvar.get_prop_value('protected') - stdname = hvar.get_prop_value('standard_name') - if stdname in CCPP_LOOP_VAR_STDNAMES: - protected = True # Cannot modify a loop variable - # End if - if protected: - subst_dict['intent'] = 'in' - else: - subst_dict['intent'] = 'inout' - # End if - hdvars.append(hvar.clone(subst_dict, - source_name=API_SOURCE_NAME)) - # End for - lnames = [x.get_prop_value('local_name') for x in apivars + hdvars] - api_vlist = ", ".join(lnames) - cap.write(_SUBHEAD.format(api_vars=api_vlist, - host_model=host_model.name, - stage=stage), 1) - # Write out any suite part use statements - for suite in api.suites: - mspc = (max_suite_len - len(suite.module))*' ' - spart_list = suite_part_list(suite, stage) - for _, spart in sorted(enumerate(spart_list)): - stmt = "use {}, {}only: {}" - cap.write(stmt.format(suite.module, mspc, spart.name), 2) - # End for - # End for - # Write out any host model DDT input var use statements - host_model.ddt_lib.write_ddt_use_statements(hdvars, cap, 2, - pad=max_suite_len) - - cap.write("", 1) - # Write out dummy argument definitions - for var in apivars: - var.write_def(cap, 2, host_model, dummy=True) - # End for - for var in hdvars: - var.write_def(cap, 2, host_model, dummy=True) - # End for - for var in host_local_vars.variable_list(): - var.write_def(cap, 2, host_model, - allocatable=var.get_prop_value('allocatable')) - # End for - cap.write('', 0) - # Write out the body clauses - errmsg_name, errflg_name = api.get_errinfo_names() - # Initialize err variables - cap.write('{errflg} = 0'.format(errflg=errflg_name), 2) - cap.write('{errmsg} = ""'.format(errmsg=errmsg_name), 2) - else_str = '' - for suite in api.suites: - stmt = "{}if (trim(suite_name) == '{}') then" - cap.write(stmt.format(else_str, suite.name), 2) - if stage == 'run': - el2_str = '' - spart_list = suite_part_list(suite, stage) - for spart in spart_list: - pname = spart.name[len(suite.name)+1:] - stmt = "{}if (trim(suite_part) == '{}') then" - cap.write(stmt.format(el2_str, pname), 3) - call_str = suite_part_call_list(host_model, const_dict, - spart, True) - cap.write("call {}({})".format(spart.name, call_str), 4) - el2_str = 'else ' - # End for - cap.write("else", 3) - emsg = "write({errmsg}, '(3a)')".format(errmsg=errmsg_name) - emsg += '"No suite part named ", ' - emsg += 'trim(suite_part), ' - emsg += '" found in suite {sname}"'.format(sname=suite.name) - cap.write(emsg, 4) - cap.write("{errflg} = 1".format(errflg=errflg_name), 4) - cap.write("end if", 3) - elif stage == 'register': - spart = suite.phase_group(stage) - dyn_const_array = suite_dynamic_constituent_array_name(host_model, suite.name) - call_str = suite_part_call_list(host_model, const_dict, spart, False, - dyn_const=True) - cap.write(f"call {suite.name}_{stage}({call_str})", 3) - cap.write(f"if ({errflg_name} /= 0) then", 3) - cap.write("return", 4) - cap.write("end if", 3) - # Allocate the suite's dynamic constituents array - size_string = "0 +" - for var in host_local_vars.variable_list(): - vtype = var.get_prop_value('type') - if vtype == 'ccpp_constituent_properties_t': - local_name = var.get_prop_value('local_name') - size_string += f"size({local_name}) +" - # end if - # end for - if not has_dyn_consts: - cap.comment("Suite does not return dynamic constituents; allocate to zero", 3) - # end if - cap.write(f"allocate({dyn_const_array}({size_string[:-1]}))", 3) - if has_dyn_consts: - cap.comment("Pack the suite-level dynamic, run-time constituents array", 3) - cap.write("num_dyn_consts = 0", 3) - for var in host_local_vars.variable_list(): - vtype = var.get_prop_value('type') - if vtype != 'ccpp_constituent_properties_t': - continue - # end if - local_name = var.get_prop_value('local_name') - cap.write(f"do const_index = 1, size({local_name})", 3) - cap.write(f"{dyn_const_array}(num_dyn_consts + const_index) = {local_name}(const_index)", 4) - cap.write("end do", 3) - cap.write(f"num_dyn_consts = num_dyn_consts + size({local_name})", 3) - cap.write(f"deallocate({local_name})", 3) - # end for - - else: - spart = suite.phase_group(stage) - call_str = suite_part_call_list(host_model, const_dict, - spart, False) - stmt = "call {}_{}({})" - cap.write(stmt.format(suite.name, stage, call_str), 3) - # End if - else_str = 'else ' - # End for - cap.write("else", 2) - emsg = "write({errmsg}, '(3a)')".format(errmsg=errmsg_name) - emsg += '"No suite named ", ' - emsg += 'trim(suite_name), "found"' - cap.write(emsg, 3) - cap.write("{errflg} = 1".format(errflg=errflg_name), 3) - cap.write("end if", 2) - cap.write(_SUBFOOT.format(host_model=host_model.name, - stage=stage), 1) - # End for - # Write the API inspection routines (e.g., list of suites) - api.write_inspection_routines(cap) - # Write the constituent initialization interfaces - err_vars = host_model.find_error_variables() - const_obj_name = constituent_model_object_name(host_model) - cap.write("", 0) - const_names_name = constituent_model_const_stdnames(host_model) - const_indices_name = constituent_model_const_indices(host_model) - dyn_const_names = [suite_dynamic_constituent_array_name(host_model, suite.name) for suite in api.suites] - ConstituentVarDict.write_host_routines(cap, host_model, reg_name, init_name, - numconsts_name, queryconsts_name, - copyin_name, copyout_name, - cleanup_name, - const_obj_name, - dyn_const_names, - const_names_name, - const_indices_name, - const_array_func, - advect_array_func, - prop_array_func, - const_index_func, - api.suites, - err_vars) - # End with - return cap_filename - -############################################################################### - -if __name__ == "__main__": - from parse_tools import init_log, set_log_to_null - _LOGGER = init_log('host_registry') - set_log_to_null(_LOGGER) - # Run doctest - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/host_model.py b/scripts/host_model.py deleted file mode 100644 index c3beb447..00000000 --- a/scripts/host_model.py +++ /dev/null @@ -1,344 +0,0 @@ -#!/usr/bin/env python3 - -""" -Parse a host-model registry XML file and return the captured variables. -""" - -# CCPP framework imports -from constituents import CONST_DDT_NAME, CONST_PROP_TYPE, CONST_OBJ_STDNAME -from metavar import Var, VarDictionary -from ddt_library import VarDDT, DDTLibrary -from parse_tools import ParseContext, ParseSource, CCPPError, ParseInternalError -from parse_tools import context_string, registered_fortran_ddt_name -from parse_tools import FORTRAN_SCALAR_REF_RE - -############################################################################### -class HostModel(VarDictionary): - """Class to hold the data from a host model""" - - def __init__(self, meta_tables, name_in, run_env): - """Initialize this HostModel object. - is a dictionary of parsed host metadata tables. - - dictionary key is title of metadata argtable - is the name for this host model. - is the CCPPFrameworkEnv object for this framework run. - """ - self.__name = name_in - self.__var_locations = {} # Local name to module map - self.__loop_vars = None # Loop control vars in interface calls - self.__used_variables = None # Local names which have been requested - self.__deferred_finds = None # Used variables that were missed at first - self.__run_env = run_env - # First, process DDT headers - meta_headers = list() - for sect in [x.sections() for x in meta_tables.values()]: - meta_headers.extend(sect) - # end for - # Initialize our dictionaries - # Initialize variable dictionary - super().__init__(self.name, run_env) - ddt_headers = [d for d in meta_headers if d.header_type == 'ddt'] - self.__ddt_lib = DDTLibrary('{}_ddts'.format(self.name), run_env, - ddts=ddt_headers) - self.__ddt_dict = VarDictionary("{}_ddt_vars".format(self.name), - run_env, parent_dict=self) - del ddt_headers - # Now, process the code headers by type - self.__metadata_tables = meta_tables - for header in [h for h in meta_headers if h.header_type != 'ddt']: - title = header.title - if run_env.logger is not None: - msg = 'Adding {} {} to host model' - run_env.logger.debug(msg.format(header.header_type, title)) - # End if - if header.header_type == 'module': - # Set the variable modules - modname = header.title - for var in header.variable_list(): - self.add_variable(var, run_env) - lname = var.get_prop_value('local_name') - self.__var_locations[lname] = modname - self.ddt_lib.check_ddt_type(var, header, lname=lname) - if var.is_ddt(): - self.ddt_lib.collect_ddt_fields(self.__ddt_dict, var, - run_env) - # End if - # End for - elif header.header_type == 'host': - if self.__name is None: - # Grab the first host name we see - self.__name = header.name - # End if - for var in header.variable_list(): - self.add_variable(var, run_env) - self.ddt_lib.check_ddt_type(var, header) - if var.is_ddt(): - self.ddt_lib.collect_ddt_fields(self.__ddt_dict, var, - run_env) - # End if - # End for - loop_vars = header.variable_list(std_vars=False, - loop_vars=True, consts=False) - loop_vars.extend(self.__ddt_dict.variable_list(std_vars=False, - loop_vars=True, - consts=False)) - if loop_vars: - # loop_vars are part of the host-model interface call - # at run time. As such, they override the host-model - # array dimensions. - self.__loop_vars = VarDictionary(self.name, run_env) - # End if - for hvar in loop_vars: - std_name = hvar.get_prop_value('standard_name') - if std_name not in self.__loop_vars: - self.__loop_vars.add_variable(hvar, run_env) - else: - ovar = self.__loop_vars[std_name] - ctx1 = context_string(ovar.context) - ctx2 = context_string(hvar.context) - lname1 = ovar.get_prop_value('local_name') - lname2 = hvar.get_prop_value('local_name') - errmsg = ("Duplicate host loop var for {n}:\n" - " Dup: {l1}{c1}\n Orig: {l2}{c2}") - raise CCPPError(errmsg.format(n=self.name, - l1=lname1, c1=ctx1, - l2=lname2, c2=ctx2)) - # End if - # End for - else: - errmsg = "Invalid host model metadata header type, {} ({}){}" - errmsg += "\nType must be 'module' or 'host'" - ctx = context_string(header.context) - raise CCPPError(errmsg.format(header.title, - header.header_type, ctx)) - # End if - # End while - if self.name is None: - errmsg = 'No name found for host model, add a host metadata entry' - raise CCPPError(errmsg) - # End if - # Add in the constituents object - if registered_fortran_ddt_name(CONST_PROP_TYPE): - prop_dict = {'standard_name' : CONST_OBJ_STDNAME, - 'local_name' : self.constituent_model_object_name(), - 'dimensions' : '()', 'units' : "None", - 'ddt_type' : CONST_DDT_NAME, 'target' : 'True'} - host_source = ParseSource(self.ccpp_cap_name(), "MODULE", - ParseContext(filename=f"{self.ccpp_cap_name()}.F90")) - const_var = Var(prop_dict, host_source, run_env) - self.add_variable(const_var, run_env) - lname = const_var.get_prop_value('local_name') - self.__var_locations[lname] = self.ccpp_cap_name() - self.ddt_lib.collect_ddt_fields(self.__ddt_dict, const_var, run_env) - # end if - # Finally, turn on the use meter so we know which module variables - # to 'use' in a host cap. - self.__used_variables = set() # Local names which have been requested - self.__deferred_finds = set() # Used variables that were missed at first - - @property - def name(self): - """Return the host model name""" - return self.__name - - @property - def loop_vars(self): - """Return this host model's loop variables""" - return self.__loop_vars - - @property - def ddt_lib(self): - """Return this host model's DDT library""" - return self.__ddt_lib - - @property - def constituent_module(self): - """Return the name of host model constituent module""" - return f"{self.name}_ccpp_constituents" - - def argument_list(self, loop_vars=True): - """Return a string representing the host model variable arg list""" - args = [v.call_string(self) - for v in self.variable_list(loop_vars=loop_vars, consts=False)] - return ', '.join(args) - - def metadata_tables(self): - """Return a copy of this host models metadata tables""" - return dict(self.__metadata_tables) - - def host_variable_module(self, local_name): - """Return the module name for a host variable""" - if local_name in self.__var_locations: - return self.__var_locations[local_name] - # End if - return None - - def variable_locations(self): - """Return a set of module-variable and module-type pairs. - These represent the locations of all host model data with a listed - source location (variables with no source or for which the - source is the CCPP host cap are omitted).""" - varset = set() - lnames = self.prop_list('local_name') - # Attempt to realize deferred lookups - if self.__deferred_finds is not None: - for std_name in list(self.__deferred_finds): - var = self.find_variable(standard_name=std_name) - if var is not None: - self.__deferred_finds.remove(std_name) - # End if - # End for - # End if - # Now, find all the used module variables - cap_modname = self.ccpp_cap_name() - for name in lnames: - module = self.host_variable_module(name) - used = self.__used_variables and (name in self.__used_variables) - if module and used and (module != cap_modname): - varset.add((module, name)) - # No else, either no module or a zero-length module name - # End if - # End for - return varset - - def find_variable(self, standard_name=None, source_var=None, - any_scope=False, clone=None, - search_call_list=False, loop_subst=False): - """Return the host model variable matching or None - If is True, substitute a begin:end range for an extent. - """ - my_var = super().find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, clone=clone, - search_call_list=search_call_list, - loop_subst=loop_subst) - if my_var is None: - # Check our DDT library - if standard_name is None: - if source_var is None: - emsg = ("One of or " + - "must be passed.") - raise ParseInternalError(emsg) - # end if - standard_name = source_var.get_prop_value('standard_name') - # end if - # Since we are the parent of the DDT library, only check that dict - my_var = self.__ddt_dict.find_variable(standard_name=standard_name, - any_scope=False) - # End if - if loop_subst: - if my_var is None: - my_var = self.find_loop_subst(standard_name) - # End if - if my_var is not None: - # If we get here, the host does not have the requested - # variable but does have a replacement set. Create a new - # variable to use to send to suites. - ##XXgoldyXX: This cannot be working since find_loop_subst - ## returns a tuple - new_name = self.new_internal_variable_name(prefix=self.name) - ctx = ParseContext(filename='host_model.py') - new_var = my_var.clone(new_name, source_name=self.name, - source_type="HOST", - context=ctx) - self.add_variable(new_var, self.__run_env) - my_var = new_var - # End if - # End if - if my_var is None: - if self.__deferred_finds is not None: - self.__deferred_finds.add(standard_name) - # End if - elif self.__used_variables is not None: - lname = my_var.get_prop_value('local_name') - # Try to add any index references (should be method?) - imatch = FORTRAN_SCALAR_REF_RE.match(lname) - if imatch is not None: - vdims = [x.strip() for x in imatch.group(2).split(',') - if ':' not in x] - for vname in vdims: - _ = self.find_variable(standard_name=vname) - # End for - # End if - if isinstance(my_var, VarDDT): - lname = my_var.get_parent_prop('local_name') - # End if - self.__used_variables.add(lname) - # End if - return my_var - - def add_variable(self, newvar, run_env, exists_ok=False, gen_unique=False, - adjust_intent=False): - """Add if it does not conflict with existing entries. - For the host model, this includes entries in used DDT variables. - If is True, attempting to add an identical copy is okay. - If is True, a new local_name will be created if a - local_name collision is detected. - if is True, adjust conflicting intents to inout.""" - standard_name = newvar.get_prop_value('standard_name') - cvar = self.find_variable(standard_name=standard_name, any_scope=False) - if cvar is None: - # Check the DDT dictionary - cvar = self.__ddt_dict.find_variable(standard_name=standard_name, - any_scope=False) - # end if - if cvar and (not exists_ok): - emsg = "Attempt to add duplicate host model variable, {}{}." - emsg += "\nVariable originally defined{}" - ntx = context_string(newvar.context) - ctx = context_string(cvar.context) - raise CCPPError(emsg.format(standard_name, ntx, ctx)) - # end if - # No collision, proceed normally - super().add_variable(newvar=newvar, run_env=run_env, - exists_ok=exists_ok, gen_unique=gen_unique, - adjust_intent=False) - - def add_host_variable_module(self, local_name, module, logger=None): - """Add a module name location for a host variable""" - if local_name not in self.__var_locations: - if logger is not None: - emsg = 'Adding variable, {}, from module, {}' - logger.debug(emsg.format(local_name, module)) - # End if - self.__var_locations[local_name] = module - else: - emsg = "Host variable, {}, already located in module" - raise CCPPError(emsg.format(self.__var_locations[local_name])) - # End if - - def call_list(self, phase): - "Return the list of variables passed by the host model to the host cap" - hdvars = list() - loop_vars = phase == 'run' - for hvar in self.variable_list(loop_vars=loop_vars, consts=False): - lname = hvar.get_prop_value('local_name') - if self.host_variable_module(lname) is None: - hdvars.append(hvar) - # End if - # End for - return hdvars - - def constituent_model_object_name(self): - """Return the variable name of the object which holds the constituent - metadata and field information.""" - return "{}_constituents_obj".format(self.name) - - def ccpp_cap_name(self): - """Return the name of the CCPP host model cap module name.""" - return f"{self.name}_ccpp_cap" - -############################################################################### - -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - from parse_tools import init_log, set_log_to_null - import doctest - import sys - # pylint: enable=ungrouped-imports - _LOGGER = init_log('host_registry') - set_log_to_null(_LOGGER) - # First, run doctest - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/metadata2html.py b/scripts/metadata2html.py deleted file mode 100755 index 7e4a540d..00000000 --- a/scripts/metadata2html.py +++ /dev/null @@ -1,131 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import logging -import importlib -import os -import sys - -# CCPP framework imports -from common import CCPP_INTERNAL_VARIABLE_DEFINITON_FILE -from parse_checkers import registered_fortran_ddt_names -from parse_tools import init_log, set_log_level -from metadata_table import MetadataTable, parse_metadata_file -from framework_env import CCPPFrameworkEnv - -############################################################################### -# Set up the command line argument parser and other global variables # -############################################################################### - -parser = argparse.ArgumentParser() -method = parser.add_mutually_exclusive_group(required=True) -method.add_argument('--config', '-c', action='store', - help='path to CCPP prebuild configuration file') -method.add_argument('--metafile', '-m', action='store', - help='name of metadata file to convert (requires -o)') -parser.add_argument('--outputdir', '-o', action='store', - help='directory where to write the html files', - required='--metafile' in sys.argv or '-m' in sys.argv) - -# List and order of variable attributes to output to HTML -ATTRIBUTES = [ 'local_name', 'standard_name', 'long_name', 'units', - 'type', 'dimensions', 'kind', 'intent' ] - -############################################################################### -# Functions and subroutines # -############################################################################### - -def parse_arguments(): - """Parse command line arguments.""" - args = parser.parse_args() - config = args.config - filename = args.metafile - outdir = args.outputdir - return (config, filename, outdir) - -def import_config(configfile, logger): - """Import the configuration from a given configuration file""" - - if not os.path.isfile(configfile): - raise Exception("Configuration file {0} not found".format(configfile)) - - # Import the host-model specific CCPP prebuild config; - # split into path and module name for import - configpath = os.path.abspath(os.path.dirname(configfile)) - configmodule = os.path.splitext(os.path.basename(configfile))[0] - sys.path.append(configpath) - ccpp_prebuild_config = importlib.import_module(configmodule) - - # Get the base directory for running metadata2html.py from - # the default build directory value in the CCPP prebuild config - basedir = os.path.join(os.getcwd()) - logger.info('Relative path to CCPP directory from CCPP prebuild config: {}'.format( - ccpp_prebuild_config.DEFAULT_BUILD_DIR)) - - config = {} - # Definitions in host-model dependent CCPP prebuild config script - config['variable_definition_files'] = ccpp_prebuild_config.VARIABLE_DEFINITION_FILES - config['scheme_files'] = ccpp_prebuild_config.SCHEME_FILES - # Add model-independent, CCPP-internal variable definition files - config['variable_definition_files'].append(CCPP_INTERNAL_VARIABLE_DEFINITON_FILE) - # Output directory for converted metadata tables - config['metadata_html_output_dir'] = ccpp_prebuild_config.METADATA_HTML_OUTPUT_DIR.format(build_dir=basedir) - - return config - -def get_metadata_files_from_config(config, logger): - """Create a list of metadata filenames for a CCPP prebuild configuration""" - filenames = [] - for sourcefile in config['variable_definition_files'] + config['scheme_files']: - metafile = os.path.splitext(sourcefile)[0]+'.meta' - if os.path.isfile(metafile): - filenames.append(metafile) - else: - # DH* Warn for now, raise exception later when - # old metadata format is no longer supported - logger.warn("Metadata file {} for source file {} not found, assuming old metadata format".format( - metafile, sourcefile)) - return filenames - -def get_output_directory_from_config(config, logger): - """Return the html output directory for a CCPP prebuild configuration""" - outdir = config['metadata_html_output_dir'] - if not os.path.isdir(outdir): - raise Exception("Output directory {} for converted metadata tables does not exist".format(outdir)) - return outdir - -def convert_to_html(filename_in, outdir, logger, run_env): - """Convert a metadata file into html (one html file for each table)""" - if not os.path.isfile(filename_in): - raise Exception("Metadata file {} not found".format(filename_in)) - logger.info("Converting file {} to HTML".format(filename_in)) - metadata_headers = parse_metadata_file(filename_in, - known_ddts=registered_fortran_ddt_names(), - run_env=run_env) - for metadata_header in metadata_headers: - for metadata_section in metadata_header.sections(): - filename_out = metadata_section.to_html(outdir, ATTRIBUTES) - if filename_out: - logger.info(" ... wrote {}".format(filename_out)) - -def main(): - # Initialize logging - logger = init_log('metadata2html') - set_log_level(logger, logging.INFO) - run_env = CCPPFrameworkEnv(logger, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - - # Convert metadata file - (configfile, filename, outdir) = parse_arguments() - if configfile: - config = import_config(configfile, logger) - filenames = get_metadata_files_from_config(config, logger) - outdir = get_output_directory_from_config(config, logger) - for filename in filenames: - convert_to_html(filename, outdir, logger, run_env) - else: - convert_to_html(filename, outdir, logger, run_env) - -if __name__ == '__main__': - main() diff --git a/scripts/metadata_parser.py b/scripts/metadata_parser.py deleted file mode 100755 index 0b25538f..00000000 --- a/scripts/metadata_parser.py +++ /dev/null @@ -1,790 +0,0 @@ -#!/usr/bin/env python3 - -import collections -import logging -import os -import re -import subprocess -import sys -from xml.etree import ElementTree as ET - -from common import encode_container, CCPP_STAGES -from common import CCPP_ERROR_CODE_VARIABLE, CCPP_ERROR_MSG_VARIABLE -from common import insert_plus_sign_for_positive_exponents -from mkcap import Var - -sys.path.append(os.path.join(os.path.split(__file__)[0], 'fortran_tools')) -from parse_fortran import FtypeTypeDecl -from parse_checkers import registered_fortran_ddt_names -from parse_tools import init_log -from metadata_table import MetadataTable, parse_metadata_file -from framework_env import CCPPFrameworkEnv - -_API_LOGGING = init_log('metadata_parser') -_DUMMY_RUN_ENV = CCPPFrameworkEnv(_API_LOGGING, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -# Output: This routine converts the argument tables for all subroutines / typedefs / kind / module variables -# into dictionaries suitable to be used with ccpp_prebuild.py (which generates the fortran code for the caps) - -# Items in this dictionary are used for checking valid entries in metadata tables. For columns with no keys/keys -# commented out, no check is performed. This is the case for 'type' and 'kind' right now, since models use their -# own derived data types and kind types. -VALID_ITEMS = { - 'header' : ['local_name', 'standard_name', 'long_name', 'units', 'rank', 'type', 'kind', 'intent'], - #'type' : ['character', 'integer', 'real', ...], - #'kind' : ['default', 'kind_phys', ...], - 'intent' : ['none', 'in', 'out', 'inout'], - } - -# Mandatory variables that every scheme needs to have -CCPP_MANDATORY_VARIABLES = { - CCPP_ERROR_MSG_VARIABLE : Var(local_name = 'errmsg', - standard_name = CCPP_ERROR_MSG_VARIABLE, - long_name = 'error message for error handling in CCPP', - units = 'none', - type = 'character', - dimensions = [], - rank = '', - kind = 'len=*', - intent = 'out', - active = 'T', - ), - CCPP_ERROR_CODE_VARIABLE : Var(local_name = 'ierr', - standard_name = CCPP_ERROR_CODE_VARIABLE, - long_name = 'error code for error handling in CCPP', - units = '1', - type = 'integer', - dimensions = [], - rank = '', - kind = '', - intent = 'out', - active = 'T', - ), - } - -# Save metadata to avoid repeated parsing of type/variable definition files -NEW_METADATA_SAVE = {} - -############################################################################### - -def merge_dictionaries(x, y): - """Merges two metadata dictionaries. For each list of elements - (variables = class Var in mkcap.py) in one dictionary, we know - that all entries are compatible. If one or more elements exist - in both x and y, we therefore have to test compatibility of - one of the items in each dictionary only.""" - z = collections.OrderedDict() - x_keys = sorted(x.keys()) - y_keys = sorted(y.keys()) - z_keys = sorted(list(set(x_keys + y_keys))) - for key in z_keys: - z[key] = collections.OrderedDict() - if key in x_keys and key in y_keys: - # Metadata dictionaries containing lists of variables of type Var for each key=standard_name - if isinstance(x[key][0], Var): - # We know that all entries within each dictionary are compatible; - # we need to test compatibility of one of the items in each only. - if not x[key][0].compatible(y[key][0]): - raise Exception('Incompatible entries in metadata for variable {0}:\n'.format(key) +\ - ' {0}\n'.format(x[key][0].print_debug()) +\ - 'vs. {0}'.format(y[key][0].print_debug())) - z[key] = x[key] + y[key] - # Physics set dictionaries containing lists of physics sets of type string for each key=standard_name - elif type(x[key][0]) is str: - z[key] = list(set(x[key] + y[key])) - else: - raise Exception("x[key][0] is of unsupported type", type(x[key][0])) - elif key in x_keys: - z[key] = x[key] - elif key in y_keys: - z[key] = y[key] - return z - -def read_new_metadata(filename, module_name, table_name, scheme_name = None, subroutine_name = None): - """Read metadata in new format and convert output to ccpp_prebuild metadata dictionary""" - if not os.path.isfile(filename): - raise Exception("New metadata file {0} not found".format(filename)) - - # Save metadata, because this routine new_metadata - # is called once for every table in that file - if filename in NEW_METADATA_SAVE.keys(): - new_metadata_headers = NEW_METADATA_SAVE[filename] - else: - new_metadata_headers = parse_metadata_file(filename, known_ddts=registered_fortran_ddt_names(), - run_env=_DUMMY_RUN_ENV) - NEW_METADATA_SAVE[filename] = new_metadata_headers - - # Record dependencies for the metadata table (only applies to schemes) - dependencies = [] - - # Convert new metadata for requested table to old metadata dictionary - metadata = collections.OrderedDict() - for new_metadata_header in new_metadata_headers: - for metadata_section in new_metadata_header.sections(): - metadata_section_title = metadata_section.title.lower() - # Module or DDT tables - if not scheme_name: - # Module property tables - if not metadata_section_title == table_name: - # Skip this table, since it is not requested right now - continue - - # Distinguish between module argument tables and DDT argument tables - if metadata_section_title == module_name: - container = encode_container(module_name) - else: - container = encode_container(module_name, metadata_section_title) - - # Add to dependencies - if new_metadata_header.dependencies_path: - dependencies += [ os.path.join(new_metadata_header.dependencies_path, x) for x in new_metadata_header.dependencies] - else: - dependencies += new_metadata_header.dependencies - else: - # Scheme property tables - if not metadata_section_title == table_name: - # Skip this table, since it is not requested right now - continue - - container = encode_container(module_name, scheme_name, table_name) - - # Add to dependencies - if new_metadata_header.dependencies_path: - dependencies += [ os.path.join(new_metadata_header.dependencies_path, x) for x in new_metadata_header.dependencies] - else: - dependencies += new_metadata_header.dependencies - - for new_var in metadata_section.variable_list(): - standard_name = new_var.get_prop_value('standard_name') - # DH* 2020-05-26 - # Legacy extension for inconsistent metadata (use of horizontal_dimension versus horizontal_loop_extent). - # Since horizontal_dimension and horizontal_loop_extent have the same attributes (otherwise it doesn't - # make sense), we swap the standard name and add a note to the long name - 2021-05-26: this is now an error. - legacy_note = '' - if standard_name == 'horizontal_loop_extent' and scheme_name and \ - (table_name.endswith("_init") or table_name.endswith("_finalize")): - #logging.warn("Legacy extension - replacing variable 'horizontal_loop_extent'" + \ - # " with 'horizontal_dimension' in table {}".format(table_name)) - #standard_name = 'horizontal_dimension' - #legacy_note = ' replaced by horizontal dimension (legacy extension)' - raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_loop_extent'" + \ - " with 'horizontal_dimension' in table {}".format(table_name)) - elif standard_name == 'horizontal_dimension' and scheme_name and table_name.endswith("_run"): - #logging.warn("Legacy extension - replacing variable 'horizontal_dimension'" + \ - # " with 'horizontal_loop_extent' in table {}".format(table_name)) - #standard_name = 'horizontal_loop_extent' - #legacy_note = ' replaced by horizontal loop extent (legacy extension)' - raise Exception("Legacy extension DISABLED: replacing variable 'horizontal_dimension'" + \ - " with 'horizontal_loop_extent' in table {}".format(table_name)) - - # Adjust dimensions - dimensions = new_var.get_prop_value('dimensions') - if scheme_name and (table_name.endswith("_init") or table_name.endswith("_finalize")) \ - and 'horizontal_loop_extent' in dimensions: - #logging.warn("Legacy extension - replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ - # "for variable {} in table {}".format(standard_name,table_name)) - #dimensions = ['horizontal_dimension' if x=='horizontal_loop_extent' else x for x in dimensions] - raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_loop_extent' with 'horizontal_dimension' " + \ - "for variable {} in table {}".format(standard_name,table_name)) - elif scheme_name and table_name.endswith("_run") and 'horizontal_dimension' in dimensions: - #logging.warn("Legacy extension - replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ - # "for variable {} in table {}".format(standard_name,table_name)) - #dimensions = ['horizontal_loop_extent' if x=='horizontal_dimension' else x for x in dimensions] - raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ - "for variable {} in table {}".format(standard_name,table_name)) - elif not scheme_name and 'horizontal_dimension' in dimensions: - raise Exception("Legacy extension DISABLED: replacing dimension 'horizontal_dimension' with 'horizontal_loop_extent' " + \ - "for variable {} in table {}".format(standard_name,table_name)) - # *DH 2020-05-26 - - if not new_var.get_prop_value('active'): - raise Exception("Unexpected result: no active attribute received from capgen metadata parser for {} / {}".format(standard_name,table_name)) - elif scheme_name and not new_var.get_prop_value('active').lower() == '.true.': - raise Exception("Scheme variable {} in table {} has metadata attribute active={}, which is not allowed".format( - standard_name, table_name, new_var.get_prop_value('active').lower())) - elif new_var.get_prop_value('active').lower() == '.true.': - active = 'T' - elif new_var.get_prop_value('active').lower() == '.false.': - active = 'F' - else: - # Replace multiple whitespaces, preserve case - active = ' '.join(new_var.get_prop_value('active').split()) - - if not new_var.get_prop_value('optional') in [False, True]: - raise Exception("Unexpected result: no optional attribute received from metadata parser for {} / {}".format(standard_name,table_name)) - elif not scheme_name and new_var.get_prop_value('optional'): - raise Exception("Host variable {} in table {} has metadata attribute optional={}, which is not allowed".format( - standard_name,table_name, new_var.get_prop_value('optional').lower())) - elif new_var.get_prop_value('optional'): - optional = 'T' - else: - optional = 'F' - - # DH* 20210812 - # Workaround for Fortran DDTs incorrectly having the type of - # the DDT copied into the kind attribute in parse_metadata_file - if new_var.is_ddt() and new_var.get_prop_value('kind'): - kind = '' - else: - kind = new_var.get_prop_value('kind') - #kind = new_var.get_prop_value('kind') - # *DH 20210812 - - # Workaround to support units with positive exponents with - # and without a plus (+) sign. Internally, we convert all - # units from capgen to the "+"-format (i.e. "m2 s-2" --> "m+2 s-2") - units = insert_plus_sign_for_positive_exponents(new_var.get_prop_value('units')) - - var = Var(standard_name = standard_name.lower(), - long_name = new_var.get_prop_value('long_name') + legacy_note, - units = units, - local_name = new_var.get_prop_value('local_name').lower(), - type = new_var.get_prop_value('type').lower(), - dimensions = [dim.lower() for dim in dimensions], - container = container, - kind = kind, - intent = new_var.get_prop_value('intent'), - active = active, - optional = optional, - ) - # Check for duplicates in same table - if standard_name in metadata.keys(): - raise Exception("Error, multiple definitions of standard name {} in new metadata table {}".format(standard_name, table_name)) - metadata[standard_name] = [var] - - return (metadata, dependencies) - -def parse_variable_tables(filepath, filename): - """Parses metadata tables on the host model side that define the available variables. - Metadata tables can refer to variables inside a module or as part of a derived - datatype, which itself is defined inside a module (depending on the location of the - metadata table). Each variable (standard_name) can exist only once, i.e. each entry - (list of variables) in the metadata dictionary contains only one element - (variable = instance of class Var defined in mkcap.py)""" - # Set debug to true if logging level is debug - debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG - - # Final metadata container for all variables in file - metadata = collections.OrderedDict() - - # Registry of modules and derived data types in file - registry = collections.OrderedDict() - - # List of dependencies for this scheme - dependencies = collections.OrderedDict() - - # Read all lines of the file at once - with (open(filename, 'r')) as file: - try: - file_lines = file.readlines() - except UnicodeDecodeError: - raise Exception("Decoding error while trying to read file {}, check that the file only contains ASCII characters".format(filename)) - - lines = [] - buffer = '' - for i in range(len(file_lines)): - file_lines[i] = file_lines[i].lower() - line = file_lines[i].rstrip('\n').strip() - # Skip empty lines - if line == '' or line == '&': - continue - # Remove line continuations: concatenate with following lines - if line.endswith('&'): - buffer += file_lines[i].rstrip('\n').replace('&', ' ') - continue - # Write out line with buffer and reset buffer - lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' ')) - buffer = '' - del file_lines - - # Find all modules within the file, and save the start and end lines - module_lines = collections.OrderedDict() - line_counter = 0 - for line in lines: - words = line.split() - if len(words) > 1 and words[0].lower() in ['module', 'program'] and not words[1].lower() == 'procedure': - module_name = words[1].strip() - if module_name in registry.keys(): - raise Exception('Duplicate module name {0}'.format(module_name)) - registry[module_name] = collections.OrderedDict() - module_lines[module_name] = { 'startline' : line_counter } - elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower() in ['module', 'program']: - try: - test_module_name = words[2] - except IndexError: - logging.warning('Encountered closing statement "end module" without module name; assume module_name is {0}'.format(module_name)) - test_module_name = module_name - if not module_name == test_module_name: - raise Exception('Module names in opening/closing statement do not match: {0} vs {1}'.format(module_name, test_module_name)) - module_lines[module_name]['endline'] = line_counter - line_counter += 1 - - # Parse each module in the file separately - for module_name in registry.keys(): - startline = module_lines[module_name]['startline'] - endline = module_lines[module_name]['endline'] - line_counter = 0 - in_type = False - for line in lines[startline:endline]: - # For the purpose of identifying module, type and scheme constructs, remove any trailing comments from line - if '!' in line and not line.startswith('!'): - line = line[:line.find('!')] - current_line_number = startline + line_counter - words = line.split() - for j in range(len(words)): - # Check for the word 'type', that it is the first word in the line, - # and that a name exists afterwards. It is assumed that definitions - # (not usage) of derived types cannot be nested - reasonable for Fortran. - # The following if / elif / else statements filter lines that do not - # contain a type definition. - # - # Ignore words containing type that are not 'type', 'type,', 'type::'; - # this includes variable declarations of a user defined type, e.g. 'type(mytype) ::' - if not (words[j].lower()=='type' or \ - words[j].lower().startswith('type,') or \ - words[j].lower().startswith('type::')): - continue - # Ignore variable declarations of a user defined type with a space - # between 'type' and '(', e.g. 'type (mytype) ::' - elif j == 0 and len(words) > 1 and words[j+1].startswith('('): - continue - # Ignore lines starting with 'type is' or 'type is(' (select type statements) - elif (words[j].lower() == 'type' and j==0 and j0: - continue - # Detect type definition using FtypeTypeDecl class, routine - # type_def_line and extract type_name - else: - type_declaration = FtypeTypeDecl.type_def_line(line.strip()) - if in_type: - raise Exception('Nested definitions of derived types not supported') - in_type = True - type_name = type_declaration[0] - if type_name in registry[module_name].keys(): - raise Exception('Duplicate derived type name {0} in module {1}'.format( - type_name, module_name)) - registry[module_name][type_name] = [current_line_number] - # Done with user defined type detection - line_counter += 1 - logging.debug('Parsing file {0} with registry {1}'.format(filename, registry)) - - # Variables can either be defined at module-level or in derived types - alongside with their tables - line_counter = 0 - in_table = False - in_type = False - for line in lines[startline:endline]: - current_line_number = startline + line_counter - - # Check for beginning of new table - words = line.split() - # This is case sensitive - if len(words) > 2 and words[0] in ['!!', '!>'] and '\section' in words[1] and 'arg_table_' in words[2]: - if in_table: - raise Exception('Encountered table start for table {0} while still in table {1}'.format(words[2].replace('arg_table_',''), table_name)) - table_name = words[2].replace('arg_table_','') - if not (table_name == module_name or table_name in registry[module_name].keys()): - raise Exception('Encountered table with name {0} without corresponding module or type name'.format(table_name)) - in_table = True - if not table_name in dependencies.keys(): - dependencies[table_name] = [] - header_line_number = current_line_number + 1 - line_counter += 1 - continue - # If an argument table is found, parse it - if in_table: - words = line.split('|') - # Separate the table headers - if current_line_number == header_line_number: - if 'htmlinclude' in line.lower(): - words = line.split() - if words[0] == '!!' and words[1] == '\\htmlinclude' and len(words) == 3: - filename_parts = filename.split('.') - metadata_filename = '.'.join(filename_parts[0:len(filename_parts)-1]) + '.meta' - (this_metadata, these_dependencies) = read_new_metadata(metadata_filename, module_name, table_name) - if these_dependencies: - # Remove duplicates when combining lists - dependencies[table_name] = list(set(dependencies[table_name] + these_dependencies)) - for var_name in this_metadata.keys(): - for var in this_metadata[var_name]: - if var_name in CCPP_MANDATORY_VARIABLES.keys() and not CCPP_MANDATORY_VARIABLES[var_name].compatible(var): - raise Exception('Entry for variable {0}'.format(var_name) + \ - ' in argument table {0}'.format(table_name) +\ - ' is incompatible with mandatory variable:\n' +\ - ' existing: {0}\n'.format(CCPP_MANDATORY_VARIABLES[var_name].print_debug()) +\ - ' vs. new: {0}'.format(var.print_debug())) - # Add variable to metadata dictionary - if not var_name in metadata.keys(): - metadata[var_name] = [var] - else: - for existing_var in metadata[var_name]: - if not existing_var.compatible(var): - raise Exception('New entry for variable {0}'.format(var_name) + \ - ' in argument table {0}'.format(table_name) +\ - ' is incompatible with existing entry:\n' +\ - ' existing: {0}\n'.format(existing_var.print_debug()) +\ - ' vs. new: {0}'.format(var.print_debug())) - - metadata[var_name].append(var) - else: - raise Exception("Invalid definition of new metadata format in file {}, \htmlinclude must be preceeded by '!! ' : {}".format(filename, line)) - line_counter += 1 - continue - # Check for blank table - if len(words) <= 1: - logging.debug('Skipping blank table {0}'.format(table_name)) - in_table = False - line_counter += 1 - continue - table_header = [x.strip() for x in words[1:-1]] - # Check that only valid table headers are used - for item in table_header: - if not item in VALID_ITEMS['header']: - raise Exception('Invalid column header {0} in argument table {1}'.format(item, table_name)) - # Locate mandatory column 'standard_name' - try: - standard_name_index = table_header.index('standard_name') - except ValueError: - raise Exception('Mandatory column standard_name not found in argument table {0}'.format(table_name)) - line_counter += 1 - # DH* warn or raise error for old metadata format - logging.warn("Old metadata table found for table {}".format(table_name)) - #raise Exception("Old metadata table found for table {}".format(table_name)) - # *DH - continue - else: - if len(words) == 1: - # End of table - if words[0].strip() == '!!': - if not current_line_number == header_line_number+1: - raise Exception("Invalid definition of new metadata format in file {0}".format(filename)) - in_table = False - line_counter += 1 - continue - else: - raise Exception('Encountered invalid line "{0}" in argument table {1}'.format(line, table_name)) - else: - raise Exception("Invalid definition of metadata in file {0}: {1}".format(filename, words)) - - line_counter += 1 - - # Informative output to screen - if debug and len(metadata.keys()) > 0: - for module_name in registry.keys(): - logging.debug('Module name: {0}'.format(module_name)) - container = encode_container(module_name) - vars_in_module = [] - for var_name in metadata.keys(): - for var in metadata[var_name]: - if var.container == container: - vars_in_module.append(var_name) - logging.debug('Module variables: {0}'.format(', '.join(vars_in_module))) - for type_name in registry[module_name].keys(): - container = encode_container(module_name, type_name) - vars_in_type = [] - for var_name in metadata.keys(): - for var in metadata[var_name]: - if var.container == container: - vars_in_type.append(var_name) - logging.debug('Variables in derived type {0}: {1}'.format(type_name, ', '.join(vars_in_type))) - - if len(metadata.keys()) > 0: - logging.info('Parsed variable definition tables in module {0}'.format(module_name)) - - # Add absolute path to dependencies - for table_name in dependencies.keys(): - if dependencies[table_name]: - dependencies[table_name] = [ os.path.join(filepath, x) for x in dependencies[table_name]] - for dependency in dependencies[table_name]: - if not os.path.isfile(dependency): - raise Exception("Dependency {} for variable table {} does not exit".format(dependency, table_name)) - - return (metadata, dependencies) - - -def parse_scheme_tables(filepath, filename): - """Parses metadata tables for a physics scheme that requests/requires variables as - input arguments. Metadata tables can only describe variables required by a subroutine - 'subroutine_name' of scheme 'scheme_name' inside a module 'module_name'. Each variable - (standard_name) can exist only once, i.e. each entry (list of variables) in the metadata - dictionary contains only one element (variable = instance of class Var defined in - mkcap.py). The metadata dictionaries of the individual schemes are merged afterwards - (called from ccpp_prebuild.py) using merge_metadata_dicts, where multiple instances - of variables are compared for compatibility and collected in a list (entry in the - merged metadata dictionary). The merged metadata dictionary of all schemes (which - contains only compatible variable instances in the list referred to by standard_name) - is then compared to the unique definition in the metadata dictionary of the variables - provided by the host model using compare_metadata in ccpp_prebuild.py.""" - - # Set debug to true if logging level is debug - debug = logging.getLogger().getEffectiveLevel() == logging.DEBUG - - # Final metadata container for all variables in file - metadata = collections.OrderedDict() - - # Registry of modules and derived data types in file - registry = collections.OrderedDict() - - # Argument lists of each subroutine in the file - arguments = collections.OrderedDict() - - # List of Dependencies for this scheme - dependencies = collections.OrderedDict() - - # Read all lines of the file at once - with (open(filename, 'r')) as file: - try: - file_lines = file.readlines() - except UnicodeDecodeError: - raise Exception("Decoding error while trying to read file {}, check that the file only contains ASCII characters".format(filename)) - - lines = [] - original_line_numbers = [] - buffer = '' - for i in range(len(file_lines)): - file_lines[i] = file_lines[i].lower() - line = file_lines[i].rstrip('\n').strip() - # Skip empty lines - if line == '' or line == '&': - continue - # Remove line continuations: concatenate with following lines - if line.endswith('&'): - buffer += file_lines[i].rstrip('\n').replace('&', ' ') - continue - # Write out line with buffer and reset buffer - lines.append(buffer + file_lines[i].rstrip('\n').replace('&', ' ')) - original_line_numbers.append(i+1) - buffer = '' - del file_lines - - # Find all modules within the file, and save the start and end lines - module_lines = collections.OrderedDict() - line_counter = 0 - for line in lines: - # For the purpose of identifying module constructs, remove any trailing comments from line - if '!' in line and not line.startswith('!'): - line = line[:line.find('!')] - words = line.split() - if len(words) > 1 and words[0].lower() == 'module' and not words[1].lower() == 'procedure': - module_name = words[1].strip() - if module_name in registry.keys(): - raise Exception('Duplicate module name {0}'.format(module_name)) - registry[module_name] = collections.OrderedDict() - module_lines[module_name] = { 'startline' : line_counter } - elif len(words) > 1 and words[0].lower() == 'end' and words[1].lower() == 'module': - try: - test_module_name = words[2] - except IndexError: - logging.warning('Warning, encountered closing statement "end module" without module name; assume module_name is {0}'.format(module_name)) - test_module_name = module_name - if not module_name == test_module_name: - raise Exception('Module names in opening/closing statement do not match: {0} vs {1}'.format(module_name, test_module_name)) - module_lines[module_name]['endline'] = line_counter - line_counter += 1 - - # Parse each module in the file separately - for module_name in registry.keys(): - startline = module_lines[module_name]['startline'] - endline = module_lines[module_name]['endline'] - line_counter = 0 - in_subroutine = False - for line in lines[startline:endline]: - # For the purpose of identifying scheme constructs, remove any trailing comments from line - if '!' in line and not line.startswith('!'): - line = line[:line.find('!')] - current_line_number = startline + line_counter - words = line.split() - for j in range(len(words)): - # Check for the word 'subroutine', that it is the first word in the line, - # and that a name exists afterwards. Nested subroutines are ignored. - if words[j].lower() == 'subroutine' and j == 0 and len(words) > 1: - if in_subroutine: - logging.debug('Warning, ignoring nested subroutine in module {0} and subroutine {1}'.format(module_name, subroutine_name)) - continue - subroutine_name = words[j+1].split('(')[0].strip() - # Consider the last substring separated by a '_' of the subroutine name as a 'postfix' - if subroutine_name.find('_') >= 0: - scheme_name = None - subroutine_suffix = None - for ccpp_stage in CCPP_STAGES: - pattern = '^(.*)_{}$'.format(ccpp_stage) - match = re.match(pattern, subroutine_name) - if match: - scheme_name = match.group(1) - subroutine_suffix = ccpp_stage - break - if match: - if not scheme_name == module_name: - raise Exception('Scheme name differs from module name: module_name="{0}" vs. scheme_name="{1}"'.format( - module_name, scheme_name)) - if not scheme_name in registry[module_name].keys(): - registry[module_name][scheme_name] = collections.OrderedDict() - if subroutine_name in registry[module_name][scheme_name].keys(): - raise Exception('Duplicate subroutine name {0} in module {1}'.format( - subroutine_name, module_name)) - registry[module_name][scheme_name][subroutine_name] = [current_line_number] - in_subroutine = True - elif words[j].lower() == 'subroutine' and j == 1 and words[j-1].lower() == 'end': - try: - test_subroutine_name = words[j+1] - except IndexError: - logging.warning('Warning, encountered closing statement "end subroutine" without subroutine name; ' +\ - ' assume subroutine_name is {0}'.format(subroutine_name)) - test_subroutine_name = subroutine_name - if in_subroutine and subroutine_name == test_subroutine_name: - in_subroutine = False - registry[module_name][scheme_name][subroutine_name].append(current_line_number) - # Avoid problems by enforcing end statements to carry a descriptor (subroutine, module, ...) - elif in_subroutine and len(words) == 1 and words[0].lower() == 'end': - raise Exception('Encountered closing statement "end" without descriptor (subroutine, module, ...): ' +\ - 'line {0}="{1}" in file {2}'.format(original_line_numbers[current_line_number], line, filename)) - line_counter += 1 - - # Check that for each registered subroutine the start and end lines were found - for scheme_name in registry[module_name].keys(): - for subroutine_name in registry[module_name][scheme_name].keys(): - if not len(registry[module_name][scheme_name][subroutine_name]) == 2: - raise Exception('Error parsing start and end lines for subroutine {0} in module {1}'.format(subroutine_name, module_name)) - logging.debug('Parsing file {0} with registry {1}'.format(filename, registry)) - - for scheme_name in registry[module_name].keys(): - # Record the dependencies for the scheme - if not scheme_name in dependencies.keys(): - dependencies[scheme_name] = [] - for subroutine_name in registry[module_name][scheme_name].keys(): - # Record the order of variables in the call list to each subroutine in a list - if not scheme_name in arguments.keys(): - arguments[scheme_name] = collections.OrderedDict() - if not subroutine_name in arguments[scheme_name].keys(): - arguments[scheme_name][subroutine_name] = [] - # Find the argument table corresponding to each subroutine by searching - # "upward" from the subroutine definition line for the "arg_table_SubroutineName" section - table_found = False - header_line_number = None - for line_number in range(registry[module_name][scheme_name][subroutine_name][0], -1, -1): - line = lines[line_number] - words = line.split() - for word in words: - if (len(words) > 2 and words[0] in ['!!', '!>'] and '\section' in words[1] and 'arg_table_{0}'.format(subroutine_name) in words[2]): - table_found = True - header_line_number = line_number + 1 - table_name = subroutine_name - break - else: - for word in words: - if 'arg_table_{0}'.format(subroutine_name) in word: - raise Exception("Malformatted table found in {0} / {1} / {2} / {3}".format(filename, module_name, scheme_name, subroutine_name)) - if table_found: - break - # If an argument table is found, parse it - if table_found: - if 'htmlinclude' in lines[header_line_number].lower(): - words = lines[header_line_number].split() - if words[0] == '!!' and words[1] == '\\htmlinclude' and len(words) == 3: - filename_parts = filename.split('.') - metadata_filename = '.'.join(filename_parts[0:len(filename_parts)-1]) + '.meta' - (this_metadata, these_dependencies) = read_new_metadata(metadata_filename, module_name, table_name, - scheme_name=scheme_name, subroutine_name=subroutine_name) - if these_dependencies: - # Remove duplicates when combining lists - dependencies[scheme_name] = list(set(dependencies[scheme_name] + these_dependencies)) - for var_name in this_metadata.keys(): - # Add standard_name to argument list for this subroutine - arguments[scheme_name][subroutine_name].append(var_name) - # For all instances of this var (can be only one) in this subroutine's metadata, - # add to global metadata and check for compatibility with existing variables - for var in this_metadata[var_name]: - if not var_name in metadata.keys(): - metadata[var_name] = [var] - else: - for existing_var in metadata[var_name]: - if not existing_var.compatible(var): - raise Exception('New entry for variable {0}'.format(var_name) + \ - ' in argument table of subroutine {0}'.format(subroutine_name) +\ - ' is incompatible with existing entry:\n' +\ - ' existing: {0}\n'.format(existing_var.print_debug()) +\ - ' vs. new: {0}'.format(var.print_debug())) - metadata[var_name].append(var) - else: - raise Exception("Invalid definition of new metadata format in file {}, \htmlinclude must be preceeded by '!! ' : {}".format(filename, lines[header_line_number])) - # Next line must denote the end of table, - # i.e. look for a line containing only '!!' - line_number = header_line_number+1 - nextline = lines[line_number] - nextwords = nextline.split() - if len(nextwords) == 1 and nextwords[0].strip() == '!!': - end_of_table = True - else: - raise Exception('Encountered invalid format "{0}" of new metadata table hook in table {1}'.format(line, table_name)) - line_number += 1 - - else: - words = lines[header_line_number].split() - if len(words) == 1 and words[0].strip() == '!!': - logging.info("Legacy extension - skip empty table for {}".format(table_name)) - end_of_table = True - line_number += 1 - else: - raise Exception("Invalid definition of metadata in file {0}: {1}".format(filename, words)) - - # After parsing entire metadata table for the subroutine, check that all - # mandatory CCPP variables are present - skip empty tables. - if arguments[scheme_name][subroutine_name]: - for var_name in CCPP_MANDATORY_VARIABLES.keys(): - if not var_name in arguments[scheme_name][subroutine_name]: - raise Exception('Mandatory CCPP variable {0} not declared in metadata table of subroutine {1}'.format( - var_name, subroutine_name)) - # Sort the dependencies to avoid differences in the auto-generated code - dependencies[scheme_name].sort() - - # Debugging output to screen and to XML - if debug and len(metadata.keys()) > 0: - # To screen - logging.debug('Module name: {}'.format(module_name)) - for scheme_name in registry[module_name].keys(): - logging.debug('Scheme name: {}'.format(scheme_name)) - logging.debug('Scheme dependencies: {}'.format(dependencies[scheme_name])) - for subroutine_name in registry[module_name][scheme_name].keys(): - container = encode_container(module_name, scheme_name, subroutine_name) - vars_in_subroutine = [] - for var_name in metadata.keys(): - for var in metadata[var_name]: - if var.container == container: - vars_in_subroutine.append(var_name) - logging.debug('Variables in subroutine {}: {}'.format(subroutine_name, ', '.join(vars_in_subroutine))) - # Standard output to screen - elif len(metadata.keys()) > 0: - for scheme_name in registry[module_name].keys(): - if dependencies[scheme_name]: - logging.info('Parsed tables in scheme {} with dependencies {}'.format(scheme_name, dependencies[scheme_name])) - else: - logging.info('Parsed tables in scheme {}'.format(scheme_name)) - - # End of loop over all module_names - - # Add absolute path to dependencies - for scheme_name in dependencies.keys(): - if dependencies[scheme_name]: - dependencies[scheme_name] = [ os.path.join(filepath, x) for x in dependencies[scheme_name]] - for dependency in dependencies[scheme_name]: - if not os.path.isfile(dependency): - raise Exception("Dependency {} for scheme table {} does not exit".format(dependency, scheme_name)) - - return (metadata, arguments, dependencies) diff --git a/scripts/metadata_table.py b/scripts/metadata_table.py deleted file mode 100755 index b657dd2c..00000000 --- a/scripts/metadata_table.py +++ /dev/null @@ -1,1504 +0,0 @@ -#!/usr/bin/env python3 -""" -There are four types of CCPP metadata tables, scheme, module, ddt, and host. -A metadata file contains one or more metadata tables. -A metadata file SHOULD NOT mix metadata table types. The exception is a - metadata file which contains one or more ddt tables followed by a module - or host table. - -Each metadata table begins with a 'ccpp-table-properties' section followed by - one or more 'ccpp-arg-table' sections. These sections are described below. -A 'ccpp-arg-table' section is followed by one or more variable declaration - sections, also described below. - -Metadata headers are in config file format. - -A 'ccpp-table-properties' section entries are: -name = : the name of the following ccpp-arg-table entries (required). - It is one of the following possibilities: - - SchemeName: the name of a scheme (i.e., the name of - a scheme interface (related to SubroutineName below). - - DerivedTypeName: a derived type name for a type which will be used - somewhere in the CCPP interface. - - ModuleName: the name of the module whose module variables will be - used somewhere in the CCPP interface - - HostName: the name of the host model. Variables in this section become - part of the CCPP UI, the CCPP routines called by the - host model (e.g., _ccpp_physics_run). -type = : The type of header (required), one of: - - scheme: A CCPP subroutine - - ddt: A header for a derived data type - - module: A header on some module data - - host: A header on data which will be part of the CCPP UI -dependencies = : Comma-separated list of module dependencies - Each item should appear in one or more use statements in the - corresponding Fortran module -dependencies_path = : A path, relative to the location of - metadata table file, where dependencies can be found. -module_name = : only needed if module name differs from filename -source_path = -dynamic_constituent_routine = ??? : @peverwhee? -kind_spec = : One or more optional Fortran kinds defined - in the corresponding Fortran file. - The format is fortran_module:ccpp_kind_name=>kind_name or - fortran_module:kind_name - - is the module name of the corresponding Fortran module - - is defined in the corresponding Fortran module - - is optional and describes the kind name used in CCPP - metadata tables and Fortran files. - These entries are added to the framework_env object and - thus to ccpp_kinds.F90 - The entries in ccpp_kinds.F90 are: - use , only - use , only => - where the first form is used if is omitted. - -The ccpp-arg-table section entries in this section are: -name = : the name of the file object which immediately follows the - argument table (required). - It is one of the following possibilities: - - SubroutineName: the name of a subroutine (i.e., the name of - a scheme interface function such as SchemeName_run) - - DerivedTypeName: a derived type name for a type which will be used - somewhere in the CCPP interface. - - ModuleName: the name of the module whose module variables will be - used somewhere in the CCPP interface - - HostName: the name of the host model. Variables in this section become - part of the CCPP UI, the CCPP routines called by the - host model (e.g., _ccpp_physics_run). -type = : The type of header (required). It must match the type of the - associated ccpp-table-properties section (see above). - -A variable declaration section begins with a variable name line (a local -variable name enclosed in square brackets) followed by one or more -variable attribute statements. -A variable attribute statement is an attribute name and the value for -that attribute separated by an equal sign. Whitespace is not -significant except inside of strings. -Variable attribute statements may be combined on a line if separated by -a vertical bar. - -An example argument table is shown below. - -[ccpp-table-properties] - name = - type = scheme - dependencies_path = - dependencies = - module_name = # only needed if module name differs from filename - source_path = - dynamic_constituent_routine = - -[ccpp-arg-table] - name = - type = scheme -[ im ] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent, start at 1 - units = index - type = integer - dimensions = () - intent = in -[ ix ] - standard_name = horizontal_loop_dimension - long_name = horizontal dimension - units = index | type = integer | dimensions = () - intent = in - ... -[ errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - type = character - len = * - dimensions = () - intent = out -[ ierr ] - standard_name = ccpp_error_code - long_name = error flag for error handling in CCPP - type = integer - units = 1 - dimensions = () - intent=out - -Notes on the input format: -- SubroutineName must match the name of the subroutine that the argument - table describes -- DerivedTypeName must match the name of the derived type that the argument - table describes -- ModuleName must match the name of the module whose variables the argument - table describes -- for variable type definitions and module variables, the intent keyword - is not functional and should be omitted -- each argument table (and its subroutine) must accept the following two arguments for error handling (the local name can vary): -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -""" - -# Python library imports -import difflib -import logging -import os.path -import re -# CCPP framework imports -from ccpp_state_machine import CCPP_STATE_MACH -from metavar import Var, VarDictionary, CCPP_CONSTANT_VARS -from parse_tools import ParseObject, ParseSource, ParseContext, context_string -from parse_tools import ParseInternalError, ParseSyntaxError, CCPPError -from parse_tools import FORTRAN_ID, FORTRAN_SCALAR_REF, FORTRAN_SCALAR_REF_RE -from parse_tools import check_fortran_ref, check_fortran_id -from parse_tools import check_fortran_intrinsic -from parse_tools import register_fortran_ddt_name, unique_standard_name - -######################################################################## - -SCHEME_HEADER_TYPE = 'scheme' -_SINGLETON_TABLE_TYPES = ['ddt', 'host', 'module'] # Only one section per table -TABLE_TYPES = _SINGLETON_TABLE_TYPES + [SCHEME_HEADER_TYPE] -HEADER_TYPES = TABLE_TYPES + ['local'] -UNKNOWN_PROCESS_TYPE = 'UNKNOWN' - -_BLANK_LINE = re.compile(r"\s*[#;]") - -def blank_metadata_line(line): - """Return True if is a valid config format blank or comment - line. Also return True if we have reached the end of the file - (no line)""" - return (not line) or (_BLANK_LINE.match(line) is not None) - -######################################################################## - -def _parse_config_line(line, context): - """Parse a config line and return a list of keyword value pairs.""" - parse_items = [] - if line is None: - pass # No properties on this line - elif blank_metadata_line(line): - pass # No properties on this line - else: - properties = line.strip().split('|') - for prop in properties: - pitems = [x.strip() for x in prop.split('=', 1)] - if len(pitems) >= 2: - parse_items.append(pitems) - else: - raise ParseSyntaxError("variable property syntax", - token=prop, context=context) - # end if - # end for - # end if - return parse_items - -######################################################################## - -def parse_metadata_file(filename, known_ddts, run_env, skip_ddt_check=False, relative_source_path=False): - """Parse and return list of parsed metadata tables""" - # Read all lines of the file at once - meta_tables = [] - table_titles = [] # Keep track of names in file - with open(filename, 'r') as infile: - fin_lines = infile.readlines() - for index, fin_line in enumerate(fin_lines): - fin_lines[index] = fin_line.rstrip('\n') - # end for - # end with - # Look for a header start - parse_obj = ParseObject(filename, fin_lines) - curr_line, curr_line_num = parse_obj.curr_line() - while curr_line is not None: - if MetadataTable.table_start(curr_line): - new_table = MetadataTable(run_env, parse_object=parse_obj, - known_ddts=known_ddts, - skip_ddt_check=skip_ddt_check, - relative_source_path=relative_source_path) - ntitle = new_table.table_name - if ntitle not in table_titles: - meta_tables.append(new_table) - table_titles.append(ntitle) - if new_table.table_type == 'ddt': - known_ddts.append(ntitle.lower()) - # end if - else: - errmsg = 'Duplicate metadata table, {}, at {}:{}' - ctx = curr_line_num + 1 - raise CCPPError(errmsg.format(ntitle, filename, ctx)) - # end if - curr_line, curr_line_num = parse_obj.curr_line() - elif blank_metadata_line(curr_line): - curr_line, curr_line_num = parse_obj.next_line() - else: - raise ParseSyntaxError('CCPP metadata line', token=curr_line, - context=parse_obj) - # end if - # end while - return meta_tables - -######################################################################## - -def find_scheme_names(filename): - """Find and return a list of all the physics scheme names in - . A scheme is identified by its ccpp-table-properties name. - """ - scheme_names = [] - with open(filename, 'r') as infile: - fin_lines = infile.readlines() - # end with - num_lines = len(fin_lines) - context = ParseContext(linenum=1, filename=filename) - while context.line_num <= num_lines: - if MetadataTable.table_start(fin_lines[context.line_num - 1]): - found_start = False - while not found_start: - line = fin_lines[context.line_num].strip() - context.line_num += 1 - if line and (line[0] == '['): - found_start = True - elif line: - props = _parse_config_line(line, context) - for prop in props: - # Look for name property - key = prop[0].lower() - value = prop[1] - if key == 'name': - scheme_names.append(value) - # end if - # end for - # end if - if context.line_num > num_lines: - break - # end if - # end while - else: - context.line_num += 1 - # end if - # end while - return scheme_names - -######################################################################## - -def register_ddts(file_list): - """Scan the metadata files in and register all - DDT tables found. - Return a list of the DDTs type names found. - """ - errors = "" - ddt_names = set() - for mfile in file_list: - if os.path.exists(mfile): - with open(mfile, 'r') as infile: - fin_lines = infile.readlines() - # end with - pobj = ParseObject(mfile, fin_lines) - in_table = False # Line number of table start - ddt_name = "" - table_is_ddt = False - # Search the file for ccpp-table-properties sections - curr_line, line_num = pobj.next_line() - while(curr_line is not None): - if in_table: - # We are in a table properties sec, look for name and type - if MetadataSection.header_start(curr_line) or \ - MetadataTable.table_start(curr_line): - # We have exited the table, record if a DDT - if table_is_ddt: - if ddt_name: - ddt_names.add(ddt_name) - else: - emsg = "Unnamed CCPP metadata table" - pobj.add_syntax_err(emsg) - # end if - # end if - if MetadataTable.table_start(curr_line): - in_table = line_num + 1 - else: - in_table = False - # end if - ddt_name = "" - table_is_ddt = False - else: - for prop in _parse_config_line(curr_line, context=pobj): - if prop[0].lower() == 'name': - ddt_name = prop[1].lower() - elif prop[0].lower() == 'type': - table_is_ddt = prop[1].lower() == 'ddt' - # end if - # end for - # end if - elif MetadataTable.table_start(curr_line): - in_table = line_num + 1 - # end if - curr_line, line_num = pobj.next_line() - # end while - if pobj.error_message: - if errors: - errors += "\n" - # end if - errors += pobj.error_message - # end if - else: - if errors: - errors += "\n" - # end if - errors += f"Metadata file, '{mfile}', not found." - # end if - # end for - if in_table: - # This is a malformed CCPP metadata file! - if errors: - errors += "\n" - # end if - errors += f"Malformed CCPP metadata file, '{mfile}'" - # end if - if errors: - raise CCPPError(f"{errors}") - else: - for ddt in ddt_names: - register_fortran_ddt_name(ddt) - # end for - # end if - return list(ddt_names) - -######################################################################## - -class MetadataTable(): - """Class to hold a CCPP Metadata table including the table header - (ccpp-table-properties section) and all of the associated table - sections (ccpp-arg-table sections).""" - - __table_start = re.compile(r"(?i)\s*\[\s*ccpp-table-properties\s*\]") - - def __init__(self, run_env, table_name_in=None, table_type_in=None, - dependencies=None, dependencies_path=None, source_path=None, - known_ddts=None, var_dict=None, module=None, parse_object=None, - skip_ddt_check=False, relative_source_path=False): - """Initialize a MetadataTable, either with a name, , and - type, , or with information from a file (). - if is None, , , and - are are also stored. - If and / or module are passed (not allowed with - ', maxsplit=1)] - if len(spec_list) > 1: - new_kind = spec_list[1] - new_ccpp_kind = spec_list[0] - else: - new_kind = new_ccpp_kind - # end if - try: - check_fortran_id(new_kind, {}, True) - except CCPPError as err: - self.__pobj.add_syntax_err(f"{err}") - new_kind = None - # end try - if new_kind and (new_ccpp_kind != new_kind): - try: - check_fortran_id(new_ccpp_kind, {}, True) - except CCPPError as err: - self.__pobj.add_syntax_err(f"{err}") - new_ccpp_kind = None - # end try - # end if - if new_kind: - emsg = run_env.add_kind_type(new_ccpp_kind, - new_kind, fort_module) - if emsg: - self.__pobj.add_syntax_err(emsg) - # end if - # end if - else: - tok_type = "metadata table start property" - self.__pobj.add_syntax_err(tok_type, token=key) - # end if - # end for - curr_line, _ = self.__pobj.next_line() - else: - # Process a metadata section - if MetadataSection.header_start(curr_line): - skip_rest_of_section = False - section = MetadataSection(self.table_name, self.table_type, - run_env, parse_object=self.__pobj, - module=self.__module_name, - known_ddts=known_ddts, - skip_ddt_check=skip_ddt_check) - # Some table types only allow for one associated section - if ((len(self.__sections) == 1) and - (self.table_type in _SINGLETON_TABLE_TYPES)): - prev_title = self.__sections[0].title - emsg = "{}, '{}', table already contains '{}'" - self.__pobj.add_syntax_err(emsg.format(self.table_type, - section.title, - prev_title)) - # end if - self.__sections.append(section) - # Note: Do not read next line, we are already on it. - curr_line, _ = self.__pobj.curr_line() - elif not blank_metadata_line(curr_line): - if not skip_rest_of_section: - self.__pobj.add_syntax_err("metadata file line", - token=curr_line) - skip_rest_of_section = True - # end if - curr_line, _ = self.__pobj.next_line() - else: - curr_line, _ = self.__pobj.next_line() - # end if - # end if - # end while - if self.__pobj.error_message: - # Time to dump out error messages - raise CCPPError(self.__pobj.error_message) - # end if - if self.table_type == "ddt": - known_ddts.append(self.table_name.lower()) - # end if - if self.__dependencies is None: - self.__dependencies = [] - # end if - - def start_context(self, with_comma=True, nodir=True): - """Return a context string for the beginning of the table""" - return context_string(self.__start_context, - with_comma=with_comma, nodir=nodir) - - def sections(self): - """Return the metadata header sections for this table""" - if self.__sections: - # Return a copy so it cannot be modified - return list(self.__sections) - return self.__sections - - @property - def table_name(self): - 'Return the name of the metadata table' - return self.__table_name - - @property - def table_type(self): - 'Return the type of structure this header documents' - return self.__table_type - - @property - def dependencies(self): - """Return the dependencies for this table""" - return self.__dependencies - - @property - def module_name(self): - """Return the module name for this metadata table""" - return self.__module_name - - @property - def dependencies_path(self): - """Return the relative path for the table's dependencies""" - return self.__dependencies_path - - @property - def fortran_source_path(self): - """Return the Fortran source path for this table""" - return self.__fortran_src_path - - @property - def run_env(self): - """Return this table's CCPPFrameworkEnv object""" - return self.__run_env - - def __repr__(self): - '''Print representation for MetadataTable objects''' - return "<{} {} @ 0X{:X}>".format(self.__class__.__name__, - self.table_name, id(self)) - - def __str__(self): - '''Print string for MetadataTable objects''' - return "<{} {}>".format(self.__class__.__name__, self.table_name) - - @classmethod - def table_start(cls, line): - """Return True iff is a ccpp-table-properties header statement. - """ - if (line is None) or blank_metadata_line(line): - match = None - else: - match = cls.__table_start.match(line) - # end if - return match is not None - -######################################################################## - -class MetadataSection(ParseSource): - """Class to hold all information from a metadata header - >>> from framework_env import CCPPFrameworkEnv - >>> _DUMMY_RUN_ENV = CCPPFrameworkEnv(None, {'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}) - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type = scheme", \ - "[ im ]", "standard_name = horizontal_loop_extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in"])) #doctest: +ELLIPSIS - - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foobar", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type = scheme", \ - "[ im ]", "standard_name = horizontal_loop_extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in"])).find_variable('horizontal_loop_extent') #doctest: +ELLIPSIS - - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foobar", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type = scheme", \ - "process = microphysics", "[ im ]", \ - "standard_name = horizontal_loop_extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in"])).find_variable('horizontal_loop_extent') #doctest: +ELLIPSIS - - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type=scheme", \ - "[ im ]", "standard_name = horizontal_loop_extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - " subroutine foo()"])).find_variable('horizontal_loop_extent') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - parse_source.ParseSyntaxError: Invalid variable property syntax, 'subroutine foo()', at foobar.txt:9 - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foobar", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type = scheme", \ - "[ im ]", "standard_name = horizontal_loop_extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - ""], line_start=0)).find_variable('horizontal_loop_extent').get_prop_value('local_name') - 'im' - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["name = footable", "type = scheme" \ - "[ im ]", "standard_name = horizontalloop extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - ""], line_start=0)).find_variable('horizontal_loop_extent') - - >>> MetadataSection("footable", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["[ccpp-arg-table]", "name = foobar", "type = scheme" \ - "[ im ]", "standard_name = horizontal loop extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - ""], line_start=0)).find_variable('horizontal_loop_extent') - - >>> MetadataSection("foobar", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["name = foobar" \ - "[ im ]", "standard_name = horizontal loop extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - ""], line_start=0)).find_variable('horizontal_loop_extent') - - >>> MetadataSection("foobar", "scheme", _DUMMY_RUN_ENV, module="foo", \ - parse_object=ParseObject("foobar.txt", \ - ["name = foobar", "foo = bar" \ - "[ im ]", "standard_name = horizontal loop extent", \ - "long_name = horizontal loop extent, start at 1", \ - "units = index | type = integer", \ - "dimensions = () | intent = in", \ - ""], line_start=0)).find_variable('horizontal_loop_extent') - - >>> MetadataSection.header_start('[ ccpp-arg-table ]') - True - >>> MetadataSection.header_start('[ qval ]') - False - >>> MetadataSection.header_start(' local_name = foo') - False - >>> MetadataSection.variable_start('[ qval ]', ParseObject('foo.meta', [])) - 'qval' - >>> MetadataSection.variable_start('[ qval(hi_mom) ]', ParseObject('foo.meta', [])) - 'qval(hi_mom)' - >>> MetadataSection.variable_start(' local_name = foo', ParseContext(filename='foo.meta', linenum=1)) - -""" - - __header_start = re.compile(r"(?i)\s*\[\s*ccpp-arg-table\s*\]") - - __var_start = re.compile(r"^\[\s*"+FORTRAN_ID+r"\s*\]$") - - __vref_start = re.compile(r"^\[\s*"+FORTRAN_SCALAR_REF+r"\s*\]$") - - __html_template__ = """ - - -{title} - - - -
-{header}{contents}
- - -""" - - def __init__(self, table_name, table_type, run_env, parse_object=None, - title=None, type_in=None, module=None, process_type=None, - var_dict=None, known_ddts=None, skip_ddt_check=False): - """Initialize a new MetadataSection object. - If is not None, initialize from the current file and - location in . - If is None, initialize from , <type>, <module>, - and <var_dict>. Note that if <parse_object> is not None, <title>, - <type>, <module>, and <var_dict> are ignored. - <table_name> and <table_type> are the name and type of the - metadata header of which this section is a part. They must match - the type and name of this section (once the name action has been - removed, e.g., name = foo_init matches type foo). - """ - self.__pobj = parse_object - self.__variables = None # In case __init__ crashes - self.__section_title = None - self.__header_type = None - self.__module_name = module - self.__process_type = UNKNOWN_PROCESS_TYPE - self.__section_valid = True - self.__run_env = run_env - if parse_object is None: - if title is not None: - self.__section_title = title - else: - raise ParseInternalError('MetadataSection requires a title') - # end if - if type_in is None: - perr = 'MetadataSection requires a header type' - raise ParseInternalError(perr) - # end if - if type_in in HEADER_TYPES: - self.__header_type = type_in - else: - self.__pobj.add_syntax_err("metadata arg table type", - token=type_in) - self.__section_valid = False - # end if - mismatch = self.section_table_mismatch(table_name, table_type) - if mismatch: - self.__pobj.add_syntax_err(mismatch) - self.__section_valid = False - # end if - mismatch = self.section_table_mismatch(table_name, table_type) - if mismatch: - raise CCPPError(mismatch) - # end if - if module is None: - perr = "MetadataSection requires a module name" - self.__pobj.add_syntax_err(perr) - self.__section_valid = False - # end if - if process_type is None: - self.__process_type = UNKNOWN_PROCESS_TYPE - else: - self.__process_type = process_type - # end if - # Initialize our ParseSource parent - super().__init__(self.title, self.header_type, self.__pobj) - self.__variables = VarDictionary(self.title, run_env) - for var in var_dict.variable_list(): # Let this crash if no dict - self.__variables.add_variable(var, run_env) - # end for - self.__start_context = None - else: - if known_ddts is None: - known_ddts = [] - # end if - self.__start_context = ParseContext(context=self.__pobj) - self.__init_from_file(table_name, table_type, known_ddts, self.module, - run_env, skip_ddt_check=skip_ddt_check) - # end if - # Register this header if it is a DDT - if self.header_type == 'ddt': - register_fortran_ddt_name(self.title) - # end if - # Categorize the variables - self._var_intents = {'in' : [], 'out' : [], 'inout' : []} - for var in self.variable_list(): - intent = var.get_prop_value('intent') - if intent is not None: - self._var_intents[intent].append(var) - # end if - # end for - - def __init_from_file(self, table_name, table_type, known_ddts, module_name, - run_env, skip_ddt_check=False): - """ Read the section preamble, assume the caller already figured out - the first line of the header using the header_start method.""" - start_ctx = context_string(self.__pobj) - curr_line, _ = self.__pobj.next_line() # Skip past [ccpp-arg-table] - self.__module_name = module_name - while ((curr_line is not None) and - (not MetadataSection.variable_start(curr_line, self.__pobj)) and - (not MetadataSection.header_start(curr_line)) and - (not MetadataTable.table_start(curr_line))): - for prop in _parse_config_line(curr_line, self.__pobj): - # Manually parse name, type, and module properties - key = prop[0].lower() - value = prop[1] - if key == 'name': - self.__section_title = value - elif key == 'type': - if value not in HEADER_TYPES: - self.__pobj.add_syntax_err("metadata table type", - token=value) - self.__section_valid = False - close = difflib.get_close_matches(value, HEADER_TYPES) - if close: - self.__header_type = close[0] # Allow error continue - # end if - # end if - # Set value even if error so future error msgs make sense - self.__header_type = value - elif key == 'process': - self.__process_type = value - else: - self.__pobj.add_syntax_err("metadata table start property", - token=key) - self.__process_type = 'INVALID' # Allow error continue - self.__section_valid = False - # end if - # end for - curr_line, _ = self.__pobj.next_line() - # end while - if self.title is None: - self.__pobj.add_syntax_err("metadata header start, no table name", - token=curr_line) - self.__section_valid = False - # end if - if self.header_type is None: - self.__pobj.add_syntax_err("metadata header start, no table type", - token=curr_line) - self.__section_valid = False - # end if - if ((self.header_type != SCHEME_HEADER_TYPE) and - (self.process_type != UNKNOWN_PROCESS_TYPE)): - emsg = "process keyword only allowed for a scheme" - self.__pobj.add_syntax_err(emsg, token=curr_line) - self.__process_type = UNKNOWN_PROCESS_TYPE # Allow error continue - self.__section_valid = False - # end if - mismatch = self.section_table_mismatch(table_name, table_type) - if mismatch: - self.__pobj.add_syntax_err(mismatch) - self.__section_valid = False - # end if - if run_env.verbose: - run_env.logger.info("Parsing {} {}{}".format(self.header_type, - self.title, start_ctx)) - # end if - if self.header_type == "ddt": - known_ddts.append(self.title.lower()) - # end if - # Initialize our ParseSource parent - super().__init__(self.title, self.header_type, self.__pobj) - # Read the variables - valid_lines = True - self.__variables = VarDictionary(self.title, run_env) - while valid_lines: - newvar, curr_line = self.parse_variable(curr_line, known_ddts, - skip_ddt_check=skip_ddt_check) - valid_lines = newvar is not None - if valid_lines: - if run_env.verbose: - dmsg = 'Adding {} to {}' - lname = newvar.get_prop_value('local_name') - run_env.logger.debug(dmsg.format(lname, self.title)) - # end if - self.__variables.add_variable(newvar, run_env) - # Check to see if we hit the end of the table - valid_lines = not MetadataSection.header_start(curr_line) - else: - # We have a bad variable, see if we have more variables - lname = MetadataSection.variable_start(curr_line, self.__pobj) - valid_lines = lname is not None - # end while - # end if - # end while - - def parse_variable(self, curr_line, known_ddts, skip_ddt_check=False): - """Parse a new metadata variable beginning on <curr_line>. - The header line has the format [ <valid_fortran_symbol> ]. - """ - newvar = None - var_ok = True # Set to False if an error is detected - valid_line = ((curr_line is not None) and - (not MetadataSection.header_start(curr_line)) and - (not MetadataTable.table_start(curr_line))) - if valid_line: - # variable_start handles exception - local_name = MetadataSection.variable_start(curr_line, self.__pobj).lower() - else: - local_name = None - # end if - if local_name is None: - # This is not a valid variable line, punt (should be end of table) - valid_line = False - # end if - # Parse lines until invalid line is found - # NB: Header variables cannot have embedded blank lines - if valid_line: - var_props = {} - var_props['local_name'] = local_name - # Grab context that points at beginning of definition - context = ParseContext(context=self.__pobj) - else: - var_props = None - # end if - while valid_line: - curr_line, _ = self.__pobj.next_line() - valid_line = ((curr_line is not None) and - (not MetadataSection.header_start(curr_line)) and - (not MetadataTable.table_start(curr_line)) and - (MetadataSection.variable_start(curr_line, - self.__pobj) is None)) - # A valid line may have multiple properties (separated by '|') - if valid_line: - properties = _parse_config_line(curr_line, self.__pobj) - for prop in properties: - pname = prop[0].lower() - pval_str = prop[1] - if ((pname == 'type') and - (not check_fortran_intrinsic(pval_str, error=False))): - if skip_ddt_check or pval_str.lower() in known_ddts: - if skip_ddt_check: - register_fortran_ddt_name(pval_str) - # end if - pval = pval_str.lower() - pname = 'ddt_type' - else: - errmsg = "Unknown DDT type, {}".format(pval_str) - self.__pobj.add_syntax_err(errmsg) - self.__section_valid = False - var_ok = False - # end if - else: - # Make sure this is a match - check_prop = Var.get_prop(pname) - if check_prop is not None: - pval = check_prop.valid_value(pval_str) - else: - emsg = "variable property name" - self.__pobj.add_syntax_err(emsg, token=pname) - self.__section_valid = False - var_ok = False - # end if - if pval is None: - errmsg = "'{}' property value" - self.__pobj.add_syntax_err(errmsg.format(pname), - token=pval_str) - self.__section_valid = False - var_ok = False - # end if - # end if - if var_ok: - # If we get this far, we have a valid property. - # Special case for dimensions, turn them into ranges - if pname == 'dimensions': - porig = pval - pval = [] - for dim in porig: - if ':' in dim: - for dim2 in dim.split(':'): - dim_ok = VarDictionary.loop_var_okay(standard_name=dim2, - is_run_phase=self.__section_title.endswith("_run")) - if not dim_ok: - emsg = "horizontal dimension" - self.__pobj.add_syntax_err(emsg, token=dim2) - self.__section_valid = False - var_ok = False - # end if - # end for - pval.append(dim.lower()) - else: - dim_ok = VarDictionary.loop_var_okay(standard_name=dim, - is_run_phase=self.__section_title.endswith("_run")) - if not dim_ok: - emsg = "horizontal dimension" - self.__pobj.add_syntax_err(emsg, token=dim) - self.__section_valid = False - var_ok = False - # end if - cone_str = 'ccpp_constant_one:{}' - pval.append(cone_str.format(dim.lower())) - # end if - # end for - # end if - # Special handling for standard_names (convert to lowercase) - if pname == 'standard_name': - pval = pval.lower() - # end if - # Add the property to our Var dictionary - var_props[pname] = pval - # end if - # end for - # end if - # end while - if var_ok and (var_props is not None): - # Check for array reference - sub_name = MetadataSection.check_array_reference(local_name, - var_props, context) - if sub_name: - var_props['local_name'] = sub_name - # end if (else just leave the local name alone) - try: - newvar = Var(var_props, self, self.run_env, context=context) - except CCPPError as verr: - self.__pobj.add_syntax_err(verr, skip_context=True) - var_ok = False - self.__section_valid = False - # end try - # No else, will return None for newvar - # end if - return newvar, curr_line - - @staticmethod - def check_array_reference(local_name, var_dict, context): - """If <local_name> is an array reference, check it against - the 'dimensions' property in <var_dict>. If <local_name> is an - array reference, return it with the colons filled in with the - dictionary dimensions, otherwise, return None. - >>> MetadataSection.check_array_reference('foo', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) - - >>> MetadataSection.check_array_reference('foo', {}, ParseContext(filename='foo.meta')) - - >>> MetadataSection.check_array_reference('foo(qux', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: Invalid scalar reference, foo(qux, in foo.meta - >>> MetadataSection.check_array_reference('foo(qux)', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: foo has rank 2 but foo(qux) has 0, in foo.meta - >>> MetadataSection.check_array_reference('foo(:,qux)', {'dimensions':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: foo has rank 2 but foo(:,qux) has 1, in foo.meta - >>> MetadataSection.check_array_reference('foo(:,qux)', {'foo':['ccpp_constant_one:bar', 'ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: Missing variable dimensions, foo(:,qux), in foo.meta - >>> MetadataSection.check_array_reference('foo(:,:,qux)', {'dimensions':['ccpp_constant_one:bar']}, ParseContext(filename='foo.meta')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: foo has rank 1 but foo(:,:,qux) has 2, in foo.meta - >>> MetadataSection.check_array_reference('foo(:,:,qux)', {'dimensions':['ccpp_constant_one:bar','ccpp_constant_one:baz']}, ParseContext(filename='foo.meta')) - 'foo(:, :, qux)' - """ - retval = None - if check_fortran_id(local_name, var_dict, False) is None: - rmatch = FORTRAN_SCALAR_REF_RE.match(local_name) - if rmatch is None: - errmsg = 'Invalid scalar reference, {}{}' - ctx = context_string(context) - raise ParseInternalError(errmsg.format(local_name, ctx)) - # end if - rname = rmatch.group(1) - rdims = [x.strip() for x in rmatch.group(2).split(',')] - if 'dimensions' in var_dict: - vdims = [x.strip() for x in var_dict['dimensions']] - else: - errmsg = 'Missing variable dimensions, {}{}' - ctx = context_string(context) - raise ParseInternalError(errmsg.format(local_name, ctx)) - # end if - colon_rank = len([x for x in rdims if x == ':']) - if colon_rank != len(vdims): - errmsg = '{} has rank {} but {} has {}{}' - ctx = context_string(context) - raise ParseInternalError(errmsg.format(rname, len(vdims), - local_name, colon_rank, - ctx)) - # end if - sub_dims = [] - sindex = 0 - for rind in rdims: - if rind == ':': - sub_dims.append(':') - sindex += 1 - else: - sub_dims.append(rind) - # end if - # end for - retval = '{}({})'.format(rname, ', '.join(sub_dims)) - # end if - return retval - - def variable_list(self, std_vars=True, loop_vars=True, consts=True): - """Return an ordered list of the header's variables""" - return self.__variables.variable_list(recursive=False, - std_vars=std_vars, - loop_vars=loop_vars, - consts=consts) - - def find_variable(self, std_name, use_local_name=False): - """Find a variable in this header's dictionary""" - var = None - if use_local_name: - var = self.__variables.find_local_name(std_name) - else: - var = self.__variables.find_variable(std_name, any_scope=False) - # end if - return var - - def convert_dims_to_standard_names(self, var, logger=None, context=None): - """Convert the dimension elements in <var> to standard names by - by using other variables in this header. - """ - std_dims = [] - vdims = var.get_dimensions() - # Check for bad dimensions - if vdims is None: - vdim_prop = var.get_prop_value('dimensions').strip() - if vdim_prop[0] == '(': - vdim_prop = vdim_prop[1:] - # end if - if vdim_prop[-1] == ')': - vdim_prop = vdim_prop[0:-1] - # end if - vdim_strs = [x.strip() for x in vdim_prop.split(',')] - lname = var.get_prop_value('local_name') - ctx = context_string(var.context) - sep = '' - errstr = "{}{}: Invalid dimension, '{}'{}" - errmsg = '' - for vdim in vdim_strs: - if not check_fortran_id(vdim, None, False): - errmsg += errstr.format(sep, lname, vdim, ctx) - sep = '\n' - # end if - # end for - raise CCPPError("{}".format(errmsg)) - # end if - for dim in vdims: - std_dim = [] - if ':' not in dim: - # Metadata dimensions always have an explicit start - var_one = CCPP_CONSTANT_VARS.find_local_name('1') - if var_one is not None: - std = var_one.get_prop_value('standard_name') - std_dim.append(std) - # end if - # end if - for item in dim.split(':'): - try: - _ = int(item) - dvar = CCPP_CONSTANT_VARS.find_local_name(item) - if dvar is not None: - # If this integer value is a CCPP standard int, use that - dname = dvar.get_prop_value('standard_name') - else: - # Some non-standard integer value - dname = item - # end if - except ValueError as verr: - # Not an integer, try to find the standard_name - if not item: - # Naked colons are okay - dname = '' - else: - dvar = self.find_variable(item, use_local_name=True) - if dvar is not None: - dname = dvar.get_prop_value('standard_name') - else: - dname = None - # end if - # end if - if dname is None: - std = var.get_prop_value('local_name') - errmsg = f"Unknown dimension element, {item}, in {std}" - errmsg += context_string(context) - if logger is not None: - errmsg = "ERROR: " + errmsg - logger.error(errmsg.format(item, std, ctx)) - dname = unique_standard_name() - else: - raise CCPPError(errmsg) from verr - # end if - # end if - # end try - if dname is not None: - std_dim.append(dname) - else: - std_dim = None - break - # end if - # end for - if std_dim is not None: - std_dims.append(':'.join(std_dim)) - else: - break - # end if - # end for - - return std_dims - - def prop_list(self, prop_name): - """Return list of <prop_name> values for this scheme's arguments""" - return self.__variables.prop_list(prop_name) - - def section_table_mismatch(self, table_title, table_type): - """Return an error string if this arg table does not match its - metadata table parent. If they match , return an empty string.""" - mismatch = "" - # The header type must match its table's type - if self.header_type is None: - mstr = "Invalid section type, 'None'" - mismatch += mstr.format(self.header_type, table_type) - elif table_type != self.header_type: - mstr = "Section type, '{}', does not match table type, '{}'" - mismatch += mstr.format(self.header_type, table_type) - # end if - if self.header_type == SCHEME_HEADER_TYPE: - # For schemes, strip off the scheme function phase (e.g., _init) - sect_func, _, _ = CCPP_STATE_MACH.function_match(self.title) - else: - sect_func = self.title - # end if - # The Fortran parser cannot tell a scheme from a host subroutine - # Detect this and adjust - if sect_func is None: - sect_func = self.title - # end if - # The header name (minus phase) must match its table's name - if table_title != sect_func: - if mismatch: - mismatch += '\n' - # end if - mstr = "Section name, '{}', does not match table title, '{}'" - mismatch += mstr.format(self.title, table_title) - # end if - if mismatch: - mismatch += context_string(self.__pobj) - # end if - return mismatch - - @staticmethod - def variable_start(line, pobj): - """Return variable name if <line> is an interface metadata table header - """ - if line is None: - match = None - else: - match = MetadataSection.__var_start.match(line) - if match is None: - match = MetadataSection.__vref_start.match(line) - if match is not None: - name = match.group(1)+'('+match.group(2)+')' - # end if - else: - name = match.group(1) - # end if - # end if - if match is not None: - if not MetadataSection.is_scalar_reference(name): - pobj.add_syntax_err("local variable name", token=name) - name = None - # end if - else: - name = None - # end if - return name - - def write_to_file(self, filename, append=False): - """Write this metadata table to <filename>. If <append> is True, - append this table to the end of <filename>, otherwise, create - or truncate the file.""" - if append: - oflag = 'a' - else: - oflag = 'w' - # end if - with open(filename, oflag) as mfile: - mfile.write("[ccpp-arg-table]") - mfile.write(" name = {}".format(self.title)) - mfile.write(" type = {}".format(self.header_type)) - for var in self.variable_list(): - var.write_metadata(mfile) - # end for - # end with - - def to_html(self, outdir, props): - """Write html file for metadata section and return filename. - Skip metadata sections without variables""" - if not self.__variables.variable_list(): - return None - # Write table header - header = f"<tr>" - for prop in props: - header += f"<th>{prop}</th>".format(prop=prop) - header += f"</tr>\n" - # Write table contents, one row per variable - contents = "" - for var in self.__variables.variable_list(): - row = f"<tr>" - for prop in props: - value = var.get_prop_value(prop) - # Pretty-print for dimensions - if prop == 'dimensions': - value = '(' + ', '.join(value) + ')' - elif value is None: - value = f"n/a" - row += f"<td>{value}</td>".format(value=value) - row += f"</tr>\n" - contents += row - filename = os.path.join(outdir, self.title + '.html') - with open(filename,"w") as f: - f.writelines(self.__html_template__.format(title=self.title + ' argument table', - header=header, contents=contents)) - return filename - - def __repr__(self): - base = super().__repr__() - pind = base.find(' object ') - if pind >= 0: - pre = base[0:pind] - else: - pre = '<MetadataSection' - # end if - bind = base.find('at 0x') - if bind >= 0: - post = base[bind:] - else: - post = '>' - # end if - return '{} {} / {} {}'.format(pre, self.module, self.title, post) - - def __del__(self): - try: - del self.__variables - except AttributeError: - pass - - def start_context(self, with_comma=True, nodir=True): - """Return a context string for the beginning of the table""" - return context_string(self.__start_context, - with_comma=with_comma, nodir=nodir) - - @property - def title(self): - """Return the name of the metadata arg_table""" - return self.__section_title - - @property - def module(self): - """Return the module name for this header (if it exists)""" - return self.__module_name - - @property - def header_type(self): - """Return the type of structure this header documents""" - return self.__header_type - - @property - def process_type(self): - """Return the type of physical process this header documents""" - return self.__process_type - - @property - def has_variables(self): - """Convenience function for finding empty headers""" - return self.__variables - - @property - def run_env(self): - """Return this section's CCPPFrameworkEnv object""" - return self.__run_env - - @property - def valid(self): - """Return True iff we did not encounter an error creating - this section""" - return self.__section_valid - - def __str__(self): - '''Print string for MetadataSection objects''' - return "<{} {}>".format(self.__class__.__name__, self.title) - - @classmethod - def header_start(cls, line): - """Return True iff <line> is a Metadata section header (ccpp-arg-table). - """ - if (line is None) or blank_metadata_line(line): - match = None - else: - match = cls.__header_start.match(line) - # end if - return match is not None - - @staticmethod - def is_scalar_reference(test_val): - """Return True iff <test_val> refers to a Fortran scalar.""" - return check_fortran_ref(test_val, None, False) is not None - -######################################################################## diff --git a/scripts/metavar.py b/scripts/metavar.py deleted file mode 100755 index cafdbf9f..00000000 --- a/scripts/metavar.py +++ /dev/null @@ -1,2139 +0,0 @@ -#!/usr/bin/env python3 - -""" -Classes and supporting code to hold all information on CCPP metadata variables -Var: Class which holds all information on a single CCPP metadata variable -VarSpec: Class to hold a standard_name description which can include dimensions -VarAction: Base class for describing actions on variables -VarLoopSubst: Class for describing a loop substitution -VarDictionary: Class to hold all CCPP variables of a CCPP unit (e.g., suite, - scheme, host) -""" - -# Python library imports -import re -from collections import OrderedDict -# CCPP framework imports -from framework_env import CCPPFrameworkEnv -from parse_tools import check_local_name, check_fortran_type, context_string -from parse_tools import FORTRAN_SCALAR_REF_RE -from parse_tools import check_units, check_dimensions, check_cf_standard_name -from parse_tools import check_diagnostic_id, check_diagnostic_fixed -from parse_tools import check_default_value, check_valid_values -from parse_tools import check_molar_mass -from parse_tools import ParseContext, ParseSource, type_name -from parse_tools import ParseInternalError, ParseSyntaxError, CCPPError -from parse_tools import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX -from var_props import CCPP_LOOP_DIM_SUBSTS, VariableProperty, VarCompatObj -from var_props import find_horizontal_dimension, find_vertical_dimension -from var_props import standard_name_to_long_name, local_name_to_diag_name, default_kind_val - -############################################################################## - -# Dictionary of standard CCPP variables -CCPP_STANDARD_VARS = { - # Variable representing the constant integer, 1 - 'ccpp_constant_one' : - {'local_name' : '1', 'protected' : 'True', - 'standard_name' : 'ccpp_constant_one', - 'long_name' : "CCPP constant one", - 'units' : '1', 'dimensions' : '()', 'type' : 'integer'}, - 'ccpp_error_code' : - {'local_name' : 'errflg', 'standard_name' : 'ccpp_error_code', - 'long_name' : "CCPP error flag", - 'units' : '1', 'dimensions' : '()', 'type' : 'integer'}, - 'ccpp_error_message' : - {'local_name' : 'errmsg', 'standard_name' : 'ccpp_error_message', - 'long_name' : "CCPP error message", - 'units' : 'none', 'dimensions' : '()', 'type' : 'character', - 'kind' : 'len=512'}, - 'horizontal_dimension' : - {'local_name' : 'total_columns', - 'standard_name' : 'horizontal_dimension', 'units' : 'count', - 'long_name' : "total number of columns", - 'dimensions' : '()', 'type' : 'integer'}, - 'horizontal_loop_extent' : - {'local_name' : 'horz_loop_ext', - 'standard_name' : 'horizontal_loop_extent', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'}, - 'horizontal_loop_begin' : - {'local_name' : 'horz_col_beg', - 'standard_name' : 'horizontal_loop_begin', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'}, - 'horizontal_loop_end' : - {'local_name' : 'horz_col_end', - 'standard_name' : 'horizontal_loop_end', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'}, - 'vertical_layer_dimension' : - {'local_name' : 'num_model_layers', - 'standard_name' : 'vertical_layer_dimension', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'}, - 'vertical_interface_dimension' : - {'local_name' : 'num_model_interfaces', - 'standard_name' : 'vertical_interface_dimension', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'}, - 'vertical_interface_index' : - {'local_name' : 'layer_index', - 'standard_name' : 'vertical_interface_index', 'units' : 'count', - 'dimensions' : '()', 'type' : 'integer'} -} - -# Pythonic version of a forward reference (CCPP_CONSTANT_VARS defined below) -CCPP_CONSTANT_VARS = {} -# Pythonic version of a forward reference (CCPP_VAR_LOOP_SUBST defined below) -CCPP_VAR_LOOP_SUBSTS = {} -# Loop variables only allowed during run phases -CCPP_LOOP_VAR_STDNAMES = ['horizontal_loop_extent', - 'horizontal_loop_begin', 'horizontal_loop_end', - 'vertical_layer_index', 'vertical_interface_index'] - -############################################################################### -# Used for creating template variables -_MVAR_DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -############################################################################## - -class Var: - """ A class to hold a metadata or code variable. - Var objects should be treated as immutable. - >>> Var.get_prop('standard_name') #doctest: +ELLIPSIS - <var_props.VariableProperty object at 0x...> - >>> Var.get_prop('standard') - - >>> Var.get_prop('type').is_match('type') - True - >>> Var.get_prop('type').is_match('long_name') - False - >>> Var.get_prop('type').valid_value('character') - 'character' - >>> Var.get_prop('type').valid_value('char') - - >>> Var.get_prop('long_name').valid_value('hi mom') - 'hi mom' - >>> Var.get_prop('dimensions').valid_value('hi mom') - - >>> Var.get_prop('dimensions').valid_value(['Bob', 'Ray']) - ['Bob', 'Ray'] - >>> Var.get_prop('active') #doctest: +ELLIPSIS - <var_props.VariableProperty object at 0x...> - >>> Var.get_prop('active').get_default_val({}) - '.true.' - >>> Var.get_prop('active').valid_value('flag_for_aerosol_physics') - 'flag_for_aerosol_physics' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'DDT', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('active') - '.true.' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in', 'active' : 'child_is_home==.true.'}, ParseSource('vname', 'DDT', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('active') - 'child_is_home==.true.' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('long_name') - 'Hi mom' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('intent') - 'in' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('units') - 'm s-1' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('units') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Required property, 'units', missing, in <standard input> - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : ' ', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV).get_prop_value('units') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: foo: ' ' is not a valid unit, in <standard input> - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'ttype' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid metadata variable property, 'ttype', in <standard input> - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Required property, 'units', missing, in <standard input> - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'inout', 'protected' : '.true.'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: foo is marked protected but is intent inout, at <standard input>:1 - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'ino'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid intent variable property, 'ino', at <standard input>:1 - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in', 'optional' : 'false'}, ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV) #doctest: +ELLIPSIS - <metavar.Var hi_mom: foo at 0x...> - - # Check that two variables that differ in their units - m vs km - are compatible - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm', \ - 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, \ - ParseSource('vname', 'SCHEME', ParseContext()), \ - _MVAR_DUMMY_RUN_ENV).compatible(Var({'local_name' : 'bar', \ - 'standard_name' : 'hi_mom', 'units' : 'km', \ - 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, \ - ParseSource('vname', 'SCHEME', ParseContext()), _MVAR_DUMMY_RUN_ENV), \ - _MVAR_DUMMY_RUN_ENV) #doctest: +ELLIPSIS - <var_props.VarCompatObj object at ...> - """ - - ## Prop lists below define all the allowed CCPP Metadata attributes - - # __spec_props are for variables defined in a specification - __spec_props = [VariableProperty('local_name', str, - check_fn_in=check_local_name), - VariableProperty('standard_name', str, - check_fn_in=check_cf_standard_name), - VariableProperty('long_name', str, optional_in=True, - default_fn_in=standard_name_to_long_name), - VariableProperty('diagnostic_name', str, optional_in=True, - default_fn_in=local_name_to_diag_name, - check_fn_in=check_diagnostic_id), - VariableProperty('units', str, - check_fn_in=check_units), - VariableProperty('dimensions', list, - check_fn_in=check_dimensions), - VariableProperty('type', str, - check_fn_in=check_fortran_type), - VariableProperty('kind', str, - optional_in=True, - default_fn_in=default_kind_val), - VariableProperty('state_variable', bool, - optional_in=True, default_in=False), - VariableProperty('protected', bool, - optional_in=True, default_in=False), - VariableProperty('allocatable', bool, - optional_in=True, default_in=False), - VariableProperty('diagnostic_name_fixed', str, - optional_in=True, default_in='', - check_fn_in=check_diagnostic_fixed), - VariableProperty('default_value', str, - optional_in=True, default_in='', - check_fn_in=check_default_value), - VariableProperty('persistence', str, optional_in=True, - valid_values_in=['timestep', 'run'], - default_in='timestep'), - VariableProperty('active', str, optional_in=True, - default_in='.true.'), - VariableProperty('polymorphic', bool, optional_in=True, - default_in=False), - VariableProperty('top_at_one', bool, optional_in=True, - default_in=False), - VariableProperty('optional', bool, optional_in=True, - default_in=False), - VariableProperty('target', bool, optional_in=True, - default_in=False)] - -# XXgoldyXX: v debug only - __to_add = VariableProperty('valid_values', str, - optional_in=True, default_in='', - check_fn_in=check_valid_values) -# XXgoldyXX: ^ debug only - - # __var_props contains properties which are not in __spec_props - __var_props = [VariableProperty('intent', str, - valid_values_in=['in', 'out', 'inout'])] - - # __constituent_props contains properties associated only with constituents - # Note that all constituent properties must be optional and contain either - # a default value or default function. - __constituent_props = [VariableProperty('advected', bool, - optional_in=True, default_in=False), - VariableProperty('molar_mass', float, - optional_in=True, default_in=0.0, - check_fn_in=check_molar_mass), - VariableProperty('constituent', bool, - optional_in=True, default_in=False)] - - __constituent_prop_dict = {x.name : x for x in __constituent_props} - - # __no_metadata_props__ contains properties to omit from metadata - __no_metadata_props__ = ['local_name'] - - __spec_propdict = {p.name : p for p in __spec_props} - __var_propdict = {p.name : p for p in __spec_props + __var_props} - __required_spec_props = list() - __required_var_props = list() - for p in __spec_props: - __var_propdict[p.name] = p - if not p.optional: - __required_spec_props.append(p.name) - __required_var_props.append(p.name) - # end if - # end for - for p in __var_props: -# XXgoldyXX: v why? -# __spec_propdict[p.name] = p -# XXgoldyXX: ^ why? -# __var_propdict[p.name] = p - if not p.optional: - __required_var_props.append(p.name) - # end if - # end for - __var_propdict.update({p.name : p for p in __constituent_props}) - # All constituent props are optional so no check - - def __init__(self, prop_dict, source, run_env, context=None, - clone_source=None, fortran_imports=None): - """Initialize a new Var object. - If <prop_dict> is really a Var object, use that object's prop_dict. - If this Var object is a clone, record the original Var object - for reference - <source> is a ParseSource object describing the source of this Var. - <run_env> is the CCPPFrameworkEnv object for this framework run. - <context> is a ParseContext object - <clone_source> is a Var object. If provided, it is used as the original - source of a cloned variable. - """ - self.__parent_var = None # for array references - self.__children = list() # This Var's array references - self.__clone_source = clone_source - self.__run_env = run_env - if isinstance(prop_dict, Var): - prop_dict = prop_dict.copy_prop_dict() - # end if - if source.ptype == 'scheme': - self.__required_props = Var.__required_var_props -# XXgoldyXX: v don't fill in default properties? -# mstr_propdict = Var.__var_propdict -# XXgoldyXX: ^ don't fill in default properties? - else: - self.__required_props = Var.__required_spec_props -# XXgoldyXX: v don't fill in default properties? - mstr_propdict = Var.__spec_propdict -# XXgoldyXX: ^ don't fill in default properties? - # end if - self.__source = source - # Grab a frozen copy of the context - if context is None: - self._context = ParseContext(context=source.context) - else: - self._context = context - # end if - # First, check the input - if 'ddt_type' in prop_dict: - # Special case to bypass normal type rules - if 'type' not in prop_dict: - prop_dict['type'] = prop_dict['ddt_type'] - # end if - if 'units' not in prop_dict: - prop_dict['units'] = "" - # end if - # DH* To investigate later: Why is the DDT type - # copied into the kind attribute? Can we remove this? - prop_dict['kind'] = prop_dict['ddt_type'] - del prop_dict['ddt_type'] - self.__intrinsic = False - else: - self.__intrinsic = True - # end if - for key in prop_dict: - if Var.get_prop(key) is None: - raise ParseSyntaxError("Invalid metadata variable property, '{}'".format(key), context=self.context) - # end if - # end for - # Make sure required properties are present - for propname in self.__required_props: - if propname not in prop_dict: - emsg = "Required property, '{}', missing" - raise ParseSyntaxError(emsg.format(propname), - context=self.context) - # end if - # end for - # Check for any mismatch - if ('protected' in prop_dict) and ('intent' in prop_dict): - if (prop_dict['intent'].lower() != 'in') and prop_dict['protected']: - emsg = "{} is marked protected but is intent {}" - raise ParseSyntaxError(emsg.format(prop_dict['local_name'], - prop_dict['intent']), - context=self.context) - # end if - # end if - # Look for any constituent properties - self.__is_constituent = False - for name, prop in Var.__constituent_prop_dict.items(): - if (name in prop_dict) and \ - (prop_dict[name] != prop.get_default_val(prop_dict, - context=self.context)): - self.__is_constituent = True - break - # end if - # end for - # Steal dict from caller - self._prop_dict = prop_dict - # Make sure all the variable values are valid - try: - for prop_name, prop_val in self.var_properties(): - prop = Var.get_prop(prop_name) - _ = prop.valid_value(prop_val, - prop_dict=self._prop_dict, error=True) - # end for - except CCPPError as cperr: - # Raise this error unless it represents an imported DDT type - if ((not fortran_imports) or (prop_name != 'type') or - (prop_val not in fortran_imports)): - lname = self._prop_dict['local_name'] - emsg = "{}: {}" - raise ParseSyntaxError(emsg.format(lname, cperr), - context=self.context) from cperr - # end if - # end try - - def compatible(self, other, run_env, is_tend=False): - """Return a VarCompatObj object which describes the equivalence, - compatibility, or incompatibility between <self> and <other>. - """ - # We accept character(len=*) as compatible with - # character(len=INTEGER_VALUE) - stype = self.get_prop_value('type') - skind = self.get_prop_value('kind') - sunits = self.get_prop_value('units') - sstd_name = self.get_prop_value('standard_name') - sloc_name = self.get_prop_value('local_name') - stopp = self.get_prop_value('top_at_one') - sdims = self.get_dimensions() - otype = other.get_prop_value('type') - okind = other.get_prop_value('kind') - ounits = other.get_prop_value('units') - ostd_name = other.get_prop_value('standard_name') - oloc_name = other.get_prop_value('local_name') - otopp = other.get_prop_value('top_at_one') - odims = other.get_dimensions() - compat = VarCompatObj(sstd_name, stype, skind, sunits, sdims, sloc_name, stopp, - ostd_name, otype, okind, ounits, odims, oloc_name, otopp, - run_env, - v1_context=self.context, v2_context=other.context, is_tend=is_tend) - if (not compat) and (run_env.logger is not None): - incompat_str = compat.incompat_reason - if incompat_str is not None: - run_env.logger.info('{}'.format(incompat_str)) - # end if (no else) - # end if - return compat - - def adjust_intent(self, src_var): - """Add an intent to this Var or adjust its existing intent. - Note: An existing intent can only be adjusted to 'inout' - """ - if 'intent' in self._prop_dict: - my_intent = self.get_prop_value('intent') - else: - my_intent = None - # end if - sv_intent = src_var.get_prop_value('intent') - if not sv_intent: - sv_intent = 'in' - # end if - if sv_intent in ['inout', 'out'] and self.get_prop_value('protected'): - lname = self.get_prop_value('local_name') - lctx = context_string(self.context) - emsg = "Attempt to set intent of {}{} to {}, only 'in' allowed " - emsg += "for 'protected' variable." - if src_var: - slname = src_var.get_prop_value('local_name') - sctx = context_string(src_var.context) - emsg += "\nintent source: {}{}".format(slname, sctx) - # end if - raise CCPPError(emsg.format(lname, lctx, sv_intent)) - # end if (else, no error) - if my_intent: - if my_intent != sv_intent: - self._prop_dict['intent'] = 'inout' - # end if (no else, intent is okay) - else: - self._prop_dict['intent'] = sv_intent - # end if - - @staticmethod - def get_prop(name, spec_type=None): - """Return VariableProperty object for <name> or None""" - prop = None - if (spec_type is None) and (name in Var.__var_propdict): - prop = Var.__var_propdict[name] - elif (spec_type is not None) and (name in Var.__spec_propdict): - prop = Var.__spec_propdict[name] - # end if (else prop = None) - return prop - - def var_properties(self): - """Return an iterator for this Var's property dictionary""" - return self._prop_dict.items() - - def copy_prop_dict(self, subst_dict=None): - """Create a copy of our prop_dict, possibly substituting properties - from <subst_dict>.""" - cprop_dict = {} - # Start with a straight copy of this variable's prop_dict - for prop, val in self.var_properties(): - cprop_dict[prop] = val - # end for - # Now add or substitute properties from <subst_dict> - if subst_dict: - for prop in subst_dict.keys(): - cprop_dict[prop] = subst_dict[prop] - # end for - # end if - # Special key for creating a copy of a DDT (see Var.__init__) - if self.is_ddt(): - cprop_dict['ddt_type'] = cprop_dict['type'] - # end if - return cprop_dict - - def clone(self, subst_dict=None, remove_intent=False, - source_name=None, source_type=None, context=None): - """Create a clone of this Var object with properties from <subst_dict> - overriding this variable's properties. <subst_dict> may also be - a string in which case only the local_name property is changed - (to the value of the <subst_dict> string). - If <remove_intent> is True, remove the 'intent' property, if present. - This can be used to promote a variable to module level. - The optional <source_name>, <source_type>, and <context> inputs - allow the clone to appear to be coming from a designated source, - by default, the source and type are the same as this Var (self). - """ - if isinstance(subst_dict, str): - subst_dict = {'local_name':subst_dict} - elif subst_dict is None: - subst_dict = {} - # end if - cprop_dict = self.copy_prop_dict(subst_dict=subst_dict) - if remove_intent and ('intent' in cprop_dict): - del cprop_dict['intent'] - # end if - if source_name is None: - source_name = self.source.name - # end if - if source_type is None: - source_type = self.source.ptype - # end if - if context is None: - context = self._context - # end if - psource = ParseSource(source_name, source_type, context) - - return Var(cprop_dict, psource, self.run_env, clone_source=self) - - def get_prop_value(self, name): - """Return the value of key, <name> if <name> is in this variable's - property dictionary. - If <name> is not in the prop dict but does have a <default_fn_in> - property, return the value specified by calling that function. - Otherwise, return None - """ - if name in self._prop_dict: - pvalue = self._prop_dict[name] - elif name in Var.__var_propdict: - vprop = Var.__var_propdict[name] - if vprop.optional: - pvalue = vprop.get_default_val(self._prop_dict, - context=self.context) - else: - pvalue = None - # end if - else: - pvalue = None - # end if - return pvalue - - def handle_array_ref(self): - """If this Var's local_name is an array ref, add in the array - reference indices to the Var's dimensions. - Return the (stripped) local_name and the full dimensions. - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() - ('foo', []) - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() - ('foo', ['ccpp_constant_one:dim1']) - >>> Var({'local_name' : 'foo(:,:,bar)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1,ccpp_constant_one:dim2)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() - ('foo', ['ccpp_constant_one:dim1', 'ccpp_constant_one:dim2', 'bar']) - >>> Var({'local_name' : 'foo(bar,:)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() - ('foo', ['bar', 'ccpp_constant_one:dim1']) - >>> Var({'local_name' : 'foo(bar)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Call dims mismatch for foo(bar), not enough colons - >>> Var({'local_name' : 'foo(:,bar,:)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Call dims mismatch for foo(:,bar,:), not enough dims - >>> Var({'local_name' : 'foo(:,:,bar)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Call dims mismatch for foo(:,:,bar), not enough dims - >>> Var({'local_name' : 'foo(:,bar)', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '(ccpp_constant_one:dim1,ccpp_constant_one:dim2)', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).handle_array_ref() #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Call dims mismatch for foo(:,bar), too many dims - """ - dimlist = self.get_dimensions() - aref = self.array_ref() - if aref is not None: - lname = aref.group(1) - # Substitute dimensions for colons in array reference - sdimlist = dimlist - num_dims = len(sdimlist) - dimlist = [x.strip() for x in aref.group(2).split(',')] - num_colons = sum(dim == ':' for dim in dimlist) - cind = 0 - if num_dims > num_colons: - emsg = 'Call dims mismatch for {}, not enough colons' - lname = self.get_prop_value('local_name') - raise CCPPError(emsg.format(lname)) - # end if - for dind, dim in enumerate(dimlist): - if dim == ':': - if cind >= num_dims: - emsg = 'Call dims mismatch for {}, not enough dims' - lname = self.get_prop_value('local_name') - raise CCPPError(emsg.format(lname)) - # end if - dimlist[dind] = sdimlist[cind] - cind += 1 - # end if - # end for - if cind < num_colons: - emsg = 'Call dims mismatch for {}, too many dims' - lname = self.get_prop_value('local_name') - raise CCPPError(emsg.format(lname)) - # end if - else: - lname = self.get_prop_value('local_name') - # end if - return lname, dimlist - - def call_dimstring(self, var_dicts=None, - explicit_dims=False, loop_subst=False): - """Return the dimensions string for a variable call. - If <var_dict> is present, find and substitute a local_name for - each standard_name in this variable's dimensions. - If <var_dict> is not present, return a colon for each dimension. - If <explicit_dims> is True, include the variable's dimensions. - If <loop_subst> is True, apply a loop substitution, if found for any - missing dimension. - """ - emsg = '' - _, dims = self.handle_array_ref() - if var_dicts is not None: - dimlist = [] - sepstr = '' - for dim in dims: - # Decide whether to list all dimensions or to replace - # a range with a colon. - dstdnames = dim.split(':') - add_dims = explicit_dims or (len(dstdnames) == 1) - dvar = None - if add_dims and loop_subst: - for vdict in var_dicts: - dvar = vdict.find_loop_dim_match(dim) - if dvar is not None: - break - # end if - # end for - if dvar: - dimlist.append(dvar) - # end if - if (not dvar) and add_dims: - dnames = [] - for stdname in dstdnames: - for vdict in var_dicts: - dvar = vdict.find_variable(standard_name=stdname, - any_scope=False) - if dvar is not None: - break - # end if - # end for - if dvar: - # vdict is the dictionary where <dvar> was found - dnames.append(dvar.call_string(vdict)) - # end if - if not dvar: - emsg += sepstr + "No variable found in " - vnames = [x.name for x in var_dicts] - if len(vnames) > 2: - vstr = ', '.join(vnames[:-1]) - vstr += ', or {}'.format(vnames[-1]) - elif len(vnames) > 1: - vstr = ' or '.join(vnames) - else: - vstr = vnames[0] - # end if - emsg += "{} for dimension '".format(vstr) - emsg += stdname + "' in {vlnam}" - sepstr = '\n' - # end if - # end for - dimlist.append(':'.join(dnames)) - elif not add_dims: - dimlist.append(':') - # end if (no else needed, we must have found loop substitution) - # end for - else: - dimlist = [':']*len(dims) - # end if - if dimlist: - dimstr = '(' + ','.join(dimlist) + ')' - else: - dimstr = '' # It ends up being a scalar reference - # end if - if emsg: - ctx = context_string(self.context) - emsg += "{ctx}" - lname = self.get_prop_value('local_name') - raise CCPPError(emsg.format(vlnam=lname, ctx=ctx)) - # end if - return dimstr - - def call_string(self, var_dict, loop_vars=None): - """Construct the actual argument string for this Var by translating - standard names to local names. - String includes array bounds unless loop_vars is None. - if <loop_vars> is not None, look there first for array bounds, - even if usage requires a loop substitution. - """ - if loop_vars is None: - call_str = self.get_prop_value('local_name') - # Look for dims in case this is an array selection variable - dind = call_str.find('(') - if dind > 0: - dimstr = call_str[dind+1:].rstrip()[:-1] - dims = [x.strip() for x in dimstr.split(',')] - call_str = call_str[:dind].strip() - else: - dims = None - # end if - else: - call_str, dims = self.handle_array_ref() - # end if - if dims: - call_str += '(' - dsep = '' - for dim in dims: - if loop_vars: - lname = loop_vars.find_loop_dim_match(dim) - else: - lname = None - # end if - if lname is None: - isep = '' - lname = "" - for item in dim.split(':'): - if item: - dvar = var_dict.find_variable(standard_name=item, - any_scope=False) - if dvar is None: - try: - dval = int(item) - iname = item - except ValueError: - iname = None - # end try - else: - iname = dvar.call_string(var_dict, - loop_vars=loop_vars) - # end if - else: - iname = '' - # end if - if iname is not None: - lname = lname + isep + iname - isep = ':' - else: - errmsg = 'No local variable {} in {}{}' - ctx = context_string(self.context) - dname = var_dict.name - raise CCPPError(errmsg.format(item, dname, ctx)) - # end if - # end for - # end if - if lname is not None: - call_str += dsep + lname - dsep = ', ' - else: - errmsg = 'Unable to convert {} to local variables in {}{}' - ctx = context_string(self.context) - raise CCPPError(errmsg.format(dim, var_dict.name, ctx)) - # end if - # end for - call_str += ')' - # end if - return call_str - - def valid_value(self, prop_name, test_value=None, error=False): - """Return a valid version of <test_value> if it is a valid value - for the property, <prop_name>. - If <test_value> is not valid, return None or raise an exception, - depending on the value of <error>. - If <test_value> is None, use the current value of <prop_name>. - """ - vprop = Var.get_prop(prop_name) - if vprop is not None: - if test_value is None: - test_val = self.get_prop_value(prop_name) - # end if - valid = vprop.valid_value(test_val, - prop_dict=self._prop_dict, error=error) - else: - valid = None - errmsg = 'Invalid variable property, {}' - raise ParseInternalError(errmsg.format(prop_name)) - # end if - return valid - - def array_ref(self, local_name=None): - """If this Var's local_name is an array reference, return a - Fortran array reference regexp match. - Otherwise, return None""" - if local_name is None: - local_name = self.get_prop_value('local_name') - # end if - match = FORTRAN_SCALAR_REF_RE.match(local_name) - return match - - def intrinsic_elements(self, check_dict=None, ddt_lib=None): - """Return a list of the standard names of this Var object's 'leaf' - intrinsic elements or this Var object's standard name if it is an - intrinsic 'leaf' variable. - If this Var object cannot be reduced to one or more intrinsic 'leaf' - variables (e.g., a DDT Var with no named elements), return None. - A 'leaf' intrinsic Var is a Var of intrinsic Fortran type which has - no children. If a Var has children, those children will be searched - to find leaves. If a Var is a DDT, its named elements are searched. - If <check_dict> is not None, it is checked for children if none are - found in this variable (via finding a variable in <check_dict> with - the same standard name). - Currently, an array of DDTs is not processed (return None) since - Fortran does not support a way to reference those elements. - """ - element_names = None - if self.is_ddt(): - dtitle = self.get_prop_value('type') - if ddt_lib and (dtitle in ddt_lib): - element_names = [] - ddt_def = ddt_lib[dtitle] - for dvar in ddt_def.variable_list(): - delems = dvar.intrinsic_elements(check_dict=check_dict, - ddt_lib=ddt_lib) - if delems: - element_names.extend(delems) - # end if - # end for - if not element_names: - element_names = None - # end if - else: - errmsg = f'No ddt_lib or ddt {dtitle} not in ddt_lib' - raise CCPPError(errmsg) - # end if - # end if - children = self.children() - if (not children) and check_dict: - stdname = self.get_prop_value("standard_name") - pvar = check_dict.find_variable(standard_name=stdname, - any_scope=True) - if pvar: - children = pvar.children() - # end if - # end if - if children: - element_names = list() - for child in children: - child_elements = child.intrinsic_elements() - if isinstance(child_elements, str): - child_elements = [child_elements] - # end if - if child_elements: - for elem in child_elements: - if elem: - element_names.append(elem) - # end if - # end for - # end if - # end for - else: - element_names = self.get_prop_value('standard_name') - # end if - return element_names - - @classmethod - def constituent_property_names(cls): - """Return a list of the names of constituent properties""" - return Var.__constituent_prop_dict.keys() - - @property - def parent(self): - """Return this variable's parent variable (or None)""" - return self.__parent_var - - @parent.setter - def parent(self, parent_var): - """Set this variable's parent if not already set""" - if self.__parent_var is not None: - emsg = 'Attempting to set parent for {} but parent already set' - lname = self.get_prop_value('local_name') - raise ParseInternalError(emsg.format(lname)) - # end if - if isinstance(parent_var, Var): - self.__parent_var = parent_var - parent_var.add_child(self) - else: - emsg = 'Attempting to set parent for {}, bad parent type, {}' - lname = self.get_prop_value('local_name') - raise ParseInternalError(emsg.format(lname, type_name(parent_var))) - # end if - - def add_child(self, cvar): - """Add <cvar> as a child of this Var object""" - if cvar not in self.__children: - self.__children.append(cvar) - # end if - - def children(self): - """Return an iterator over this object's children or None if the - object has no children.""" - children = self.__children - if not children: - pvar = self - while (not children) and pvar.clone_source: - pvar = pvar.clone_source - children = pvar.children() - # end while - # end if - return iter(children) if children else None - - @property - def var(self): - "Return this object (base behavior for derived classes such as VarDDT)" - return self - - @property - def context(self): - """Return this variable's parsed context""" - return self._context - - @property - def source(self): - """Return the source object for this variable""" - return self.__source - - @source.setter - def source(self, new_source): - """Reset this Var's source if <new_source> seems legit""" - if isinstance(new_source, ParseSource): - self.__source = new_source - else: - errmsg = 'Attemping to set source of {} ({}) to "{}"' - stdname = self.get_prop_value('standard_name') - lname = self.get_prop_value('local_name') - raise ParseInternalError(errmsg.format(stdname, lname, new_source)) - # end if - - @property - def clone_source(self): - """Return this Var object's clone source (or None)""" - return self.__clone_source - - @property - def host_interface_var(self): - """True iff self is included in the host model interface calls""" - return self.source.ptype == 'host' - - @property - def run_env(self): - """Return the CCPPFrameworkEnv object used to create this Var object.""" - return self.__run_env - - def get_dimensions(self): - """Return a list with the variable's dimension strings""" - dims = self.valid_value('dimensions') - return dims - - def get_dim_stdnames(self, include_constants=True): - """Return a set of all the dimension standard names for this Var""" - dimset = set() - for dim in self.get_dimensions(): - for name in dim.split(':'): - # Weed out the integers - try: - _ = int(name) - except ValueError: - # Not an integer, maybe add it - if include_constants or (not name in CCPP_CONSTANT_VARS): - dimset.add(name) - # end if - # end try - # end for - # end for - return dimset - - def get_rank(self): - """Return the variable's rank (zero for scalar)""" - dims = self.get_dimensions() - return len(dims) - - def has_horizontal_dimension(self, dims=None): - """Return horizontal dimension standard name string for - <self> or <dims> (if present) if a horizontal dimension is - present in the list""" - if dims is None: - vdims = self.get_dimensions() - else: - vdims = dims - # end if - return find_horizontal_dimension(vdims)[0] - - def has_vertical_dimension(self, dims=None): - """Return vertical dimension standard name string for - <self> or <dims> (if present) if a vertical dimension is - present in the list""" - if dims is None: - vdims = self.get_dimensions() - else: - vdims = dims - # end if - return find_vertical_dimension(vdims)[0] - - def conditional(self, vdicts): - """Convert conditional expression from active attribute - (i.e. in standard name format) to local names based on vdict. - Return conditional and a list of variables needed to evaluate - the conditional. - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real',}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([{}]) - ('.true.', []) - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'False'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables={})]) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - Exception: Cannot find variable 'false' for generating conditional for 'False' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'mom_gone'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([ VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'bar', 'standard_name' : 'mom_home', 'units' : '', 'dimensions' : '()', 'type' : 'logical'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV)]) ]) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - Exception: Cannot find variable 'mom_gone' for generating conditional for 'mom_gone' - >>> Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'mom_home'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([ VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'bar', 'standard_name' : 'mom_home', 'units' : '', 'dimensions' : '()', 'type' : 'logical'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV)]) ])[0] - 'bar' - >>> len(Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'active' : 'mom_home'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV).conditional([ VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'bar', 'standard_name' : 'mom_home', 'units' : '', 'dimensions' : '()', 'type' : 'logical'}, ParseSource('vname', 'HOST', ParseContext()), _MVAR_DUMMY_RUN_ENV)]) ])[1]) - 1 - """ - - active = self.get_prop_value('active') - conditional = '' - vars_needed = [] - - # Find all words in the conditional, for each of them look - # for a matching standard name in the list of known variables - items = FORTRAN_CONDITIONAL_REGEX.findall(active) - for item in items: - item = item.lower() - if item in FORTRAN_CONDITIONAL_REGEX_WORDS: - conditional += item - else: - # Keep integers - try: - int(item) - conditional += item - except ValueError: - dvar = None - for vdict in vdicts: - dvar = vdict.find_variable(standard_name=item, any_scope=True) # or any_scope=False ? - if dvar: - break - if not dvar: - raise Exception(f"Cannot find variable '{item}' for generating conditional for '{active}'") - conditional += dvar.get_prop_value('local_name') - vars_needed.append(dvar) - return (conditional, vars_needed) - - def write_def(self, outfile, indent, wdict, allocatable=False, target=False, - dummy=False, add_intent=None, extra_space=0, public=False): - """Write the definition line for the variable to <outfile>. - If <dummy> is True, include the variable's intent. - If <dummy> is True but the variable has no intent, add the - intent indicated by <add_intent>. This is intended for host model - variables and it is an error to not pass <add_intent> if <dummy> - is True and the variable has no intent property.""" - stdname = self.get_prop_value('standard_name') - if stdname in CCPP_CONSTANT_VARS: - # There is no declaration line for a constant - return - # end if - if self.is_ddt(): - vtype = 'type' - else: - vtype = self.get_prop_value('type') - # end if - kind = self.get_prop_value('kind') - name = self.get_prop_value('local_name') - aref = self.array_ref(local_name=name) - if aref is not None: - name = aref.group(1) - # end if - dims = self.get_dimensions() - if dims: - if allocatable or dummy: - dimstr = '(:' + ',:'*(len(dims) - 1) + ')' - else: - dimstr = self.call_dimstring(var_dicts=[wdict]) - else: - dimstr = '' - # end if - protected = self.get_prop_value('protected') - polymorphic = self.get_prop_value('polymorphic') - if dummy: - intent = self.get_prop_value('intent') - else: - intent = None - # end if - if protected and allocatable: - errmsg = "Cannot create allocatable variable from protected, {}" - raise CCPPError(errmsg.format(name)) - # end if - if dummy and (intent is None): - if add_intent is not None: - intent = add_intent - else: - errmsg = f"<add_intent> is missing for dummy argument, {name}" - raise CCPPError(errmsg) - # end if - # end if - optional = self.get_prop_value('optional') - if protected and dummy: - intent_str = 'intent(in) ' - elif allocatable: - if dimstr or polymorphic: - intent_str = 'allocatable ' - if target: - intent_str = 'allocatable,' - intent_str += ' target' - else: - intent_str = ' '*13 - # end if - elif intent is not None: - alloval = self.get_prop_value('allocatable') - if (intent.lower()[-3:] == 'out') and alloval: - intent_str = f"allocatable, intent({intent}){' '*(5 - len(intent))}" - elif optional: - intent_str = f"intent({intent}),{' '*(5 - len(intent))}" - intent_str += 'target, optional ' - else: - intent_str = f"intent({intent}){' '*(5 - len(intent))}" - # end if - elif not dummy: - intent_str = ' '*20 - else: - intent_str = ' '*13 - # end if - if intent_str.strip(): - comma = ',' - else: - comma = ' ' - # end if - if self.get_prop_value('target'): - targ = ", target" - else: - targ = "" - # end if - comma = targ + comma - extra_space -= len(targ) - if self.is_ddt(): - if polymorphic: - dstr = "class({kind}){cspace}{intent} :: {name}{dims}" - cspace = comma + ' '*(extra_space + 12 - len(kind)) - else: - dstr = "type({kind}){cspace}{intent} :: {name}{dims}" - cspace = comma + ' '*(extra_space + 13 - len(kind)) - # end if - else: - if kind: - dstr = "{type}({kind}){cspace}{intent} :: {name}{dims}" - cspace = comma + ' '*(extra_space + 17 - len(vtype) - len(kind)) - else: - dstr = "{type}{cspace}{intent} :: {name}{dims}" - cspace = comma + ' '*(extra_space + 19 - len(vtype)) - # end if - # end if - outfile.write(dstr.format(type=vtype, kind=kind, intent=intent_str, - name=name, dims=dimstr, cspace=cspace, - sname=stdname), indent) - - def write_ptr_def(self, outfile, indent, name, kind, dimstr, vtype, extra_space=0): - """Write the definition line for local null pointer declaration to <outfile>.""" - comma = ', ' - if kind: - dstr = "{type}({kind}){cspace}pointer :: {name}{dims}{cspace2} => null()" - cspace = comma + ' '*(extra_space + 20 - len(vtype) - len(kind)) - cspace2 = ' '*(20 -len(name) - len(dimstr)) - else: - dstr = "{type}{cspace}pointer :: {name}{dims}{cspace2} => null()" - cspace = comma + ' '*(extra_space + 22 - len(vtype)) - cspace2 = ' '*(20 -len(name) - len(dimstr)) - # end if - outfile.write(dstr.format(type=vtype, kind=kind, name=name, dims=dimstr, - cspace=cspace, cspace2=cspace2), indent) - - def is_ddt(self): - """Return True iff <self> is a DDT type.""" - return not self.__intrinsic - - def is_constituent(self): - """Return True iff <self> is a constituent variable.""" - return self.__is_constituent - - def __str__(self): - """Print representation or string for Var objects""" - return "<Var {standard_name}: {local_name}>".format(**self._prop_dict) - - def __repr__(self): - """Object representation for Var objects""" - base = super().__repr__() - pind = base.find(' object ') - if pind >= 0: - pre = base[0:pind] - else: - pre = '<Var' - # end if - bind = base.find('at 0x') - if bind >= 0: - post = base[bind:] - else: - post = '>' - # end if - return '{} {}: {} {}'.format(pre, self._prop_dict['standard_name'], - self._prop_dict['local_name'], post) - -############################################################################### - -class FortranVar(Var): - """A class to hold the metadata for a Fortran variable which can - contain properties not used in CCPP metadata. - """ - - __fortran_props = [VariableProperty('optional', bool, - optional_in=True, default_in=False)] - - def __init__(self, prop_dict, source, run_env, context=None, - clone_source=None, fortran_imports=None): - """Initialize a FortranVar object. - """ - - # Remove and save any Fortran-only properties - save_dict = {} - for prop in self.__fortran_props: - if prop.name in prop_dict: - save_dict[prop.name] = prop_dict[prop.name] - del prop_dict[prop.name] - # end if - # end for - # Initialize Var - super().__init__(prop_dict, source, run_env, context=context, - clone_source=clone_source, - fortran_imports=fortran_imports) - # Now, restore the saved properties - for prop in save_dict: - self._prop_dict[prop] = save_dict[prop] - # end for - - -############################################################################### - -class VarSpec: - """A class to hold a standard_name description of a variable. - A scalar variable is just a standard name while an array also - contains a comma-separated list of dimension standard names in parentheses. - """ - - def __init__(self, var): - """Initialize the common properties of this VarSpec-based object""" - self.__name = var.get_prop_value('standard_name') - self.__dims = var.get_dimensions() - if not self.__dims: - self.__dims = None - # end if - - @property - def name(self): - """Return the name of this VarSpec-based object""" - return self.__name - - def get_dimensions(self): - """Return the dimensions of this VarSpec-based object.""" - rdims = self.__dims - return rdims - - def __repr__(self): - """Return a representation of this object""" - if self.__dims is not None: - repr_str = f"{self.__name}({', '.join(self.__dims)})" - else: - repr_str = self.__name - # end if - return repr_str - -############################################################################### - -__CCPP_PARSE_CONTEXT = ParseContext(filename='metavar.py') - -############################################################################### - -def ccpp_standard_var(std_name, source_type, run_env, - context=None, intent='out'): - """If <std_name> is a CCPP standard variable name, return a variable - with that name. - Otherwise return None. - """ - if std_name in CCPP_STANDARD_VARS: - # Copy the dictionary because Var can change it - vdict = dict(CCPP_STANDARD_VARS[std_name]) - if context is None: - psource = ParseSource('ccpp_standard_vars', source_type, - __CCPP_PARSE_CONTEXT) - else: - psource = ParseSource('ccpp_standard_vars', source_type, context) - # end if - if source_type.lower() == 'scheme': - vdict['intent'] = intent - # end if - newvar = Var(vdict, psource, run_env) - else: - newvar = None - # end if - return newvar - -############################################################################### - -class VarAction: - """A base class for variable actions such as loop substitutions or - temporary variable handling.""" - - def __init__(self): - """Initialize this action (nothing to do)""" - # pass # Nothing general here yet - - def add_local(self, vadict, source): - """Add any variables needed by this action to <dict>. - Variable(s) will appear to originate from <source>.""" - raise ParseInternalError('VarAction add_local method must be overriden') - - def write_action(self, vadict, dict2=None, any_scope=False): - """Return a string setting implementing the action of <self>. - Variables must be in <dict> or <dict2>""" - errmsg = 'VarAction write_action method must be overriden' - raise ParseInternalError(errmsg) - - def equiv(self, vmatch): - """Return True iff <vmatch> is equivalent to <self>. - Equivalence at this level is tested by comparing the type - of the objects. - equiv should be overridden with a method that first calls this - method and then tests class-specific object data.""" - return vmatch.__class__ == self.__class__ - - def add_to_list(self, vlist): - """Add <self> to <vlist> unless <self> or its equivalent is - already in <vlist>. This method should not need to be overriden. - Return the (possibly modified) list""" - ok_to_add = True - for vlist_action in vlist: - if vlist_action.equiv(self): - ok_to_add = False - break - # end if - # end for - if ok_to_add: - vlist.append(self) - # end if - return vlist - -############################################################################### - -class VarLoopSubst(VarAction): - """A class to handle required loop substitutions where the host model - (or a suite part) does not provide a loop-like variable used by a - suite part or scheme or where a host model passes a subset of a - dimension at run time.""" - - def __init__(self, missing_stdname, required_stdnames, - local_name, set_action): - """Initialize this variable loop substitution""" - self._missing_stdname = missing_stdname - self._local_name = local_name - if isinstance(required_stdnames, Var): - self._required_stdnames = (required_stdnames,) - else: - # Make sure required_stdnames is iterable - try: - _ = (v for v in required_stdnames) - self._required_stdnames = required_stdnames - except TypeError: - emsg = "required_stdnames must be a tuple or a list" - raise ParseInternalError(emsg) - # end try - # end if - self._set_action = set_action - super().__init__() - - def has_subst(self, vadict, any_scope=False): - """Determine if variables for the required standard names of this - VarLoopSubst object are present in <vadict> (or in the parents of - <vadict>) if <any_scope> is True. - Return a list of the required variables on success, None on failure. - """ - # A template for 'missing' should be in the standard variable list - subst_list = list() - for name in self.required_stdnames: - svar = vadict.find_variable(standard_name=name, any_scope=any_scope) - if svar is None: - subst_list = None - break - # end i - subst_list.append(svar) - # end for - return subst_list - - def add_local(self, vadict, source, run_env): - """Add a Var created from the missing name to <vadict>""" - if self.missing_stdname not in vadict: - lname = self._local_name - local_name = vadict.new_internal_variable_name(prefix=lname) - prop_dict = {'standard_name':self.missing_stdname, - 'local_name':local_name, - 'type':'integer', 'units':'count', 'dimensions':'()'} - var = Var(prop_dict, source, run_env) - vadict.add_variable(var, run_env, exists_ok=True, gen_unique=True) - # end if - - def equiv(self, vmatch): - """Return True iff <vmatch> is equivalent to <self>. - Equivalence is determined by matching the missing standard name - and the required standard names""" - is_equiv = super().equiv(vmatch) - if is_equiv: - is_equiv = vmatch.missing_stdname == self.missing_stdname - # end if - if is_equiv: - for dim1, dim2 in zip(vmatch.required_stdnames, - self.required_stdnames): - if dim1 != dim2: - is_equiv = False - break - # end if - # end for - # end if - return is_equiv - - def write_action(self, vadict, dict2=None, any_scope=False): - """Return a string setting the correct values for our - replacement variable. Variables must be in <vadict> or <dict2>""" - action_dict = {} - if self._set_action: - for stdname in self.required_stdnames: - var = vadict.find_variable(standard_name=stdname, - any_scope=any_scope) - if (var is None) and (dict2 is not None): - var = dict2.find_variable(standard_name=stdname, - any_scope=any_scope) - # end if - if var is None: - errmsg = "Required variable, {}, not found" - raise CCPPError(errmsg.format(stdname)) - # end if - action_dict[stdname] = var.get_prop_value('local_name') - # end for - var = vadict.find_variable(standard_name=self.missing_stdname) - if var is None: - errmsg = "Required variable, {}, not found" - raise CCPPError(errmsg.format(self.missing_stdname)) - # end if - action_dict[self.missing_stdname] = var.get_prop_value('local_name') - # end if - return self._set_action.format(**action_dict) - - def write_metadata(self, mfile): - """Write our properties as metadata to <mfile>""" - pass # Currently no properties to write - - @property - def required_stdnames(self): - """Return the _required_stdnames for this object""" - return self._required_stdnames - - @property - def missing_stdname(self): - """Return the _missing_stdname for this object""" - return self._missing_stdname - - def __repr__(self): - """Return string representing this VarLoopSubst object""" - action_dict = {} - repr_str = '' - if self._set_action: - for stdname in self.required_stdnames: - action_dict[stdname] = stdname - # end for - action_dict[self.missing_stdname] = self.missing_stdname - repr_str = self._set_action.format(**action_dict) - else: - repr_str = "{} => {}".format(self.missing_stdname, - ':'.join(self.required_stdnames)) - # end if - return repr_str - - def __str__(self): - """Return print string for this VarLoopSubst object""" - return "<{}>".format(self.__repr__()) - -# Substitutions where a new variable must be created -CCPP_VAR_LOOP_SUBSTS = { - 'horizontal_loop_extent' : - VarLoopSubst('horizontal_loop_extent', - ('horizontal_loop_begin', 'horizontal_loop_end'), 'ncol', - '{} = {} - {} + 1'.format('{horizontal_loop_extent}', - '{horizontal_loop_end}', - '{horizontal_loop_begin}')), - 'horizontal_loop_begin' : - VarLoopSubst('horizontal_loop_begin', - ('ccpp_constant_one',), 'one', '{horizontal_loop_begin} = 1'), - 'horizontal_loop_end' : - VarLoopSubst('horizontal_loop_end', - ('horizontal_loop_extent',), 'ncol', - '{} = {}'.format('{horizontal_loop_end}', - '{horizontal_loop_extent}')), - 'vertical_layer_dimension' : - VarLoopSubst('vertical_layer_dimension', - ('vertical_layer_index',), 'layer_index', ''), - 'vertical_interface_dimension' : - VarLoopSubst('vertical_interface_dimension', - ('vertical_interface_index',), 'level_index', '') -} - -############################################################################### - -class VarDictionary(OrderedDict): - """ - A class to store and cross-check variables from one or more metadata - headers. The class also serves as a scoping construct so that a variable - can be found in an innermost available scope. - The dictionary is organized by standard_name. It is an error to try - to add a variable if its standard name is already in the dictionary. - Scoping is a tree of VarDictionary objects. - >>> VarDictionary('foo', _MVAR_DUMMY_RUN_ENV) - VarDictionary(foo) - >>> VarDictionary('bar', _MVAR_DUMMY_RUN_ENV, variables={}) - VarDictionary(bar) - >>> test_dict = VarDictionary('baz', _MVAR_DUMMY_RUN_ENV, variables=Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)) - >>> print(test_dict.name) - baz - >>> print(test_dict.variable_list()) #doctest: +ELLIPSIS - [<metavar.Var hi_mom: foo at 0x...>] - >>> print("{}".format(VarDictionary('baz', _MVAR_DUMMY_RUN_ENV, variables=Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)))) - VarDictionary(baz, ['hi_mom']) - >>> test_dict = VarDictionary('qux', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)]) - >>> print(test_dict.name) - qux - >>> print(test_dict.variable_list()) #doctest: +ELLIPSIS - [<metavar.Var hi_mom: foo at 0x...>] - >>> VarDictionary('boo', _MVAR_DUMMY_RUN_ENV).add_variable(Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV), _MVAR_DUMMY_RUN_ENV) - - >>> VarDictionary('who', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)]).prop_list('local_name') - ['foo'] - >>> VarDictionary('who', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'who_var1', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV),Var({'local_name' : 'who_var', 'standard_name' : 'bye_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)]).new_internal_variable_name() - 'who_var2' - >>> VarDictionary('who', _MVAR_DUMMY_RUN_ENV, variables=[Var({'local_name' : 'who_var1', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)]).new_internal_variable_name(prefix='bar') - 'bar' - >>> VarDictionary('glitch', _MVAR_DUMMY_RUN_ENV, variables=Var({'local_name' : 'foo', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname', 'scheme', ParseContext()), _MVAR_DUMMY_RUN_ENV)).add_variable(Var({'local_name' : 'bar', 'standard_name' : 'hi_mom', 'units' : 'm s-1', 'dimensions' : '()', 'type' : 'real', 'intent' : 'in'}, ParseSource('vname2', 'DDT', ParseContext()), _MVAR_DUMMY_RUN_ENV), _MVAR_DUMMY_RUN_ENV) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseSyntaxError: Invalid Duplicate standard name, 'hi_mom', at <standard input>: - """ - - def __init__(self, name, run_env, variables=None, - parent_dict=None): - """Unlike dict, VarDictionary only takes a Var or Var list""" - super().__init__() - self.__name = name - self.__run_env = run_env - self.__parent_dict = parent_dict - if parent_dict is not None: - parent_dict.add_sub_scope(self) - # end if - self.__sub_dicts = list() - self.__local_names = {} # local names in use - if isinstance(variables, Var): - self.add_variable(variables, run_env) - elif isinstance(variables, list): - for var in variables: - self.add_variable(var, run_env) - # end for - elif isinstance(variables, VarDictionary): - for stdname in variables.keys(): - self[stdname] = variables[stdname] - # end for - elif isinstance(variables, dict): - # variables may not be in 'order', but we accept them anyway - for key in variables.keys(): - var = variables[key] - stdname = var.get_prop_value('standard_name') - self[stdname] = variables[key] - # end for - elif variables is not None: - raise ParseInternalError(f'Illegal type for variables, {type_name(variables)} in {self.name}') - # end if - - @property - def name(self): - """Return this dictionary's name""" - return self.__name - - @property - def parent(self): - """Return the parent dictionary of this dictionary""" - return self.__parent_dict - - @staticmethod - def include_var_in_list(var, std_vars, loop_vars, consts): - """Return True iff <var> is of a type allowed by the logicals, - <std_vars> (not constants or loop_vars), - <loop_vars> a variable ending in '_extent', '_begin', '_end', or - <consts> a variable with the 'protected' property. - """ - standard_name = var.get_prop_value('standard_name') - const_var = standard_name in CCPP_CONSTANT_VARS - loop_var = standard_name in CCPP_LOOP_VAR_STDNAMES - include_var = (consts and const_var) or (loop_var and loop_vars) - if not include_var: - std_var = not (loop_var or const_var) - include_var = std_vars and std_var - # end if - return include_var - - def variable_list(self, recursive=False, - std_vars=True, loop_vars=True, consts=True): - """Return a list of all variables""" - if recursive and (self.__parent_dict is not None): - vlist = self.__parent_dict.variable_list(recursive=recursive, - std_vars=std_vars, - loop_vars=loop_vars, - consts=consts) - else: - vlist = list() - # end if - for stdnam in self: - var = self[stdnam] - if self.include_var_in_list(var, std_vars=std_vars, - loop_vars=loop_vars, consts=consts): - vlist.append(var) - # end if - # end for - return vlist - - def add_variable(self, newvar, run_env, exists_ok=False, gen_unique=False, - adjust_intent=False): - """Add <newvar> if it does not conflict with existing entries - If <exists_ok> is True, attempting to add an identical copy is okay. - If <gen_unique> is True, a new local_name will be created if a - local_name collision is detected. - if <adjust_intent> is True, adjust conflicting intents to inout.""" - standard_name = newvar.get_prop_value('standard_name') - cvar = self.find_variable(standard_name=standard_name, any_scope=False) - if (standard_name in self) and (not exists_ok): - # We already have a matching variable, error! - if self.__run_env.logger is not None: - emsg = "Attempt to add duplicate variable, {} from {}" - self.__run_env.logger.error(emsg.format(standard_name, - newvar.source.name)) - # end if - emsg = "(duplicate) standard name in {}" - if cvar is not None: - emsg += ", defined at {}".format(cvar.context) - # end if - raise ParseSyntaxError(emsg.format(self.name), - token=standard_name, context=newvar.context) - # end if - if cvar is not None: - compat = cvar.compatible(newvar, run_env) - if compat.compat: - # Check for intent mismatch - vintent = cvar.get_prop_value('intent') - dintent = newvar.get_prop_value('intent') - # XXgoldyXX: Add special case for host variables here? - if vintent != dintent: - if adjust_intent: - if (vintent == 'in') and (dintent in ['inout', 'out']): - cvar.adjust_intent(newvar) - elif ((vintent == 'out') and - (dintent in ['inout', 'in'])): - cvar.adjust_intent(newvar) - # No else, variables are compatible - else: - emsg = "Attempt to add incompatible variable to {}" - emsg += "\nintent mismatch: {} ({}){} != {} ({}){}" - nlname = newvar.get_prop_value('local_name') - clname = cvar.get_prop_value('local_name') - nctx = context_string(newvar.context) - cctx = context_string(cvar.context) - raise CCPPError(emsg.format(self.name, - clname, vintent, cctx, - nlname, dintent, nctx)) - # end if - # end if - else: - if self.__run_env.logger is not None: - emsg = "Attempt to add incompatible variable, {} from {}" - emsg += "\n{}".format(compat.incompat_reason) - self.__run_env.logger.error(emsg.format(standard_name, - newvar.source.name)) - # end if - nlname = newvar.get_prop_value('local_name') - clname = cvar.get_prop_value('local_name') - cstr = context_string(cvar.context, with_comma=True) - errstr = "new variable, {}, incompatible {} between {}{} and" - raise ParseSyntaxError(errstr.format(nlname, - compat.incompat_reason, - clname, cstr), - token=standard_name, - context=newvar.context) - # end if - # end if - # Check if local_name exists in Group. If applicable, Create new - # variable with unique name. There are two instances when new names are - # created: - # - Same <local_name> used in different DDTs. - # - Different <standard_name> using the same <local_name> in a Group. - # During the Group analyze phase, <gen_unique> is True. - lname = newvar.get_prop_value('local_name') - lvar = self.find_local_name(lname) - if lvar is not None: - # Check if <lvar> is part of a different DDT than <newvar>. - # The API uses the full variable references when calling the Group Caps, - # <lvar.call_string(self))> and <newvar.call_string(self)>. - # Within the context of a full reference, it is allowable for local_names - # to be the same in different data containers. - newvar_callstr = newvar.call_string(self) - lvar_callstr = lvar.call_string(self) - if newvar_callstr and lvar_callstr: - if newvar_callstr != lvar_callstr: - if not gen_unique: - exists_ok = True - # end if - # end if - # end if - if gen_unique: - new_lname = self.new_internal_variable_name(prefix=lname) - newvar = newvar.clone(new_lname) - # Local_name needs to be the local_name for the new - # internal variable, otherwise multiple instances of the same - # local_name in the Group cap will all be overwritten with the - # same local_name - lname = new_lname - elif not exists_ok: - errstr = f"Invalid local_name: {lname} already registered" - raise ParseSyntaxError(errstr, context=newvar.source.context) - # end if (no else, things are okay) - # end if (no else, things are okay) - # Check if this variable has a parent (i.e., it is an array reference) - aref = newvar.array_ref(local_name=lname) - if aref is not None: - pname = aref.group(1).strip() - pvar = self.find_local_name(pname) - if pvar is not None: - newvar.parent = pvar - # end if - # end if - # If we make it to here without an exception, add the variable - if standard_name not in self: - self[standard_name] = newvar - # end if - lname = lname.lower() - if lname not in self.__local_names: - self.__local_names[lname] = standard_name - # end if - - def remove_variable(self, standard_name): - """Remove <standard_name> from the dictionary. - Ignore if <standard_name> is not in dict - """ - if standard_name in self: - del self[standard_name] - # end if - - def add_variable_dimensions(self, var, ignore_sources, suite_type, - to_dict=None, adjust_intent=False): - """Attempt to find a source for each dimension in <var> and add that - Variable to this dictionary or to <to_dict>, if passed. - Dimension variables which are found but whose Source is in - <ignore_sources> are not added to this dictionary. - Dimension variabes which are found at the suite level (determined - by <suite_type>) are also not added to this dictionary because - module-level suite variables are accessible by any phase. - Return an error string on failure.""" - - err_ret = '' - ctx = '' - vdims = var.get_dim_stdnames(include_constants=False) - for dimname in vdims: - if to_dict: - present = to_dict.find_variable(standard_name=dimname, - any_scope=False) - else: - present = None - # end if - if not present: - present = self.find_variable(standard_name=dimname, - any_scope=False) - # end if - if not present: - dvar = self.find_variable(standard_name=dimname, any_scope=True) - if dvar and dvar.source.ptype == suite_type: - # Do nothing - this is a module-level variable so we don't - # need to add it to any dictionaries - return - # end if - if dvar and (dvar.source.ptype not in ignore_sources): - if to_dict: - to_dict.add_variable(dvar, self.__run_env, - exists_ok=True, - adjust_intent=adjust_intent) - else: - self.add_variable(dvar, self.__run_env, exists_ok=True, - adjust_intent=adjust_intent) - # end if - else: - if err_ret: - err_ret += '\n' - else: - ctx = context_string(var.context) - # end if - vstdname = var.get_prop_value('standard_name') - err_ret += f"{self.name}: " - err_ret += f"Cannot find variable for dimension, {dimname}, of {vstdname}{ctx}" - if dvar: - lname = dvar.get_prop_value('local_name') - dctx = context_string(dvar.context) - err_ret += f"\nFound {lname} from excluded source, '{dvar.source.ptype}'{dctx}" - # end if - # end if - # end if - # end for - return err_ret - - def find_variable(self, standard_name=None, source_var=None, - any_scope=True, clone=None, - search_call_list=False, loop_subst=False): - """Attempt to return the variable matching <standard_name>. - if <standard_name> is None, the standard name from <source_var> is used. - It is an error to pass both <standard_name> and <source_var> if - the standard name of <source_var> is not the same as <standard_name>. - If <any_scope> is True, search parent scopes if not in current scope. - If the variable is not found and <clone> is not None, add a clone of - <clone> to this dictionary. - If the variable is not found and <clone> is None, return None. - <search_call_list> and <loop_subst> are not used in this base class - but are included to provide a consistent interface. - """ - if standard_name is None: - if source_var is None: - emsg = "One of <standard_name> or <source_var> must be passed." - raise ParseInternalError(emsg) - # end if - standard_name = source_var.get_prop_value('standard_name') - elif source_var is not None: - stest = source_var.get_prop_value('standard_name') - if stest != standard_name: - emsg = ("<standard_name> and <source_var> must match " + - "if both are passed.") - raise ParseInternalError(emsg) - # end if - # end if - if standard_name in CCPP_CONSTANT_VARS: - var = CCPP_CONSTANT_VARS[standard_name] - elif standard_name in self: - var = self[standard_name] - elif any_scope and (self.__parent_dict is not None): - src_clist = search_call_list - var = self.__parent_dict.find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, - clone=clone, - search_call_list=src_clist, - loop_subst=loop_subst) - else: - var = None - # end if - if (var is None) and (clone is not None): - lname = clone.get_prop_value['local_name'] - new_name = self.new_internal_variable_name(prefix=lname) - var = clone.clone(new_name) - # end if - return var - - def find_local_name(self, local_name, any_scope=False): - """Return a variable in this dictionary with local_name = <local_name> - or return None if no such variable is currently in the dictionary""" - pvar = None - lname = local_name.lower() # Case is insensitive for local names - if lname in self.__local_names: - stdname = self.__local_names[lname] - pvar = self.find_variable(standard_name=stdname, any_scope=False) - if not pvar: - emsg = 'VarDictionary {} should have standard_name, {}, ' - emsg += 'based on local_name {}' - raise ParseInternalError(emsg.format(self.name, - stdname, local_name)) - # end if (no else, pvar is fine) - elif any_scope and (self.__parent_dict is not None): - pvar = self.__parent_dict.find_local_name(local_name, - any_scope=any_scope) - # end if - return pvar - - def find_error_variables(self, any_scope=False, clone_as_out=False): - """Find and return a consistent set of error variables in this - dictionary. - First, attempt to find the set of errflg and errmsg. - Currently, there is no alternative but it will be inserted here. - If a consistent set is not found, return an empty list. - """ - err_vars = list() - # Look for the combo of errflg and errmsg - errflg = self.find_variable(standard_name="ccpp_error_code", - any_scope=any_scope) - errmsg = self.find_variable(standard_name="ccpp_error_message", - any_scope=any_scope) - if (errflg is not None) and (errmsg is not None): - if clone_as_out: - eout = errmsg.get_prop_value('intent') - if eout != 'out': - subst_dict = {'intent':'out'} - errmsg = errmsg.clone(subst_dict) - # end if - # end if - err_vars.append(errmsg) - if clone_as_out: - eout = errflg.get_prop_value('intent') - if eout != 'out': - subst_dict = {'intent':'out'} - errflg = errflg.clone(subst_dict) - # end if - # end if - err_vars.append(errflg) - # end if - return err_vars - - def add_sub_scope(self, sub_dict): - """Add a child dictionary to enable traversal""" - self.__sub_dicts.append(sub_dict) - - def sub_dictionaries(self): - """Return a list of this dictionary's sub-dictionaries""" - return list(self.__sub_dicts) - - def prop_list(self, prop_name, std_vars=True, loop_vars=True, consts=True): - """Return a list of the <prop_name> property for each variable. - std_vars are variables which are neither constants nor loop variables. - """ - plist = list() - for var in self.values(): - if self.include_var_in_list(var, std_vars=std_vars, - loop_vars=loop_vars, consts=consts): - plist.append(var.get_prop_value(prop_name)) - # end if - # end for - return plist - - def declare_variables(self, outfile, indent, dummy=False, - std_vars=True, loop_vars=True, consts=True): - """Write out the declarations for this dictionary's variables""" - for standard_name in self.keys(): - var = self.find_variable(standard_name=standard_name, - any_scope=False) - if self.include_var_in_list(var, std_vars=std_vars, - loop_vars=loop_vars, consts=consts): - self[standard_name].write_def(outfile, indent, self, - dummy=dummy) - # end if - # end for - - def merge(self, other_dict, run_env): - """Add new entries from <other_dict>""" - for ovar in other_dict.variable_list(): - self.add_variable(ovar, run_env) - # end for - - @staticmethod - def loop_var_okay(standard_name, is_run_phase): - """If <standard_name> is a loop variable, return True only if it - is appropriate for the phase (e.g., horizontal_loop_extent is okay - during a run phase only while horizontal_dimension is not allowed - during a run phase). - If <standard_name> is not a loop variable, return True""" - if (standard_name in CCPP_LOOP_VAR_STDNAMES) and (not is_run_phase): - # Prohibit looking for loop variables except in run phases - retval = False - elif (standard_name == "horizontal_dimension") and is_run_phase: - # horizontal_dimension should not be used in run phase - retval = False - else: - retval = True - # end if - return retval - - def __str__(self): - """Return a string that represents this dictionary object""" - return f"VarDictionary({self.name}, {list(self.keys())})" - - def __repr__(self): - """Return an unique representation for this object""" - srepr = super().__repr__() - vstart = len("VarDictionary") + 1 - if len(srepr) > vstart + 1: - comma = ", " - else: - comma = "" - # end if - return f"VarDictionary({self.name}{comma}{srepr[vstart:]}" - - def __del__(self): - """Attempt to delete all of the variables in this dictionary""" - self.clear() - - def __eq__(self, other): - """Override == to restore object equality, not dictionary - list equality""" - return self is other - - @classmethod - def loop_var_match(cls, standard_name): - """Return a VarLoopSubst if <standard_name> is a loop variable, - otherwise, return None""" - # Strip off 'ccpp_constant_one:', if present - if standard_name[0:18] == 'ccpp_constant_one:': - beg = 18 - else: - beg = 0 - # end if - if standard_name[beg:] in CCPP_VAR_LOOP_SUBSTS: - vmatch = CCPP_VAR_LOOP_SUBSTS[standard_name[beg:]] - else: - vmatch = None - # end if - return vmatch - - def find_loop_dim_match(self, dim_string): - """Find a match in local dict for <dim_string>. That is, if - <dim_string> has a loop dim substitution, and each standard name - in that substitution is in self, return the equivalent local - name string.""" - ldim_string = None - if dim_string in CCPP_LOOP_DIM_SUBSTS: - lnames = list() - std_subst = CCPP_LOOP_DIM_SUBSTS[dim_string].split(':') - for ssubst in std_subst: - svar = self.find_variable(standard_name=ssubst, any_scope=False) - if svar is not None: - lnames.append(svar.call_string(self)) - else: - break - # end if - # end for - if len(lnames) == len(std_subst): - ldim_string = ':'.join(lnames) - # end if - # end if - return ldim_string - - @classmethod - def find_loop_dim_from_index(cls, index_string): - """Given a loop index standard name, find the related loop dimension. - """ - loop_dim_string = None - for dim_string in CCPP_LOOP_DIM_SUBSTS: - if index_string == CCPP_LOOP_DIM_SUBSTS[dim_string]: - loop_dim_string = dim_string - break - # end if - # end for - return loop_dim_string - - def find_loop_subst(self, standard_name, any_scope=True, context=None): - """If <standard_name> is of the form <standard_name>_extent and that - variable is not in the dictionary, substitute a tuple of variables, - (<standard_name>_begin, <standard_name>_end), if those variables are - in the dictionary. - If <standard_name>_extent *is* present, return that variable as a - range, ('ccpp_constant_one', <standard_name>_extent) - In other cases, return None - """ - loop_var = VarDictionary.loop_var_match(standard_name) - logger_str = None - if loop_var is not None: - # Let us see if we can fix a loop variable - dict_var = self.find_variable(standard_name=standard_name, - any_scope=any_scope) - if dict_var is not None: - var_one = CCPP_CONSTANT_VARS['ccpp_constant_one'] - my_var = (var_one, dict_var) - if self.__run_env.logger is not None: - lstr = "loop_subst: found {}{}" - logger_str = lstr.format(standard_name, - context_string(context)) - # end if - else: - my_vars = [self.find_variable(standard_name=x, - any_scope=any_scope) - for x in loop_var] - if None not in my_vars: - my_var = tuple(my_vars) - if self.__run_env.logger is not None: - names = [x.get_prop_value('local_name') - for x in my_vars] - lstr = "loop_subst: {} ==> ({}){}" - logger_str = lstr.format(standard_name, - ', '.join(names), - context_string(context)) - # end if - else: - if self.__run_env.logger is not None: - lstr = "loop_subst: {} ==> (??) FAILED{}" - logger_str = lstr.format(standard_name, - context_string(context)) - # end if - my_var = None - # end if - # end if - else: - if self.__run_env.logger is not None: - lstr = "loop_subst: {} is not a loop variable{}" - logger_str = lstr.format(standard_name, - context_string(context)) - # end if - my_var = None - # end if - if logger_str is not None: - self.__run_env.logger.debug(logger_str) - # end if - return my_var - - def var_call_string(self, var, loop_vars=None): - """Construct the actual argument string for <var> by translating - standard names to local names. String includes array bounds. - if <loop_vars> is present, look there first for array bounds, - even if usage requires a loop substitution. - """ - return var.call_string(self, loop_vars=loop_vars) - - def new_internal_variable_name(self, prefix=None, max_len=63): - """Find a new local variable name for this dictionary. - The new name begins with <prefix>_<self.name> or with <self.name> - (where <self.name> is this VarDictionary's name) if <prefix> is None. - The new variable name is kept to a maximum length of <max_len>. - """ - index = 0 - if prefix is None: - var_prefix = '{}_var'.format(self.name) - else: - var_prefix = '{}'.format(prefix) - # end if - varlist = [x for x in self.__local_names.keys() if var_prefix in x] - newvar = None - while newvar is None: - if index == 0: - newvar = var_prefix - else: - newvar = '{}{}'.format(var_prefix, index) - # end if - index = index + 1 - if len(newvar) > max_len: - var_prefix = var_prefix[:-1] - newvar = None - elif newvar in varlist: - newvar = None - # end if - # end while - return newvar - -############################################################################### - -# List of constant variables which are universally available -CCPP_CONSTANT_VARS = \ - VarDictionary('CCPP_CONSTANT_VARS', _MVAR_DUMMY_RUN_ENV, - variables=[ccpp_standard_var('ccpp_constant_one', 'module', - _MVAR_DUMMY_RUN_ENV)]) - -############################################################################### diff --git a/scripts/mkcap.py b/scripts/mkcap.py deleted file mode 100755 index c5e88362..00000000 --- a/scripts/mkcap.py +++ /dev/null @@ -1,831 +0,0 @@ -#!/usr/bin/env python3 -# -# Script to generate a cap module and subroutines -# from a scheme xml file. -# - -import copy -import logging -import os -import sys -import getopt -import xml.etree.ElementTree as ET - -from common import CCPP_INTERNAL_VARIABLES -from common import STANDARD_VARIABLE_TYPES, STANDARD_CHARACTER_TYPE -from common import isstring, string_to_python_identifier -from conversion_tools import unit_conversion - -############################################################################### - -class Var(object): - - def __init__(self, **kwargs): - self._standard_name = None - self._long_name = None - self._units = None - self._local_name = None - self._type = None - self._dimensions = [] - self._container = None - self._kind = None - self._intent = None - self._active = None - self._optional = None - self._pointer = False - self._target = None - self._actions = { 'in' : None, 'out' : None } - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - @property - def standard_name(self): - '''Get the name of the variable.''' - return self._standard_name - - @standard_name.setter - def standard_name(self, value): - self._standard_name = value - - @property - def long_name(self): - '''Get the name of the variable.''' - return self._long_name - - @long_name.setter - def long_name(self, value): - self._long_name = value - - @property - def units(self): - '''Get the units of the variable.''' - return self._units - - @units.setter - def units(self, value): - self._units = value - - @property - def local_name(self): - '''Get the local variable name of the variable.''' - return self._local_name - - @local_name.setter - def local_name(self, value): - self._local_name = value - - @property - def type(self): - '''Get the type of the variable.''' - return self._type - - @type.setter - def type(self, value): - self._type = value - - @property - def dimensions(self): - '''Get the dimensions of the variable.''' - return self._dimensions - - @dimensions.setter - def dimensions(self, value): - if not isinstance(value, list): - raise TypeError('Invalid type for variable property dimensions, must be a list') - self._dimensions = value - - @property - def rank(self): - '''Get the rank of the variable. Originally, this was an integer indicating - the number of dimensions (therefore the name), now it is a list of colons to use - for assumed-size array definitions in Fortran.''' - if len(self._dimensions) == 0: - return '' - else: - return '('+ ','.join([':'] * len(self._dimensions)) +')' - - @property - def kind(self): - '''Get the kind of the variable.''' - return self._kind - - @kind.setter - def kind(self, value): - self._kind = value - - @property - def intent(self): - '''Get the intent of the variable.''' - return self._intent - - @intent.setter - def intent(self, value): - if not value in ['none', 'in', 'out', 'inout']: - raise ValueError('Invalid value {0} for variable property intent'.format(value)) - self._intent = value - - @property - def active(self): - '''Get the active attribute of the variable.''' - return self._active - - @active.setter - def active(self, value): - if not isinstance(value, str): - raise ValueError('Invalid value {0} for variable property active, must be a string'.format(value)) - self._active = value - - @property - def optional(self): - '''Get the optional attribute of the variable.''' - return self._optional - - @optional.setter - def optional(self, value): - if not isinstance(value, str): - raise ValueError('Invalid value {0} for variable property optional, must be a string'.format(value)) - self._optional = value - - # Pointer is not set by parsing metadata attributes, but by mkstatic. - # This is a quick and dirty solution! - @property - def pointer(self): - '''Get the pointer attribute of the variable.''' - return self._pointer - - @pointer.setter - def pointer(self, value): - if not isinstance(value, bool): - raise ValueError('Invalid value {0} for variable property pointer, must be a logical'.format(value)) - self._pointer = value - - @property - def target(self): - '''Get the target of the variable.''' - return self._target - - @target.setter - def target(self, value): - self._target = value - - @property - def container(self): - '''Get the container of the variable.''' - return self._container - - @container.setter - def container(self, value): - self._container = value - - @property - def actions(self): - '''Get the action strings for the variable.''' - return self._actions - - @actions.setter - def actions(self, values): - if isinstance(values, dict): - for key in values.keys(): - if key in ['in', 'out'] and isstring(values[key]): - self._actions[key] = values[key] - else: - raise Exception('Invalid values for variable attribute actions.') - else: - raise Exception('Invalid values for variable attribute actions.') - - def compatible(self, other): - """Test if the variable is compatible another variable. This requires - that certain variable attributes are identical. Others, for example - len=... for character variables have less strict requirements: accept - character(len=*) as compatible with character(len=INTEGER_VALUE). - We defer testing units here and catch incompatible units later when - unit-conversion code is autogenerated.""" - if self.type == 'character': - if (self.kind == 'len=*' and other.kind.startswith('len=')) or \ - (self.kind.startswith('len=') and other.kind == 'len=*'): - return self.standard_name == other.standard_name \ - and self.type == other.type \ - and self.rank == other.rank - return self.standard_name == other.standard_name \ - and self.type == other.type \ - and self.kind == other.kind \ - and self.rank == other.rank - - def convert_to(self, units): - """Generate action to convert data in the variable's units to other units""" - function_name = '{0}__to__{1}'.format(string_to_python_identifier(self.units), string_to_python_identifier(units)) - try: - function = getattr(unit_conversion, function_name) - logging.info('Automatic unit conversion from {0} to {1} for {2} after returning from {3}'.format(self.units, units, self.standard_name, self.container)) - except AttributeError: - raise Exception('Error, automatic unit conversion from {0} to {1} for {2} in {3} not implemented'.format(self.units, units, self.standard_name, self.container)) - conversion = function() - self._actions['out'] = function() - - def convert_from(self, units): - """Generate action to convert data in other units to the variable's units""" - function_name = '{1}__to__{0}'.format(string_to_python_identifier(self.units), string_to_python_identifier(units)) - try: - function = getattr(unit_conversion, function_name) - logging.info('Automatic unit conversion from {0} to {1} for {2} before entering {3}'.format(self.units, units, self.standard_name, self.container)) - except AttributeError: - raise Exception('Error, automatic unit conversion from {1} to {0} for {2} in {3} not implemented'.format(self.units, units, self.standard_name, self.container)) - conversion = function() - self._actions['in'] = function() - - def dimstring_local_names(self, metadata, assume_shape = False): - '''Create the dimension string for assumed shape or explicit arrays - in Fortran. Requires a metadata dictionary to resolve the dimensions, - which are in CCPP standard names, to local variable names. If the - optional argument assume_shape is True, return an assumed shape - dimension string with the upper bound being left undefined.''' - # Simplest case: scalars - if len(self.dimensions) == 0: - return '' - dimstring = [] - # Arrays - for dim in self.dimensions: - # Handle dimensions like "A:B", "A:3", "-1:Z" - if ':' in dim: - dims = [ x.lower() for x in dim.split(':')] - try: - dim0 = int(dims[0]) - dim0 = dims[0] - except ValueError: - if not dims[0].lower() in metadata.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in metadata'.format( - dims[0].lower(), self.standard_name)) - dim0 = metadata[dims[0].lower()][0].local_name - try: - dim1 = int(dims[1]) - dim1 = dims[1] - except ValueError: - if not dims[1].lower() in metadata.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in metadata'.format( - dims[1].lower(), self.standard_name)) - dim1 = metadata[dims[1].lower()][0].local_name - # Single dimensions - else: - dim0 = 1 - try: - dim1 = int(dim) - dim1 = dim - except ValueError: - if not dim.lower() in metadata.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in metadata'.format( - dim.lower(), self.standard_name)) - dim1 = metadata[dim.lower()][0].local_name - if assume_shape: - dimstring.append('{}:'.format(dim0)) - else: - dimstring.append('{}:{}'.format(dim0, dim1)) - return '({})'.format(','.join(dimstring)) - - def print_module_use(self): - '''Print the module use line for the variable.''' - for item in self.container.split(' '): - if item.startswith('MODULE_'): - module = item.replace('MODULE_', '') - break - str = 'use {module}, only: {varname}'.format(module=module,varname=self.local_name) - return str - - def print_def_intent(self, metadata): - '''Print the definition line for the variable, using intent. Use the metadata - dictionary to resolve lower bounds for array dimensions.''' - # Resolve dimensisons to local names using undefined upper bounds (assumed shape) - dimstring = self.dimstring_local_names(metadata, assume_shape = True) - # It is an error for host model variables to have the optional attribute in the metadata - if self.optional == 'T': - error_message = "This routine should only be called for host model variables" + \ - " that cannot have the optional metadata attribute, but got self.optional=T" - raise Exception(error_message) - # If the host variable is potentially unallocated, add optional and target to variable declaration - elif not self.active == 'T': - optional = ', optional, target' - else: - # Always declare as target variable so that locally defined pointers can point to it - optional = ', target' - # - if self.type in STANDARD_VARIABLE_TYPES: - if self.kind: - str = "{s.type}({s._kind}), intent({s.intent}){optional} :: {s.local_name}{dimstring}" - else: - str = "{s.type}, intent({s.intent}){optional} :: {s.local_name}{dimstring}" - else: - if self.kind: - error_message = "Generating variable definition statements for derived types with" + \ - " kind attributes not implemented; variable: {0}".format(self.standard_name) - raise Exception(error_message) - else: - str = "type({s.type}), intent({s.intent}){optional} :: {s.local_name}{dimstring}" - return str.format(s=self, optional=optional, dimstring=dimstring) - - def print_def_local(self, metadata): - '''Print the definition line for the variable, assuming it is a local variable.''' - # It is an error for local variables to have the active attribute - if not self.active == 'T': - error_message = "This routine should only be called for local variables" + \ - " that cannot have an active attribute other than the" +\ - " default T, but got self.active=T" - raise Exception(error_message) - - # If it is a pointer, everything is different! - if self.pointer: - if self.type in STANDARD_VARIABLE_TYPES: - if self.kind: - if self.rank: - str = "{s.type}({s._kind}), dimension{s.rank}, pointer :: p => null()" - else: - str = "{s.type}({s._kind}), pointer :: p => null()" - else: - if self.rank: - str = "{s.type}, dimension{s.rank}, pointer :: p => null()" - else: - str = "{s.type}, pointer :: p => null()" - else: - if self.kind: - error_message = "Generating variable definition statements for derived types with" + \ - " kind attributes not implemented; variable: {0}".format(self.standard_name) - raise Exception(error_message) - else: - if self.rank: - str = "type({s.type}), dimension{s.rank}, pointer :: p => null()" - else: - str = "type({s.type}), pointer :: p => null()" - return str.format(s=self) - else: - # DH* 20241022 WORKAROUND TO ACCOUNT FOR MISSING UPDATES TO CCPP PHYSICS - # W.R.T. DECLARING OPTIONAL VARIABLES IN METADATA AND CODE. ALWAYS USE TARGET - ## If the host variable is potentially unallocated, the active attribute is - ## also set accordingly for the local variable; add target to variable declaration - #if self.optional == 'T': - # target = ', target' - #else: - # target = '' - target = ', target' - # *DH - if self.type in STANDARD_VARIABLE_TYPES: - if self.kind: - if self.rank: - str = "{s.type}({s._kind}), dimension{s.rank}, allocatable{target} :: {s.local_name}" - else: - str = "{s.type}({s._kind}){target} :: {s.local_name}" - else: - if self.rank: - str = "{s.type}, dimension{s.rank}, allocatable{target} :: {s.local_name}" - else: - str = "{s.type}{target} :: {s.local_name}" - else: - if self.kind: - error_message = "Generating variable definition statements for derived types with" + \ - " kind attributes not implemented; variable: {0}".format(self.standard_name) - raise Exception(error_message) - else: - if self.rank: - str = "type({s.type}), dimension{s.rank}, allocatable{target} :: {s.local_name}" - else: - str = "type({s.type}){target} :: {s.local_name}" - return str.format(s=self, target=target) - - def print_debug(self): - '''Print the data retrieval line for the variable.''' - # Scheme variables don't have the active attribute - if 'SCHEME' in self.container: - str='''Contents of {s} (* = mandatory for compatibility): - standard_name = {s.standard_name} * - long_name = {s.long_name} - units = {s.units} * - local_name = {s.local_name} - type = {s.type} * - dimensions = {s.dimensions} - rank = {s.rank} * - kind = {s.kind} * - intent = {s.intent} - optional = {s.optional} - target = {s.target} - container = {s.container} - actions = {s.actions}''' - # Host model variables don't have the optional attribute - else: - str='''Contents of {s} (* = mandatory for compatibility): - standard_name = {s.standard_name} * - long_name = {s.long_name} - units = {s.units} * - local_name = {s.local_name} - type = {s.type} * - dimensions = {s.dimensions} - rank = {s.rank} * - kind = {s.kind} * - intent = {s.intent} - active = {s.active} - target = {s.target} - container = {s.container} - actions = {s.actions}''' - return str.format(s=self) - -class CapsMakefile(object): - - header=''' -# All CCPP caps are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -CAPS_F90 =''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, caps): - if (self.filename is not sys.stdout): - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for cap in caps: - contents += ' \\\n\t {0}'.format(cap) - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class CapsCMakefile(object): - - header=''' -# All CCPP caps are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -set(CAPS -''' - footer=''') -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, caps): - if (self.filename is not sys.stdout): - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for cap in caps: - contents += ' {0}\n'.format(cap) - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class CapsSourcefile(object): - - header=''' -# All CCPP caps are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -export CCPP_CAPS="''' - footer='''" -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, caps): - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for cap in caps: - contents += '{0};'.format(cap) - contents = contents.rstrip(';') - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class SchemesMakefile(object): - - header=''' -# All CCPP schemes are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -SCHEMES_F = - -SCHEMES_F90 = - -SCHEMES_f = - -SCHEMES_f90 =''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, schemes): - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - schemes_F = 'SCHEMES_F =' - schemes_F90 = 'SCHEMES_F90 =' - schemes_f = 'SCHEMES_f =' - schemes_f90 = 'SCHEMES_f90 =' - for scheme in schemes: - if scheme.endswith('.F'): - schemes_F += ' \\\n\t {0}'.format(scheme) - elif scheme.endswith('.F90'): - schemes_F90 += ' \\\n\t {0}'.format(scheme) - elif scheme.endswith('.f'): - schemes_f += ' \\\n\t {0}'.format(scheme) - elif scheme.endswith('.f90'): - schemes_f90 += ' \\\n\t {0}'.format(scheme) - contents = contents.replace('SCHEMES_F =', schemes_F) - contents = contents.replace('SCHEMES_F90 =', schemes_F90) - contents = contents.replace('SCHEMES_f =', schemes_f) - contents = contents.replace('SCHEMES_f90 =', schemes_f90) - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class SchemesCMakefile(object): - - header=''' -# All CCPP schemes are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -set(SCHEMES -''' - footer=''') -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, schemes): - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for scheme in schemes: - contents += ' {0}\n'.format(scheme) - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class SchemesSourcefile(object): - - header=''' -# All CCPP schemes are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -export CCPP_SCHEMES="''' - footer='''" -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, schemes): - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for scheme in schemes: - contents += '{0};'.format(scheme) - contents = contents.rstrip(';') - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class TypedefsMakefile(object): - - header=''' -# All CCPP types are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -TYPEDEFS =''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, typedefs): - if (self.filename is not sys.stdout): - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for typedef in typedefs: - contents += ' \\\n\t {0}'.format(typedef) - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class TypedefsCMakefile(object): - - header=''' -# All CCPP types are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -set(TYPEDEFS -''' - footer=''') -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, typedefs): - if (self.filename is not sys.stdout): - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for typedef in typedefs: - contents += ' {0}\n'.format(typedef) - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -class TypedefsSourcefile(object): - - header=''' -# All CCPP types are defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -export CCPP_TYPEDEFS="''' - footer='''" -''' - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, typedefs): - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - f = open(self.filename, 'w') - else: - f = sys.stdout - - contents = self.header - for typedef in typedefs: - contents += '{0};'.format(typedef) - contents = contents.rstrip(';') - contents += self.footer - f.write(contents) - - if (f is not sys.stdout): - f.close() - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - -############################################################################### -if __name__ == "__main__": - main() diff --git a/scripts/mkdoc.py b/scripts/mkdoc.py deleted file mode 100755 index 018f88f6..00000000 --- a/scripts/mkdoc.py +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env python3 - -# -# Functions to generate basic documentation in HTML and LaTeX for CCPP metadata -# - -# DH* TODO: create a Python module metadata.py with a class Metadata -# and use this for ccpp_prebuild.py; create to_html and to_latex routines for it - -import logging -import os - -from common import decode_container, escape_tex - -############################################################################### - -def metadata_to_html(metadata, model, filename): - """Create an HTML page with a table that lists each variable provided - by the model. Contrary to metadata_to_latex below, this table does not - include information on variables requested by schemes. The primary use - of the HTML table is to help physics scheme developers to identify the - variables they need when writing a CCPP-compliant scheme.""" - - shading = { 0 : 'darkgray', 1 : 'lightgray' } - success = True - - # Header - html = '''<html> -<title>CCPP variables provided by model {model} - -

CCPP variables provided by model {model}

- - - - - - - - - - - -'''.format(model=model, bgcolor = shading[0]) - - count = 0 - for var_name in sorted(metadata.keys()): - for var in metadata[var_name]: - # Alternate shading, count is 0 1 0 1 ... - count = (count+1) % 2 - # ... create html row ... - line = ''' - - - - - - - - -'''.format(v=var, rank=var.rank.count(':'), container = decode_container(var.container), bgcolor=shading[count]) - html += line - - # Footer - html += '''
standard_namelong_name units rank type kind source {model} name
{v.standard_name}{v.long_name} {v.units} {rank} {v.type} {v.kind} {container} {v.local_name}
- - -''' - - filepath = os.path.split(os.path.abspath(filename))[0] - if not os.path.isdir(filepath): - os.makedirs(filepath) - with open(filename, 'w') as f: - f.write(html) - - logging.info('Metadata table for model {0} written to {1}'.format(model, filename)) - return success - - -def metadata_to_latex(metadata_define, metadata_request, model, filename): - """Create a LaTeX document with a table that lists each variable provided - and/or requested. Uses the GMTB LaTeX templates and style definitons in gmtb.sty.""" - - shading = { 0 : 'darkgray', 1 : 'lightgray' } - success = True - - var_names = sorted(list(set(list(metadata_define.keys()) + list(metadata_request.keys())))) - - styledir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), - '../doc/DevelopersGuide')) - - latex = '''\\documentclass[12pt,letterpaper,oneside,landscape]{{scrbook}} - -\\usepackage{{import}} -\\import{{{styledir}/}}{{gmtb.sty}} -\\renewcommand{{\\thesection}}{{\\arabic{{section}}}} -\\renewcommand{{\\thesubsection}}{{\\arabic{{section}}.\\arabic{{subsection}}}} - -\\begin{{document}} - -\\section{{CCPP variables provided by model {model} vs requested by pool of physics}}\\label{{sec_ccpp_variables}} -\\subsection{{List of variables}} -\\begin{{longtable}}{{l}}'''.format(model=model, styledir=styledir) - - for var_name in var_names: - if var_name in metadata_define.keys(): - var = metadata_define[var_name][0] - else: - var = metadata_request[var_name][0] - line = ''' -\hyperlink{{{standard_name_ref}}}{{\\blue\\underline{{\\execout{{{standard_name}}}}}}} \\\\'''.format( - standard_name=escape_tex(var.standard_name), standard_name_ref=var.standard_name) - latex += line - - latex += ''' -\\end{longtable}\\pagebreak -\\subsection{Description of variables} -{{\\small\\begin{description} -''' - - for var_name in var_names: - if var_name in metadata_define.keys(): - var = metadata_define[var_name][0] - target = escape_tex(decode_container(var.container)) - local_name = escape_tex(var.local_name) - else: - var = metadata_request[var_name][0] - target = 'MISSING' - local_name = 'MISSING' - if var_name in metadata_request.keys(): - requested_list = [ escape_tex(decode_container(v.container)) if v.container else 'none' for v in metadata_request[var_name] ] - # for the purpose of the table, just output the name of the subroutine - for i in range(len(requested_list)): - entry = requested_list[i] - requested_list[i] = entry[entry.find('SUBROUTINE')+len('SUBROUTINE')+1:] - requested = '\\newline '.join(sorted(requested_list)) - else: - requested = 'NOT REQUESTED' - - # Create output - text = ''' -\\begin{{samepage}}\\item{{ -\hypertarget{{{standard_name_ref}}}{{\\blue\\exec{{{standard_name}}}}}}}\\\\ \\nopagebreak -\\begin{{tabular}}{{ll}} -\\execout{{long\_name }} & \\execout{{{long_name} }} \\\\ -\\execout{{units }} & \\execout{{{units} }} \\\\ -\\execout{{rank }} & \\execout{{{rank} }} \\\\ -\\execout{{type }} & \\execout{{{type} }} \\\\ -\\execout{{kind }} & \\execout{{{kind} }} \\\\ -\\execout{{source }} & \\execout{{{target} }} \\\\ -\\execout{{local\_name}} & \\execout{{{local_name} }} \\\\ -\\execout{{requested }} & \\execout{{\\vtop{{{requested}}}}} \\\\ -\\end{{tabular}} -\\vspace{{4pt}} -\\end{{samepage}}'''.format(standard_name=escape_tex(var.standard_name), standard_name_ref=var.standard_name, - long_name=escape_tex(var.long_name), - units=escape_tex(var.units), - rank=var.rank.count(':'), - type=escape_tex(var.type), - kind=escape_tex(var.kind), - target=target, - local_name=local_name, - requested=requested) - latex += text - # Footer - latex += ''' -\\end{description}}} -\\end{document} -''' - - filepath = os.path.split(os.path.abspath(filename))[0] - if not os.path.isdir(filepath): - os.makedirs(filepath) - with open(filename, 'w') as f: - f.write(latex) - - logging.info('Metadata table for model {0} written to {1}'.format(model, filename)) - return success diff --git a/scripts/mkstatic.py b/scripts/mkstatic.py deleted file mode 100755 index e33164a3..00000000 --- a/scripts/mkstatic.py +++ /dev/null @@ -1,2145 +0,0 @@ -#!/usr/bin/env python3 -# - -import collections -import copy -import getopt -import filecmp -import logging -import os -import re -import sys -import types -import xml.etree.ElementTree as ET - -from common import encode_container -from common import lowercase_keys, lowercase_xml -from common import CCPP_STAGES -from common import CCPP_T_INSTANCE_VARIABLE, CCPP_ERROR_CODE_VARIABLE, CCPP_ERROR_MSG_VARIABLE, CCPP_LOOP_COUNTER, CCPP_LOOP_EXTENT -from common import CCPP_BLOCK_NUMBER, CCPP_BLOCK_COUNT, CCPP_BLOCK_SIZES, CCPP_THREAD_NUMBER, CCPP_THREAD_COUNT, CCPP_INTERNAL_VARIABLES -from common import CCPP_HORIZONTAL_LOOP_BEGIN, CCPP_HORIZONTAL_LOOP_END, CCPP_CHUNK_EXTENT -from common import CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_DIMENSION, CCPP_HORIZONTAL_LOOP_EXTENT, CCPP_NUM_INSTANCES -from common import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX -from common import CCPP_TYPE, STANDARD_VARIABLE_TYPES, STANDARD_CHARACTER_TYPE -from common import CCPP_STATIC_API_MODULE, CCPP_STATIC_SUBROUTINE_NAME -from metadata_parser import CCPP_MANDATORY_VARIABLES -from mkcap import Var - -############################################################################### - -# Limit suite names to 37 characters; this keeps cap names below 64 characters -# Cap names of 64 characters or longer can cause issues with some compilers. -SUITE_NAME_MAX_CHARS = 37 - -# Maximum number of dimensions of an array allowed by the Fortran 2008 standard -FORTRAN_ARRAY_MAX_DIMS = 15 - -# These variables always need to be present for creating suite and group caps -CCPP_SUITE_VARIABLES = { **CCPP_MANDATORY_VARIABLES, - CCPP_LOOP_COUNTER : Var(local_name = 'loop_cnt', - standard_name = CCPP_LOOP_COUNTER, - long_name = 'loop counter for subcycling loops in CCPP', - units = 'index', - type = 'integer', - dimensions = [], - rank = '', - kind = '', - intent = 'in', - active = 'T', - ), - CCPP_LOOP_EXTENT : Var(local_name = 'loop_max', - standard_name = CCPP_LOOP_EXTENT, - long_name = 'loop counter for subcycling loops in CCPP', - units = 'count', - type = 'integer', - dimensions = [], - rank = '', - kind = '', - intent = 'in', - active = 'T', - ), - } - -# Type and variable declarations for arrays of pointers, required for optional/inactive variables -TMPPTR_ARR_TYPE_DECLARATION = '''type :: {pointer_type_name} - {tmpptr_def} - end type {pointer_type_name}''' -TMPPTR_ARR_DECLARATION = '''type({pointer_type_name}), dimension({dims}) :: {localname}_array''' - -############################################################################### - -def extract_parents_and_indices_from_local_name(local_name): - """Break apart local_name into the different components (members of DDTs) - to determine all variables that are required; this must work for complex - constructs such as Atm(mytile)%q(:,:,:,Atm2(mytile2)%graupel), with - result parent = 'Atm', indices = [mytile, Atm2, mytile2]""" - # First, extract all variables/indices in parentheses (used for subsetting) - indices = [] - while '(' in local_name: - for i in range(len(local_name)): - if local_name[i] == '(': - last_open = i - elif local_name[i] == ')': - last_closed = i - break - index_set = local_name[last_open+1:last_closed].split(',') - for index_group in index_set: - for index in index_group.split(':'): - if index: - if '%' in index: - indices.append(index[:index.find('%')]) - else: - # Skip hard-coded integers that are not variables - try: - int(index) - except ValueError: - indices.append(index) - # Remove this innermost index group (...) from local_name - local_name = local_name.replace(local_name[last_open:last_closed+1], '') - # Remove duplicates from indices - indices = list(set(indices)) - # Derive parent of actual variable (now that all subsets have been processed) - if '%' in local_name: - parent = local_name[:local_name.find('%')] - else: - parent = local_name - # Remove whitespaces - parent = parent.strip() - indices = [ x.strip() for x in indices ] - return (parent, indices) - -def extract_dimensions_from_local_name(local_name): - """Extract the dimensions from a local_name. - Throw away any parent information.""" - # First, find delimiter '%' between parent(s) and child '%' - parent_delimiter_index = -1 - if '%' in local_name: - i = len(local_name)-1 - opened = 0 - while i >= 0: - if local_name[i] == ')': - opened += 1 - elif local_name[i] == '(': - opened -= 1 - elif local_name[i] == '%' and opened == 0: - parent_delimiter_index = i - break - i -= 1 - if '(' in local_name[parent_delimiter_index+1:]: - dim_string_start = local_name[parent_delimiter_index+1:].find('(') - dim_string_end = local_name[parent_delimiter_index+1:].rfind(')') - dim_string = local_name[parent_delimiter_index+1:][dim_string_start:dim_string_end+1] - # Now that we have a dim_string, find all dimensions in this string; - # ignore outermost opening and closing parentheses. - opened = 0 - dim = '' - i = 1 - dimensions = [] - while i <= len(dim_string)-1: - if dim_string[i] == ',' and opened == 0: - dimensions.append(dim) - dim = '' - elif i == len(dim_string)-1 and dim: - dimensions.append(dim) - else: - dim += dim_string[i] - i += 1 - else: - dimensions = [] - dim_string = '' - return (dimensions, dim_string) - -def create_argument_list_wrapped(arguments): - """Create a wrapped argument list, remove trailing ',' """ - argument_list = '' - length = 0 - for argument in arguments: - argument_list += argument + ',' - length += len(argument)+1 - # Split args so that lines don't exceed 260 characters (for PGI) - if length > 70 and not argument == arguments[-1]: - argument_list += ' &\n ' - length = 0 - if argument_list: - argument_list = argument_list.rstrip(',') - return argument_list - -def create_argument_list_wrapped_explicit(arguments, additional_vars_following = False): - """Create a wrapped argument list with explicit arguments x=y. If no additional - variables are added (additional_vars_following == False), remove trailing ',' """ - argument_list = '' - length = 0 - for argument in arguments: - argument_list += argument + '=' + argument + ',' - length += 2*len(argument)+2 - # Split args so that lines don't exceed 260 characters (for PGI) - if length > 70 and not argument == arguments[-1]: - argument_list += ' &\n ' - length = 0 - if argument_list and not additional_vars_following: - argument_list = argument_list.rstrip(',') - return argument_list - -def create_arguments_module_use_var_defs(variable_dictionary, metadata_define, tmpvars = None, tmpptrs = None): - """Given a dictionary of standard names and variables, and a metadata - dictionary with the variable definitions by the host model, create a list - of arguments (local names), module use statements (for derived data types - and non-standard kinds), and the variable definition statements.""" - arguments = [] - module_use = [] - var_defs = [] - local_kind_and_type_vars = [] - local_pointer_type_defs = [] - - # We need to run through this loop twice. In the first pass, process all scalars. - # In the second pass, process all arrays. This is so that any potential dimension - # that is used in the following array variable definitions is defined first to avoid - # violating the Fortran 2008 standard. - # https://community.intel.com/t5/Intel-Fortran-Compiler/Order-of-declaration-statements-with-and-without-implicit-typing/td-p/1176155 - iteration = 1 - while iteration <= 2: - for standard_name in variable_dictionary.keys(): - if iteration == 1 and variable_dictionary[standard_name].dimensions: - continue - elif iteration == 2 and not variable_dictionary[standard_name].dimensions: - continue - # Add variable local name and variable definitions - arguments.append(variable_dictionary[standard_name].local_name) - var_defs.append(variable_dictionary[standard_name].print_def_intent(metadata_define)) - # Add special kind variables and derived data type definitions to module use statements - if variable_dictionary[standard_name].type in STANDARD_VARIABLE_TYPES and variable_dictionary[standard_name].kind \ - and not variable_dictionary[standard_name].type == STANDARD_CHARACTER_TYPE: - kind_var_standard_name = variable_dictionary[standard_name].kind - if not kind_var_standard_name in local_kind_and_type_vars: - if not kind_var_standard_name in metadata_define.keys(): - raise Exception("Kind {kind}, required by {std_name}, not defined by host model".format( - kind=kind_var_standard_name, std_name=standard_name)) - kind_var = metadata_define[kind_var_standard_name][0] - module_use.append(kind_var.print_module_use()) - local_kind_and_type_vars.append(kind_var_standard_name) - elif not variable_dictionary[standard_name].type in STANDARD_VARIABLE_TYPES: - type_var_standard_name = variable_dictionary[standard_name].type - if not type_var_standard_name in local_kind_and_type_vars: - if not type_var_standard_name in metadata_define.keys(): - raise Exception("Type {type}, required by {std_name}, not defined by host model".format( - type=type_var_standard_name, std_name=standard_name)) - type_var = metadata_define[type_var_standard_name][0] - module_use.append(type_var.print_module_use()) - local_kind_and_type_vars.append(type_var_standard_name) - iteration += 1 - - # Add any local variables (required for unit conversions, array transformations, ...), - # and add any local pointers (required for conditionally allocated arrays) - if tmpvars or tmpptrs: - var_defs.append('') - var_defs.append('! Local variables/pointers for unit conversions, array transformations, ...') - for tmpvar in list(tmpvars) + list(tmpptrs): - # Regular variables - if tmpvar in list(tmpvars): - var_defs.append(tmpvar.print_def_local(metadata_define)) - # Pointers are more complicated - else: - if tmpvar.type == 'character' and 'len=' in tmpvar.kind: - pointer_type_name = f"{tmpvar.type}_{tmpvar.kind.replace('=','')}_r{len(tmpvar.dimensions)}_ptr_arr_type" - elif tmpvar.kind: - pointer_type_name = f"{tmpvar.type}_{tmpvar.kind}_rank{len(tmpvar.dimensions)}_ptr_arr_type" - else: - pointer_type_name = f"{tmpvar.type}_default_kind_rank{len(tmpvar.dimensions)}_ptr_arr_type" - if not pointer_type_name in local_pointer_type_defs: - var_defs.append(TMPPTR_ARR_TYPE_DECLARATION.format(pointer_type_name=pointer_type_name, - tmpptr_def=tmpvar.print_def_local(metadata_define))) - local_pointer_type_defs.append(pointer_type_name) - var_defs.append(TMPPTR_ARR_DECLARATION.format(pointer_type_name=pointer_type_name, - dims=f'1:{CCPP_INTERNAL_VARIABLES[CCPP_THREAD_COUNT]}', localname=tmpvar.local_name)) - # Add special kind variables - if tmpvar.type in STANDARD_VARIABLE_TYPES and tmpvar.kind and not tmpvar.type == STANDARD_CHARACTER_TYPE: - kind_var_standard_name = tmpvar.kind - if not kind_var_standard_name in local_kind_and_type_vars: - if not kind_var_standard_name in metadata_define.keys(): - raise Exception("Kind {kind} not defined by host model".format(kind=kind_var_standard_name)) - kind_var = metadata_define[kind_var_standard_name][0] - module_use.append(kind_var.print_module_use()) - local_kind_and_type_vars.append(kind_var_standard_name) - - return (arguments, module_use, var_defs) - -class API(object): - - header=''' -! -! This work (Common Community Physics Package), identified by NOAA, NCAR, -! CU/CIRES, is free of known copyright restrictions and is placed in the -! public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -! - -!> -!! @brief Auto-generated API for the CCPP static build -!! -! -module {module} - -{module_use} - implicit none - - private - public :: {subroutines} - - contains - - ! Necessary to convert incoming suite and group names to lowercase - function to_lower(str) result(lower) - implicit none - character(len=*), intent(in) :: str - character(len=len(str)) :: lower - integer, parameter :: upper_to_lower = ichar('a') - ichar('A') - integer :: i, ichar_val - - do i = 1, len(str) - ichar_val = ichar(str(i:i)) - if (ichar_val >= ichar('A') .and. ichar_val <= ichar('Z')) then - lower(i:i) = char(ichar_val + upper_to_lower) - else - lower(i:i) = str(i:i) - end if - end do - end function to_lower -''' - - sub = ''' - subroutine {subroutine}({ccpp_var_name}, suite_name, group_name, ierr) - - use ccpp_types, only : ccpp_t - - implicit none - - type(ccpp_t), intent(inout) :: {ccpp_var_name} - character(len=*), intent(in) :: suite_name - character(len=*), optional, intent(in) :: group_name - integer, intent(out) :: ierr - - ierr = 0 - -{suite_switch} - else - - write({ccpp_var_name}%errmsg,'(*(a))') 'Invalid suite ' // to_lower(trim(suite_name)) - ierr = 1 - - end if - - {ccpp_var_name}%errflg = ierr - - end subroutine {subroutine} -''' - - footer = ''' -end module {module} -''' - - def __init__(self, **kwargs): - self._filename = CCPP_STATIC_API_MODULE + '.F90' - self._module = CCPP_STATIC_API_MODULE - self._subroutines = None - self._suites = [] - self._directory = '.' - self._update_api = True - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - @property - def filename(self): - '''Get the filename to write API to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - - @property - def directory(self): - '''Get the directory to write API to.''' - return self._directory - - @directory.setter - def directory(self, value): - self._directory = value - - @property - def update_api(self): - '''Get the update_api flag.''' - return self._update_api - - @update_api.setter - def update_api(self, value): - self._update_api = value - - @property - def module(self): - '''Get the module name of the API.''' - return self._module - - @module.setter - def module(self, value): - self._module = value - - @property - def subroutines(self): - '''Get the subroutines names of the API to.''' - return self._subroutines - - def write(self): - """Write API for static build""" - if not self._suites: - raise Exception("No suites specified for generating API") - suites = self._suites - - # Module use statements for suite and group caps - module_use = '' - for suite in suites: - for subroutine in suite.subroutines: - module_use += ' use {module}, only: {subroutine}\n'.format(module=suite.module, subroutine=subroutine) - for group in suite.groups: - for subroutine in group.subroutines: - module_use += ' use {module}, only: {subroutine}\n'.format(module=group.module, subroutine=subroutine) - - # Add all variables required to module use statements. This is for the API only, - # because the static API imports all variables from modules instead of receiving them - # via the argument list. Special handling for a single variable of type CCPP_TYPE (ccpp_t), - # which comes in as a scalar for any potential block/thread via the argument list. - ccpp_var = None - parent_standard_names = [] - for ccpp_stage in CCPP_STAGES.keys(): - for suite in suites: - for parent_standard_name in suite.parents[ccpp_stage].keys(): - if not parent_standard_name in parent_standard_names: - parent_var = suite.parents[ccpp_stage][parent_standard_name] - # Identify which variable is of type CCPP_TYPE (need local name) - if parent_var.type == CCPP_TYPE: - if ccpp_var and not ccpp_var.local_name==parent_var.local_name: - raise Exception('There can be only one variable of type {0}, found {1} and {2}'.format( - CCPP_TYPE, ccpp_var.local_name, parent_var.local_name)) - ccpp_var = parent_var - continue - module_use += ' {0}\n'.format(parent_var.print_module_use()) - parent_standard_names.append(parent_standard_name) - if not ccpp_var: - raise Exception('No variable of type {0} found - need a scalar instance.'.format(CCPP_TYPE)) - elif not ccpp_var.rank == '': - raise Exception('CCPP variable {0} of type {1} must be a scalar.'.format(ccpp_var.local_name, CCPP_TYPE)) - del parent_standard_names - - # Create a subroutine for each stage - self._subroutines=[] - subs = '' - for ccpp_stage in CCPP_STAGES.keys(): - suite_switch = '' - for suite in suites: - # Calls to groups of schemes for this stage - group_calls = '' - for group in suite.groups: - # The and groups require special treatment, - # since they can only be run in the respective stage (init/finalize) - if (group.init and not ccpp_stage == 'init') or \ - (group.finalize and not ccpp_stage == 'finalize'): - continue - if not group_calls: - clause = 'if' - else: - clause = 'else if' - argument_list_group = create_argument_list_wrapped_explicit(group.arguments[ccpp_stage]) - group_calls += ''' - {clause} (to_lower(trim(group_name))=="{group_name}") then - ierr = {suite_name}_{group_name}_{stage}_cap({arguments})'''.format(clause=clause, - suite_name=group.suite, - group_name=group.name, - stage=CCPP_STAGES[ccpp_stage], - arguments=argument_list_group) - group_calls += ''' - else - write({ccpp_var_name}%errmsg, '(*(a))') 'Group ' // to_lower(trim(group_name)) // ' not found' - ierr = 1 - end if -'''.format(ccpp_var_name=ccpp_var.local_name, group_name=group.name) - - # Call to entire suite for this stage - - # Create argument list for calling the full suite - argument_list_suite = create_argument_list_wrapped_explicit(suite.arguments[ccpp_stage]) - suite_call = ''' - ierr = {suite_name}_{stage}_cap({arguments}) -'''.format(suite_name=suite.name, stage=CCPP_STAGES[ccpp_stage], arguments=argument_list_suite) - - # Add call to all groups of this suite and to the entire suite - if not suite_switch: - clause = 'if' - else: - clause = 'else if' - suite_switch += ''' - {clause} (to_lower(trim(suite_name))=="{suite_name}") then - - if (present(group_name)) then -{group_calls} - else -{suite_call} - end if -'''.format(clause=clause, suite_name=suite.name, group_calls=group_calls, suite_call=suite_call) - - subroutine = CCPP_STATIC_SUBROUTINE_NAME.format(stage=ccpp_stage) - self._subroutines.append(subroutine) - subs += API.sub.format(subroutine=subroutine, - ccpp_var_name=ccpp_var.local_name, - suite_switch=suite_switch) - - # Write output to stdout or file - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - # If the file exists, write to temporary file first and compare them: - # - if identical, delete the temporary file and keep the existing one - # and set the API update flag to false - # - if different, replace existing file with temporary file and set - # the API update flag to true (default value) - # - always replace the file if any of the suite caps has changed - # If the file does not exist, write the API an set the flag to true - if os.path.isfile(self.filename) and \ - not any([suite.update_cap for suite in suites]): - write_to_test_file = True - test_filename = self.filename + '.test' - f = open(test_filename, 'w') - else: - write_to_test_file = False - f = open(self.filename, 'w') - else: - f = sys.stdout - f.write(API.header.format(module=self._module, - module_use=module_use, - subroutines=','.join(self._subroutines))) - f.write(subs) - f.write(Suite.footer.format(module=self._module)) - if (f is not sys.stdout): - f.close() - # See comment above on updating the API or not - if write_to_test_file: - if filecmp.cmp(self.filename, test_filename): - # Files are equal, delete the test API and set update flag to False - os.remove(test_filename) - self.update_api = False - else: - # Files are different, replace existing API with - # the test API and set update flag to True - # Python 3 only: os.replace(test_filename, self.filename) - os.remove(self.filename) - os.rename(test_filename, self.filename) - self.update_api = True - else: - self.update_api = True - return - - def write_includefile(self, source_filename, type): - success = True - filepath = os.path.split(source_filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - # If the file exists, write to temporary file first and compare them: - # - if identical, delete the temporary file and keep the existing one - # - if different, replace existing file with temporary file - # - however, always replace the file if the API update flag is true - if os.path.isfile(source_filename) and not self.update_api: - write_to_test_file = True - test_filename = source_filename + '.test' - f = open(test_filename, 'w') - else: - write_to_test_file = False - f = open(source_filename, 'w') - - if type == 'shell': - # Contents of shell/source file - contents = """# The CCPP static API is defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -export CCPP_STATIC_API=\"{filename}\" -""".format(filename=os.path.abspath(os.path.join(self.directory,self.filename))) - elif type == 'cmake': - # Contents of cmake include file - contents = """# The CCPP static API is defined here. -# -# This file is auto-generated using ccpp_prebuild.py -# at compile time, do not edit manually. -# -set(API \"{filename}\") -""".format(filename=os.path.abspath(os.path.join(self.directory,self.filename))) - else: - logging.error('Encountered unknown type of file "{type}" when writing include file for static API'.format(type=type)) - success = False - return - - f.write(contents) - f.close() - # See comment above on updating the API or not - if write_to_test_file: - if filecmp.cmp(source_filename, test_filename): - # Files are equal, delete the test file - os.remove(test_filename) - else: - # Files are different, replace existing file - # Python 3 only: os.replace(test_filename, source_filename) - os.remove(source_filename) - os.rename(test_filename, source_filename) - return success - - -class Suite(object): - - header=''' -! -! This work (Common Community Physics Package), identified by NOAA, NCAR, -! CU/CIRES, is free of known copyright restrictions and is placed in the -! public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -! - -!> -!! @brief Auto-generated cap module for the CCPP suite -!! -! -module {module} - -{module_use} - - implicit none - - private - public :: {subroutines} - - contains -''' - - sub = ''' - function {subroutine}({arguments}) result(ierr) - - {module_use} - - implicit none - - integer :: ierr - {var_defs} - - ierr = 0 - -{body} - - end function {subroutine} -''' - - footer = ''' -end module {module} -''' - - def __init__(self, **kwargs): - self._name = None - self._filename = sys.stdout - self._sdf_name = None - self._all_schemes_called = None - self._all_subroutines_called = None - self._call_tree = {} - self._caps = None - self._module = None - self._subroutines = None - self._parents = { ccpp_stage : collections.OrderedDict() for ccpp_stage in CCPP_STAGES.keys() } - self._arguments = { ccpp_stage : [] for ccpp_stage in CCPP_STAGES.keys() } - self._update_cap = True - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - @property - def name(self): - '''Get the name of the suite.''' - return self._name - - @property - def sdf_name(self): - '''Get the name of the suite definition file.''' - return self._sdf_name - - @sdf_name.setter - def sdf_name(self, value): - self._sdf_name = value - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - - @property - def update_cap(self): - '''Get the update_cap flag.''' - return self._update_cap - - @update_cap.setter - def update_cap(self, value): - self._update_cap = value - - def parse(self, make_call_tree=False): - '''Parse the suite definition file.''' - success = True - - if not os.path.exists(self._sdf_name): - logging.critical("Suite definition file {0} not found.".format(self._sdf_name)) - success = False - return success - - tree = ET.parse(self._sdf_name) - suite_xml = lowercase_xml(tree.getroot()) - self._name = suite_xml.get('name') - - # Check if suite name is too long - if len(self._name) > SUITE_NAME_MAX_CHARS: - logging.critical(f"Suite name {self._name} has more than the allowed {SUITE_NAME_MAX_CHARS} characters") - success = False - return success - - # Flattened lists of all schemes and subroutines in SDF - self._all_schemes_called = [] - self._all_subroutines_called = [] - - if make_call_tree: - # Call tree of all schemes in SDF. call_tree is a dictionary, with keys corresponding to each group in a suite, and - # the value associated with each key being an ordered list of the schemes in each group (with duplicates and subcycles) - self._call_tree = {} - - # Build hierarchical structure as in SDF - self._groups = [] - for group_xml in suite_xml: - subcycles = [] - - self._call_tree[group_xml.attrib['name']] = [] - # Add suite-wide init scheme to group 'init', similar for finalize - if group_xml.tag == 'init' or group_xml.tag == 'finalize': - self._all_schemes_called.append(group_xml.text) - self._all_subroutines_called.append(group_xml.text + '_' + group_xml.tag) - schemes = [group_xml.text] - subcycles.append(Subcycle(loop=1, schemes=schemes)) - if group_xml.tag == 'init': - self._groups.append(Group(name=group_xml.tag, subcycles=subcycles, suite=self._name, init=True)) - elif group_xml.tag == 'finalize': - self._groups.append(Group(name=group_xml.tag, subcycles=subcycles, suite=self._name, finalize=True)) - continue - - # Parse subcycles of all regular groups - for subcycle_xml in group_xml: - schemes = [] - for scheme_xml in subcycle_xml: - self._all_schemes_called.append(scheme_xml.text) - schemes.append(scheme_xml.text) - loop=int(subcycle_xml.get('loop')) - for ccpp_stage in CCPP_STAGES: - self._all_subroutines_called.append(scheme_xml.text + '_' + CCPP_STAGES[ccpp_stage]) - - subcycles.append(Subcycle(loop=loop, schemes=schemes)) - - if make_call_tree: - # Populate call tree from SDF's heirarchical structure, including multiple calls in subcycle loops - for loop in range(0,int(subcycle_xml.get('loop'))): - for scheme_xml in subcycle_xml: - self._call_tree[group_xml.attrib['name']].append(scheme_xml.text) - - self._groups.append(Group(name=group_xml.get('name'), subcycles=subcycles, suite=self._name)) - - # Remove duplicates from list of all subroutines an schemes - self._all_schemes_called = list(set(self._all_schemes_called)) - self._all_subroutines_called = list(set(self._all_subroutines_called)) - - return success - - def print_debug(self): - '''Basic debugging output about the suite.''' - print("ALL SUBROUTINES:") - print(self._all_subroutines_called) - print("STRUCTURED:") - print(self._groups) - for group in self._groups: - group.print_debug() - - @property - def all_schemes_called(self): - '''Get the list of all schemes.''' - return self._all_schemes_called - - @property - def call_tree(self): - '''Get the call tree of the suite (all schemes, in order, with duplicates and loops).''' - return self._call_tree - - @property - def all_subroutines_called(self): - '''Get the list of all subroutines.''' - return self._all_subroutines_called - - @property - def module(self): - '''Get the list of the module generated for this suite.''' - return self._module - - @property - def subroutines(self): - '''Get the list of all subroutines generated for this suite.''' - return self._subroutines - - @property - def caps(self): - '''Get the list of all caps.''' - return self._caps - - @property - def groups(self): - '''Get the list of groups in this suite.''' - return self._groups - - @property - def parents(self): - '''Get the parent variables for the suite.''' - return self._parents - - @parents.setter - def parents(self, value): - self._parents = value - - @property - def arguments(self): - '''Get the argument list for the suite.''' - return self._arguments - - @arguments.setter - def arguments(self, value): - self._arguments = value - - def write(self, metadata_request, metadata_define, arguments, debug): - """Create caps for all groups in the suite and for the entire suite - (calling the group caps one after another). Add additional code for - debugging if debug flag is True.""" - # Set name of module and filename of cap - self._module = 'ccpp_{suite_name}_cap'.format(suite_name=self._name) - self.filename = '{module_name}.F90'.format(module_name=self._module) - # Init - self._subroutines = [] - # Write group caps and generate module use statements; combine the argument lists - # and variable definitions for all groups into a suite argument list. This may - # require adjusting the intent of the variables. - module_use = '' - for group in self._groups: - group.write(metadata_request, metadata_define, arguments, debug) - for subroutine in group.subroutines: - module_use += ' use {m}, only: {s}\n'.format(m=group.module, s=subroutine) - for ccpp_stage in CCPP_STAGES.keys(): - for parent_standard_name in group.parents[ccpp_stage].keys(): - if parent_standard_name in self.parents[ccpp_stage]: - if self.parents[ccpp_stage][parent_standard_name].intent == 'in' and \ - not group.parents[ccpp_stage][parent_standard_name].intent == 'in': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - elif self.parents[ccpp_stage][parent_standard_name].intent == 'out' and \ - not group.parents[ccpp_stage][parent_standard_name].intent == 'out': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - else: - self.parents[ccpp_stage][parent_standard_name] = copy.deepcopy(group.parents[ccpp_stage][parent_standard_name]) - subs = '' - for ccpp_stage in CCPP_STAGES.keys(): - # Create a wrapped argument list for calling the suite, - # get module use statements and variable definitions - (self.arguments[ccpp_stage], sub_module_use, sub_var_defs) = \ - create_arguments_module_use_var_defs(self.parents[ccpp_stage], metadata_define) - argument_list_suite = create_argument_list_wrapped(self.arguments[ccpp_stage]) - body = '' - for group in self._groups: - # Groups 'init'/'finalize' are only run in stages 'init'/'finalize' - if (group.init and not ccpp_stage == 'init') or \ - (group.finalize and not ccpp_stage == 'finalize'): - continue - # Create a wrapped argument list for calling the group - (arguments_group, dummy, dummy) = create_arguments_module_use_var_defs(group.parents[ccpp_stage], metadata_define) - argument_list_group = create_argument_list_wrapped_explicit(arguments_group) - - # Write to body that calls the groups for this stage - body += ''' - ierr = {suite_name}_{group_name}_{stage}_cap({arguments}) - if (ierr/=0) return -'''.format(suite_name=self._name, group_name=group.name, stage=CCPP_STAGES[ccpp_stage], arguments=argument_list_group) - # Add name of subroutine in the suite cap to list of subroutine names - subroutine = '{name}_{stage}_cap'.format(name=self._name, stage=CCPP_STAGES[ccpp_stage]) - self._subroutines.append(subroutine) - # Add subroutine to output - subs += Suite.sub.format(subroutine=subroutine, - arguments=argument_list_suite, - module_use='\n '.join(sub_module_use), - var_defs='\n '.join(sub_var_defs), - body=body) - - # Write cap to stdout or file - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - # If the file exists, write to temporary file first and compare them: - # - if identical, delete the temporary file and keep the existing one - # and set the suite cap update flag to false - # - if different, replace existing file with temporary file and set - # the suite cap update flag to true (default value) - # - however, if any of the group caps has changed, rewrite the suite - # cap as well and set the suite cap update flag to true - # If the file does not exist, write the cap an set the flag to true - if os.path.isfile(self.filename) and \ - not any([group.update_cap for group in self._groups]): - write_to_test_file = True - test_filename = self.filename + '.test' - f = open(test_filename, 'w') - else: - write_to_test_file = False - f = open(self.filename, 'w') - else: - f = sys.stdout - f.write(Suite.header.format(module=self._module, - module_use=module_use, - subroutines=', &\n '.join(self._subroutines))) - f.write(subs) - f.write(Suite.footer.format(module=self._module)) - if (f is not sys.stdout): - f.close() - # See comment above on updating the suite cap or not - if write_to_test_file: - if filecmp.cmp(self.filename, test_filename): - # Files are equal, delete the test cap - # and set update flag to False - os.remove(test_filename) - self.update_cap = False - else: - # Files are different, replace existing cap - # with test cap and set flag to True - # Python 3 only: os.replace(test_filename, self.filename) - os.remove(self.filename) - os.rename(test_filename, self.filename) - self.update_cap = True - else: - self.update_cap = True - - # Create list of all caps generated (for groups and suite) - self._caps = [ self.filename ] - for group in self._groups: - self._caps.append(group.filename) - - -############################################################################### -class Group(object): - - header=''' -! -! This work (Common Community Physics Package), identified by NOAA, NCAR, -! CU/CIRES, is free of known copyright restrictions and is placed in the -! public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -! - -!> -!! @brief Auto-generated cap module for the CCPP {group} group -!! -! -module {module} - -{module_use} - - implicit none - - private - public :: {subroutines} - - logical, dimension({num_instances}), save :: initialized = .false. - - contains -''' - - sub = ''' - function {subroutine}({argument_list}) result(ierr) - - {module_use} - - implicit none - - ! Error handling - integer :: ierr - - {var_defs} - - ierr = 0 - -{initialized_test_block} - -{body} - -{initialized_set_block} - - end function {subroutine} -''' - - footer = ''' -end module {module} -''' - - initialized_test_blocks = { - 'init' : ''' - if (initialized({ccpp_var_name}%ccpp_instance)) return -''', - 'timestep_init' : ''' - if (.not.initialized({ccpp_var_name}%ccpp_instance)) then - write({target_name_msg},'(*(a))') '{name}_timestep_init called before {name}_init' - {target_name_flag} = 1 - return - end if -''', - 'run' : ''' - if (.not.initialized({ccpp_var_name}%ccpp_instance)) then - write({target_name_msg},'(*(a))') '{name}_run called before {name}_init' - {target_name_flag} = 1 - return - end if -''', - 'timestep_finalize' : ''' - if (.not.initialized({ccpp_var_name}%ccpp_instance)) then - write({target_name_msg},'(*(a))') '{name}_timestep_finalize called before {name}_init' - {target_name_flag} = 1 - return - end if -''', - 'finalize' : ''' - if (.not.initialized({ccpp_var_name}%ccpp_instance)) return -''', - } - - initialized_set_blocks = { - 'init' : ''' - initialized({ccpp_var_name}%ccpp_instance) = .true. -''', - 'timestep_init' : '', - 'run' : '', - 'timestep_finalize' : '', - 'finalize' : ''' - initialized = .false. -''', - } - - def __init__(self, **kwargs): - self._name = '' - self._suite = None - self._filename = sys.stdout - self._init = False - self._finalize = False - self._module = None - self._subroutines = None - self._pset = None - self._parents = { ccpp_stage : collections.OrderedDict() for ccpp_stage in CCPP_STAGES } - self._arguments = { ccpp_stage : [] for ccpp_stage in CCPP_STAGES } - self._update_cap = True - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - def write(self, metadata_request, metadata_define, arguments_in, debug): - """Create caps for all stages of this group. Add additional code for - debugging if debug flag is True.""" - - # First, convert all keys in arguments_in to lowercase (recursively) - arguments = lowercase_keys(arguments_in) - - # Create an inverse lookup table of local variable names defined (by the host model) and standard names - standard_name_by_local_name_define = collections.OrderedDict() - for standard_name in metadata_define.keys(): - standard_name_by_local_name_define[metadata_define[standard_name][0].local_name] = standard_name - - # First get target names of standard CCPP variables for subcycling and error handling - ccpp_loop_counter_target_name = metadata_request[CCPP_LOOP_COUNTER][0].target - ccpp_loop_extent_target_name = metadata_request[CCPP_LOOP_EXTENT][0].target - ccpp_error_code_target_name = metadata_request[CCPP_ERROR_CODE_VARIABLE][0].target - ccpp_error_msg_target_name = metadata_request[CCPP_ERROR_MSG_VARIABLE][0].target - # Then, identify the variable name of the mandatory ccpp_t variable defined by the host model - ccpp_var = metadata_define[CCPP_T_INSTANCE_VARIABLE][0] - - # Init - module_use = '' - self._module = 'ccpp_{suite}_{name}_cap'.format(name=self._name, suite=self._suite) - self._filename = '{module_name}.F90'.format(module_name=self._module) - self._subroutines = [] - local_subs = '' - # - for ccpp_stage in CCPP_STAGES.keys(): - # The special init and finalize routines are only run in that stage - if self._init and not ccpp_stage == 'init': - continue - elif self._finalize and not ccpp_stage == 'finalize': - continue - # For mapping local variable names to standard names - local_vars = collections.OrderedDict() - # For mapping temporary variable names (for unit conversions, etc) to local variable names - tmpvar_cnt = 0 - tmpvars = collections.OrderedDict() - # For mapping temporary pointer names (for potentially unallocated arrays) to local variable names - tmpptr_cnt = 0 - tmpptrs = collections.OrderedDict() - # - body = '' - # Variable definitions automatically added for subroutines - var_defs = '' - # List of manual variable definitions, for example for handling blocked data structures - var_defs_manual = [] - # Conditionals for variables (used or allocated only under certain conditions) - conditionals = {} - # - for subcycle in self._subcycles: - subcycle_body = '' - # Call all schemes - for scheme_name in subcycle.schemes: - # actions_before and actions_after capture operations such - # as unit conversions, transformations that have to happen - # before and/or after the call to the subroutine (scheme) - actions_before = '' - actions_after = '' - # - module_name = scheme_name - subroutine_name = scheme_name + '_' + ccpp_stage - container = encode_container(module_name, scheme_name, subroutine_name) - # Skip entirely empty routines or non-existent routines - if not subroutine_name in arguments[scheme_name].keys() or not arguments[scheme_name][subroutine_name]: - continue - error_check = '' - args = '' - length = 0 - - # First, add a few mandatory variables to the list of required - # variables. This is mostly for handling horizontal dimensions - # correctly for the different CCPP phases and for chunked arrays - additional_variables_required = [] - if CCPP_HORIZONTAL_LOOP_EXTENT in metadata_define.keys(): - for add_var in [ CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_LOOP_EXTENT]: - if not add_var in local_vars.keys() \ - and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: - logging.debug("Adding variable {} for handling blocked data structures".format(add_var)) - additional_variables_required.append(add_var) - elif ccpp_stage == 'run' and \ - CCPP_HORIZONTAL_LOOP_BEGIN in metadata_define.keys() and \ - CCPP_HORIZONTAL_LOOP_END in metadata_define.keys() and \ - CCPP_CHUNK_EXTENT in metadata_define.keys(): - for add_var in [ CCPP_HORIZONTAL_LOOP_BEGIN, CCPP_HORIZONTAL_LOOP_END, CCPP_CHUNK_EXTENT]: - if not add_var in local_vars.keys() \ - and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: - logging.debug("Adding variable {} for handling chunked data arrays".format(add_var)) - additional_variables_required.append(add_var) - # Next, identify all dimensions needed to handle the arguments - # and add them to the list of required variables for the cap - for var_standard_name in arguments[scheme_name][subroutine_name]: - if not var_standard_name in metadata_define.keys(): - raise Exception('Variable {standard_name} not defined in host model metadata'.format( - standard_name=var_standard_name)) - var = metadata_define[var_standard_name][0] - # dim_expression can be 'A', '1', '1:A', ... - for dim_expression in var.dimensions: - dims = dim_expression.split(':') - for dim in dims: - dim = dim.lower() - try: - dim = int(dim) - except ValueError: - if not dim in local_vars.keys() and \ - not dim in additional_variables_required + arguments[scheme_name][subroutine_name]: - if not dim in metadata_define.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( - dim, var_standard_name)) - logging.debug("Adding dimension {} for variable {}".format(dim, var_standard_name)) - additional_variables_required.append(dim) - - # If blocked data structures need to be converted, add necessary variables - if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in var.local_name: - for add_var in [ CCPP_BLOCK_COUNT, CCPP_HORIZONTAL_DIMENSION]: - if not add_var in local_vars.keys() \ - and not add_var in additional_variables_required + arguments[scheme_name][subroutine_name]: - logging.debug("Adding variable {} for handling blocked data structures".format(add_var)) - additional_variables_required.append(add_var) - - # If the variable is only active/used under certain conditions, add necessary variables - # also record the conditional for later use in unit conversions / blocked data conversions. - if var.active == 'T': - conditional = '.true.' - elif var.active == 'F': - conditional = '.false.' - else: - # Convert conditional expression in standard_name format to local names known to the host model - conditional = '' - # Find all words in the conditional, for each of them look for a matching - # standard name in the list of known variables - items = FORTRAN_CONDITIONAL_REGEX.findall(var.active) - for item in items: - item = item.lower() - if item in FORTRAN_CONDITIONAL_REGEX_WORDS: - conditional += item - else: - # Detect integers, following Python's "easier to ask forgiveness than permission" mentality - try: - int(item) - conditional += item - except ValueError: - if not item in metadata_define.keys(): - raise Exception("Variable {} used in conditional for {} not known to host model".format( - item, var_standard_name)) - var2 = metadata_define[item][0] - conditional += var2.local_name - # Add to list of required variables for the cap - if not item in local_vars.keys() \ - and not item in additional_variables_required + arguments[scheme_name][subroutine_name]: - logging.debug("Adding variable {} for handling conditionals".format(item)) - additional_variables_required.append(item) - # Conditionals are identical per requirement, no need to test for consistency again - if not var_standard_name in conditionals.keys(): - conditionals[var_standard_name] = conditional - - # Extract all variables needed (including indices for components/slices of arrays and - # including their parents). We need to run this twice, because the dimensions of parent - # variables get added to additional_variables_required in the first pass. - iteration = 1 - while iteration <= 2: - for var_standard_name in additional_variables_required + arguments[scheme_name][subroutine_name]: - # Pick the correct variable for this module/scheme/subroutine - # from the list of requested variables, if it is in that list - if var_standard_name in arguments[scheme_name][subroutine_name]: - for var in metadata_request[var_standard_name]: - if container == var.container: - break - # This is a dimension or required variable added automatically (e.g. for handling blocked data) - else: - # Create a copy of the variable in the metadata dictionary - # of host model variables and set necessary default values - var = copy.deepcopy(metadata_define[var_standard_name][0]) - var.intent = 'in' - - if not var_standard_name in local_vars.keys(): - # The full name of the variable as known to the host model - var_local_name_define = metadata_define[var_standard_name][0].local_name - - # Break apart var_local_name_define into the different components (members of DDTs) - # to determine all variables that are required - (parent_local_name_define, parent_local_names_define_indices) = \ - extract_parents_and_indices_from_local_name(var_local_name_define) - - parent_standard_name = None - parent_var = None - # Check for each of the derived parent local names as defined by the host model - # if they are registered (i.e. if there is a standard name for it). Note that - # the output of extract_parents_and_indices_from_local_name is stripped of any - # array subset information, i.e. a local name 'Atm(:)%...' will produce a - # parent local name 'Atm'. Since the rank of the parent variable is not known - # at this point and since the local name in the host model metadata table could - # contain '(:)', '(:,:)', ... (up to the rank of the array), we search for the - # maximum number of dimensions allowed by the Fortran standard. - for local_name_define in [parent_local_name_define] + parent_local_names_define_indices: - parent_standard_name = None - parent_var = None - for i in range(FORTRAN_ARRAY_MAX_DIMS+1): - if i==0: - dims_string = '' - else: - # (:) for i==1, (:,:) for i==2, ... - dims_string = '(' + ','.join([':' for j in range(i)]) + ')' - if local_name_define+dims_string in standard_name_by_local_name_define.keys(): - parent_standard_name = standard_name_by_local_name_define[local_name_define+dims_string] - parent_var = metadata_define[parent_standard_name][0] - break - if not parent_var: - raise Exception('Parent variable {parent} of {child} with standard name '.format( - parent=local_name_define, child=var_local_name_define)+\ - '{standard_name} not defined in host model metadata'.format( - standard_name=var_standard_name)) - - # Reset local name for entire array to a notation without (:), (:,:), etc.; - # this is needed for the var.print_def_intent() routine to work correctly - parent_var.local_name = local_name_define - - # Add the parent_var's dimensions to the locally defined dimensions: - # dim_expression can be 'A', '1', '1:A', ... - for dim_expression in parent_var.dimensions: - dims = dim_expression.split(':') - for dim in dims: - dim = dim.lower() - try: - dim = int(dim) - except ValueError: - if not dim in local_vars.keys() and \ - not dim in additional_variables_required + arguments[scheme_name][subroutine_name]: - if not dim in metadata_define.keys(): - raise Exception('Dimension {}, required by parent variable {}, not defined in host model metadata'.format( - dim, parent_standard_name)) - logging.debug("Adding dimension {} for parent variable {}".format(dim, parent_standard_name)) - additional_variables_required.append(dim) - - # Add variable to dictionary of parent variables, if not already there. - # Set or update intent, depending on whether the variable is an index - # in var_local_name_define or the actual parent of that variable. - if not parent_standard_name in self.parents[ccpp_stage].keys(): - self.parents[ccpp_stage][parent_standard_name] = copy.deepcopy(parent_var) - # Copy the intent of the actual variable being processed - if local_name_define == parent_local_name_define: - self.parents[ccpp_stage][parent_standard_name].intent = var.intent - # It's an index for the actual variable being processed --> intent(in) - else: - self.parents[ccpp_stage][parent_standard_name].intent = 'in' - elif self.parents[ccpp_stage][parent_standard_name].intent == 'in': - # Adjust the intent if the actual variable is not intent(in) - if local_name_define == parent_local_name_define and not var.intent == 'in': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - # It's an index for the actual variable being processed, intent is ok - #else: - # # nothing to do - elif self.parents[ccpp_stage][parent_standard_name].intent == 'out': - # Adjust the intent if the actual variable is not intent(out) - if local_name_define == parent_local_name_define and not var.intent == 'out': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - # Adjust the intent, because the variable is also used as index variable - else: - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - - # Record the parent information for this variable (with standard name var_standard_name) - if local_name_define == parent_local_name_define: - local_vars[var_standard_name] = { - 'name' : metadata_define[var_standard_name][0].local_name, - 'kind' : metadata_define[var_standard_name][0].kind, - 'parent_standard_name' : parent_standard_name - } - - # Reset parent to actual parent of the variable with standard name var_standard_name - if local_vars[var_standard_name]['parent_standard_name']: - parent_standard_name = local_vars[var_standard_name]['parent_standard_name'] - parent_var = metadata_define[parent_standard_name][0] - - elif local_vars[var_standard_name]['parent_standard_name']: - parent_standard_name = local_vars[var_standard_name]['parent_standard_name'] - parent_var = metadata_define[parent_standard_name][0] - # Update intent information if necessary - if self.parents[ccpp_stage][parent_standard_name].intent == 'in' and not var.intent == 'in': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - elif self.parents[ccpp_stage][parent_standard_name].intent == 'out' and not var.intent == 'out': - self.parents[ccpp_stage][parent_standard_name].intent = 'inout' - - # End of iteration (while) loop, increase iteration counter - iteration += 1 - - # Loop over actual arguments for this subroutine and create the argument list. - # This is not required for the additional dimensions and variables. - for var_standard_name in arguments[scheme_name][subroutine_name]: - # Pick the correct variable for this module/scheme/subroutine - # from the list of requested variables - for var in metadata_request[var_standard_name]: - if container == var.container: - break - - # We need some information about the host model variable - (dimensions_target_name, dim_string_target_name) = extract_dimensions_from_local_name(var.target) - - # Derive correct horizontal loop extent for this variable for the rest of this function - if var.rank: - array_size = [] - dim_substrings = [] - for dim in var.dimensions: - - # Work around for GNU compiler bugs related to allocatable strings - # in older versions of GNU (at least 9.2.0) - if var.rank and var.type == 'character': - use_explicit_dimension = False - else: - use_explicit_dimension = True - - # This is not supported/implemented: tmpvar would have one dimension less - # than the original array, and the metadata requesting the variable would - # not pass the initial test that host model variables and scheme variables - # have the same rank. - if dim == CCPP_BLOCK_NUMBER: - raise Exception("{} cannot be part of the dimensions of variable {}".format( - CCPP_BLOCK_NUMBER, var_standard_name)) - else: - # Handle dimensions like "A:B", "A:3", "-1:Z" - if ':' in dim: - dims = [ x.lower() for x in dim.split(':')] - try: - dim0 = int(dims[0]) - dim0 = dims[0] - except ValueError: - if not dims[0].lower() in metadata_define.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( - dims[0].lower(), var_standard_name)) - dim0 = metadata_define[dims[0].lower()][0].local_name - try: - dim1 = int(dims[1]) - dim1 = dims[1] - except ValueError: - # Use correct horizontal variables in run phase - if ccpp_stage == 'run' and dims[1].lower() == CCPP_HORIZONTAL_LOOP_EXTENT: - # Provide backward compatibility with blocked data structures - # and bypass the unresolved problems with inactive data - - # For this, we need to check if the host model variable - # is a contiguous array (it's horizontal dimension is - # CCPP_HORIZONTAL_DIMENSION) or part of a blocked data - # structure (it's horizontal dimension is CCPP_HORIZONTAL_LOOP_EXTENT) - for dim in metadata_define[var_standard_name][0].dimensions: - if ':' in dim: - host_var_dims = [x.lower() for x in dim.split(':')] - # Single dimensions are indices and should not be recorded as a dimension! - else: - raise Exception("THIS SHOULD NOT HAPPEN WITH CAPGEN'S METADATA PARSER") - if CCPP_HORIZONTAL_DIMENSION in host_var_dims: - host_var_is_contiguous = True - elif CCPP_HORIZONTAL_LOOP_EXTENT in host_var_dims: - host_var_is_contiguous = False - if CCPP_HORIZONTAL_LOOP_BEGIN in metadata_define.keys() and host_var_is_contiguous: - dim0 = metadata_define[CCPP_HORIZONTAL_LOOP_BEGIN][0].local_name - dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_END][0].local_name - use_explicit_dimension = True - else: - dim0 = metadata_define[CCPP_CONSTANT_ONE][0].local_name - dim1 = metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name - # Remove this variable so that we can catch errors - # if it doesn't get set even though it should - del host_var_is_contiguous - else: - if not dims[1].lower() in metadata_define.keys(): - raise Exception('Dimension {}, required by variable {}, not defined in host model metadata'.format( - dims[1].lower(), var_standard_name)) - dim1 = metadata_define[dims[1].lower()][0].local_name - # Single dimensions are indices and should not be recorded as a dimension! - else: - raise Exception("THIS SHOULD NOT HAPPEN WITH CAPGEN'S METADATA PARSER") - - # DH* TODO REMOVE THIS ENTIRE BLOCK in a future PR to feature/capgen - # This block should not be needed, the metadata parser should take care - # of flagging invalid dimensions for the host model or the physics. - # TODO: create a test suite to make sure these things are caught - # by the metadata parser - do this on the feature/capgen branch! - if ccpp_stage == 'run': - # This should not happen when parsing metadata with capgen's metadata parser, remove? - if dims[1] == CCPP_HORIZONTAL_LOOP_EXTENT and not dim0: - raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ - f"horizontal dimension for {var_standard_name} is {var.dimensions}") - # This should not happen when parsing metadata with capgen's metadata parser, remove? - elif CCPP_HORIZONTAL_LOOP_BEGIN in dims or CCPP_HORIZONTAL_LOOP_END in dims or \ - CCPP_HORIZONTAL_DIMENSION in dims: - raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ - f"horizontal dimension for {var_standard_name} is {var.dimensions}") - else: - # This should not happen when parsing metadata with capgen's metadata parser, remove? - if dims[1] == CCPP_HORIZONTAL_DIMENSION and not dim0: - raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ - f"horizontal dimension for {var_standard_name} is {var.dimensions}") - # This should not happen when parsing metadata with capgen's metadata parser, remove? - if CCPP_HORIZONTAL_LOOP_BEGIN in dims or CCPP_HORIZONTAL_LOOP_END in dims or \ - CCPP_LOOP_EXTENT in dims: - raise Exception(f"Invalid metadata for scheme {scheme_name}: " + \ - f"horizontal dimension for {var_standard_name} is {var.dimensions}") - # *DH - - # DH* TODO: WE CANNOT ACTIVATE USING EXPLICIT HORIZONTAL DIMENSIONS - # UNTIL WE HAVE SOLVED THE PROBLEM WITH INACTIVE (NON-ALLOCATED) - # ARRAYS. THIS MUST BE ADDRESSED BEFORE WE SWITCH TO CONTIGUOUS - # ARRAYS FOR MODELS LIKE THE UFS-WEATHER-MODEL! - if use_explicit_dimension: - if dim0 == dim1: - array_size.append('1') - dim_substrings.append(f'{dim1}') - else: - array_size.append(f'({dim1}-{dim0}+1)') - dim_substrings.append(f'{dim0}:{dim1}') - else: - if dim0 == dim1: - array_size.append('1') - dim_substrings.append(f':') - else: - array_size.append(f'({dim1}-{dim0}+1)') - dim_substrings.append(f':') - - # Now we need to compare dim_substrings with a possible dim_string_target_name and merge them - if dimensions_target_name: - if len(dimensions_target_name) < len(dim_substrings): - raise Exception("THIS SHOULD NOT HAPPEN") - dim_counter = 0 - # We need two different dim strings for the following. The first, - # called 'dim_string' is used for the incoming variable (host model - # variable) and must contain all dimensions and indices. The second, - # called 'dim_string_allocate' is used for the allocation of temporary - # variables used for unit conversions etc. This 'dim_string_allocate' - # only contains the dimensions, not the indices of the target variable. - # Example: a scheme requests a variable foo that is a slice of a host - # model variable bar: foo(1:n) = bar(1:n,1). If a variable transformation - # is required, mkstatic creates a temporary variable tmpvar of rank 1. - # The allocation and assignment of this variable must then be - # allocate(tmpvar(1:n)) - # tmpvar(1:n) = bar(1:n,1) - # Likewise, when scheme baz is called, the callstring must be - # call baz(...,foo=tmpvar(1:n),...) - # Hence the need for two different dim strings in the following code. - # See also https://github.com/NCAR/ccpp-framework/issues/598. - dim_string = '(' - dim_string_allocate = '(' - for dim in dimensions_target_name: - if ':' in dim: - dim_string += dim_substrings[dim_counter] + ',' - dim_string_allocate += dim_substrings[dim_counter] + ',' - dim_counter += 1 - else: - dim_string += dim + ',' - # Don't add to dim_string_allocate! - dim_string = dim_string.rstrip(',') + ')' - dim_string_allocate = dim_string_allocate.rstrip(',') + ')' - # Consistency check to make sure all dimensions from metadata are 'used' - if dim_counter < len(dim_substrings): - raise Exception(f"Mismatch of derived dimensions from metadata {dim_substrings} " + \ - f"vs target local name {dimensions_target_name} for {var_standard_name} and " + \ - f"scheme {scheme_name} / phase {ccpp_stage}") - else: - dim_string = '({})'.format(','.join(dim_substrings)) - dim_string_allocate = dim_string - var_size_expected = '({})'.format('*'.join(array_size)) - else: - if dimensions_target_name: - dim_string = dim_string_target_name - else: - dim_string = '' - # A scalar variable doesn't get allocated, set to safe value - dim_string_allocate = '' - var_size_expected = 1 - - # To assist debugging efforts, check if arrays have the correct size (ignore scalars for now) - assign_test = '' - if debug: - if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ - CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name'] and \ - '{}:{}'.format(CCPP_CONSTANT_ONE,CCPP_HORIZONTAL_DIMENSION) in var.dimensions: - # We don't need extra tests for blocked arrays, because the de-blocking logic below - # will catch any out of bound reads with the appropriate compiler flags. It naturally - # deals with non-uniform block sizes etc. - pass - # Some older versions of GNU currently in use can not do these variable size tests on strings - # 0x5b6fdd gimplify_expr(tree_node**, gimple**, gimple**, bool (*)(tree_node*), int) - # /tmp/role.apps/spack-stage/spack-stage-gcc-9.2.0-ku6r4f5qa5obpfnqpa6pezhogxq6sp7h/spack-src/gcc/gimplify.c:13477 - elif var.rank and not var.type == 'character': - assign_test = ''' ! Check if variable {var_name} is associated/allocated and has the correct size - if (size({var_name}{dim_string})/={var_size_expected}) then - write({ccpp_errmsg}, '(2(a,i8))') 'Detected size mismatch for variable {var_name}{dim_string} in group {group_name} before {subroutine_name}, expected ', & - {var_size_expected}, ' but got ', size({var_name}{dim_string}) - ierr = 1 - return - end if -'''.format(var_name=local_vars[var_standard_name]['name'].replace(dim_string_target_name, ''), - dim_string=dim_string, - var_size_expected=var_size_expected, - ccpp_errmsg=CCPP_INTERNAL_VARIABLES[CCPP_ERROR_MSG_VARIABLE], group_name = self.name, - subroutine_name=subroutine_name) - # end if debug - - # kind_string is used for automated unit conversions, i.e. foo_kind_phys - kind_string = '_' + local_vars[var_standard_name]['kind'] if local_vars[var_standard_name]['kind'] else '' - - # conditional is the conditional allocation, which can be '.true.', '.false.', or any regular Fortran logical expression - conditional=conditionals[var_standard_name] - - # If the host variable is conditionally allocated, create a pointer for it - if not conditional == '.true.': - # Reuse existing temporary pointer variable, if possible; otherwise add a local pointer (tmpptr) - if local_vars[var_standard_name]['name'] in tmpptrs.keys(): - tmpptr = tmpptrs[local_vars[var_standard_name]['name']] - else: - tmpptr_cnt += 1 - tmpptr = copy.deepcopy(var) - tmpptr.local_name = '{0}_{1}_ptr'.format(var.local_name, tmpptr_cnt) - tmpptr.pointer = True - tmpptrs[local_vars[var_standard_name]['name']] = tmpptr - - # Convert blocked data in init and finalize steps - only required for variables with block number and horizontal_dimension - if ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ - CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name'] and \ - '{}:{}'.format(CCPP_CONSTANT_ONE,CCPP_HORIZONTAL_DIMENSION) in var.dimensions: - # Reuse existing temporary variable, if possible; otherwise add a local variable (tmpvar) - if local_vars[var_standard_name]['name'] in tmpvars.keys(): - tmpvar = tmpvars[local_vars[var_standard_name]['name']] - actions_in = tmpvar.actions['in'] - actions_out = tmpvar.actions['out'] - else: - tmpvar_cnt += 1 - tmpvar = copy.deepcopy(var) - tmpvar.local_name = '{0}_{1}_local'.format(var.local_name, tmpvar_cnt) - - # Create string for allocating the temporary array by converting the dimensions - # (in standard_name format) to local names as known to the host model - alloc_dimensions = [] - for dim in tmpvar.dimensions: - # This is not supported/implemented: tmpvar would have one dimension less - # than the original array, and the metadata requesting the variable would - # not pass the initial test that host model variables and scheme variables - # have the same rank. - if dim == CCPP_BLOCK_NUMBER: - raise Exception("{} cannot be part of the dimensions of variable {}".format( - CCPP_BLOCK_NUMBER, var_standard_name)) - else: - # Handle dimensions like "A:B", "A:3", "-1:Z" - if ':' in dim: - dims = [ x.lower() for x in dim.split(':')] - try: - dim0 = int(dims[0]) - except ValueError: - dim0 = metadata_define[dims[0]][0].local_name - try: - dim1 = int(dims[1]) - except ValueError: - dim1 = metadata_define[dims[1]][0].local_name - # Single dimensions - else: - dim0 = 1 - try: - dim1 = int(dim) - except ValueError: - dim1 = metadata_define[dim][0].local_name - alloc_dimensions.append('{}:{}'.format(dim0,dim1)) - - # Padding of additional dimensions - before and after the horizontal dimension - hdim_index = tmpvar.dimensions.index('{}:{}'.format(CCPP_CONSTANT_ONE,CCPP_HORIZONTAL_DIMENSION)) - dimpad_before = '' + ':,'*(len(tmpvar.dimensions[:hdim_index])) - dimpad_after = '' + ',:'*(len(tmpvar.dimensions[hdim_index+1:])) - - # Add necessary local variables for looping over blocks - var_defs_manual.append('integer :: ib, nb') - - # Define actions before. Always copy data in, independent of intent. - # We intentionally omit the dim string for the assignment on the right hand side, - # since it worked without until now, since coding this up together with chunked array - # logic is tricky, and since all this logic will go away after the models transitioned - # to chunked arrays. - actions_in = ''' ! Allocate local variable to copy blocked data {var} into a contiguous array - allocate({tmpvar}({dims})) - ib = 1 - do nb=1,{block_count} - {tmpvar}({dimpad_before}ib:ib+{block_size}-1{dimpad_after}) = {var} - ib = ib+{block_size} - end do -'''.format(tmpvar=tmpvar.local_name, - block_count=metadata_define[CCPP_BLOCK_COUNT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - block_size=metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - var=tmpvar.target.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - dims=','.join(alloc_dimensions), - dimpad_before=dimpad_before, - dimpad_after=dimpad_after, - ) - # Define actions after, depending on intent. - if var.intent in [ 'inout', 'out' ]: - actions_out = ''' ib = 1 - do nb=1,{block_count} - {var} = {tmpvar}({dimpad_before}ib:ib+{block_size}-1{dimpad_after}) - ib = ib+{block_size} - end do - deallocate({tmpvar}) -'''.format(tmpvar=tmpvar.local_name, - block_count=metadata_define[CCPP_BLOCK_COUNT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - block_size=metadata_define[CCPP_HORIZONTAL_LOOP_EXTENT][0].local_name.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - var=tmpvar.target.replace(CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER],'nb'), - dimpad_before=dimpad_before, - dimpad_after=dimpad_after, - ) - else: - actions_out = ''' deallocate({tmpvar}) -'''.format(tmpvar=tmpvar.local_name) - - # Set/update actions for this temporary variable - tmpvar.actions = {'in' : actions_in, 'out' : actions_out} - tmpvars[local_vars[var_standard_name]['name']] = tmpvar - - # Add unit conversions, if necessary - if var.actions['in']: - # Add unit conversion before entering the subroutine, after allocating the temporary - # array holding the non-blocked data and copying the blocked data to it - actions_in = actions_in + \ - ' {t} = {c}\n'.format(t=tmpvar.local_name, - c=var.actions['in'].format(var=tmpvar.local_name, - kind=kind_string)) - - # If the variable is conditionally allocated, assign pointer - if not conditional == '.true.': - # We don't want the dimstring here - this can lead to dimension mismatches. - # We know for sure that we need to reference the entire de-blocked array anyway. - actions_in += ' {p} => {t}\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p", - t=tmpvar.local_name, d=dim_string) - - if var.actions['out']: - # Add unit conversion after returning from the subroutine, before copying the non-blocked - # data back to the blocked data and deallocating the temporary array - actions_out = ' {t} = {c}\n'.format(t=tmpvar.local_name, - c=var.actions['out'].format(var=tmpvar.local_name, - kind=kind_string)) + \ - actions_out - - # If the variable is conditionally allocated, nullify pointer - if not conditional == '.true.': - actions_out += ' nullify({p})\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - # Add the conditionals for the "before" operations - actions_before += ''' - if ({conditional}) then -{actions_in} - end if -'''.format(conditional=conditional, actions_in=actions_in.rstrip('\n')) - # Add the conditionals for the "after" operations - actions_after += ''' - if ({conditional}) then -{actions_out} - end if -'''.format(conditional=conditional, actions_out=actions_out.rstrip('\n')) - - # Add to argument list - if conditional == '.true.': - arg = '{local_name}={var_name},'.format(local_name=var.local_name, var_name=tmpvar.local_name) - else: - arg = '{local_name}={ptr_name},'.format(local_name=var.local_name, - ptr_name=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - # Variables stored in blocked data structures but without horizontal dimension not supported at this time (doesn't make sense anyway) - elif ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ - CCPP_INTERNAL_VARIABLES[CCPP_BLOCK_NUMBER] in local_vars[var_standard_name]['name']: - raise Exception("Variables stored in blocked data structures but without horizontal dimension not supported in phases ' + \ - 'init, timestep_init, timestep_finalize, finalize at this time: {} in {}".format(var_standard_name, subroutine_name)) - - # Limitations for UFS: Variables stored in threaded data structures (i.e. only for one block at a time) in GFS_interstitial DDT - # are not supported at this time (doesn't make sense anyway) - elif ccpp_stage in ['init', 'timestep_init', 'timestep_finalize', 'finalize'] and \ - CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER] in local_vars[var_standard_name]['name']: - raise Exception("Variables stored in thread-specific data structures (GFS_interstitial DDT) are not supported in phases ' + \ - 'init, timestep_init, timestep_finalize, finalize at this time: {} in {}".format(var_standard_name, subroutine_name)) - - # Unit conversions without converting blocked data structures - elif var.actions['in'] or var.actions['out']: - # If requested, check that arrays are allocated/associated and have the correct size - if debug: - actions_in = assign_test - else: - actions_in = '' - actions_out = '' - if local_vars[var_standard_name]['name'] in tmpvars.keys(): - # If the variable already has a local variable (tmpvar), reuse it - tmpvar = tmpvars[local_vars[var_standard_name]['name']] - else: - # Add a local variable (tmpvar) for this variable - tmpvar_cnt += 1 - tmpvar = copy.deepcopy(var) - tmpvar.local_name = 'tmpvar_{0}'.format(tmpvar_cnt) - tmpvars[local_vars[var_standard_name]['name']] = tmpvar - if tmpvar.rank: - # Add allocate statement if the variable has a rank > 0 using the dimstring derived above - actions_in += f' allocate({tmpvar.local_name}{dim_string_allocate})\n' - if var.actions['in']: - # Add unit conversion before entering the subroutine - actions_in += ' {t} = {c}{d}\n'.format(t=tmpvar.local_name, - c=var.actions['in'].format(var=tmpvar.target.replace(dim_string_target_name, ''), - kind=kind_string), - d=dim_string) - # If the variable is conditionally allocated, assign pointer - if not conditional == '.true.': - actions_in += ' {p} => {t}{d}\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p", - t=tmpvar.local_name, d=dim_string_allocate) - if var.actions['out']: - # Add unit conversion after returning from the subroutine - actions_out += ' {v}{d} = {c}\n'.format(v=tmpvar.target.replace(dim_string_target_name, ''), - d=dim_string, - c=var.actions['out'].format(var=tmpvar.local_name, - kind=kind_string)) - # If the variable is conditionally allocated, nullify pointer - if not conditional == '.true.': - actions_out += ' nullify({p})\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - if tmpvar.rank: - # Add deallocate statement if the variable has a rank > 0 - actions_out += ' deallocate({t})\n'.format(t=tmpvar.local_name) - - # Add the conditionals for the "before" operations - actions_before += ''' - if ({conditional}) then -{actions_in} - end if -'''.format(conditional=conditional, actions_in=actions_in.rstrip('\n')) - # Add the conditionals for the "after" operations - actions_after += ''' - if ({conditional}) then -{actions_out} - end if -'''.format(conditional=conditional, actions_out=actions_out.rstrip('\n')) - - # Add to argument list - if conditional == '.true.': - arg = '{local_name}={var_name}{dim_string},'.format(local_name=var.local_name, - var_name=tmpvar.local_name.replace(dim_string_target_name, ''), dim_string=dim_string_allocate) - else: - arg = '{local_name}={ptr_name},'.format(local_name=var.local_name, - ptr_name=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - # Ordinary variables, no blocked data or unit conversions - elif var_standard_name in arguments[scheme_name][subroutine_name]: - if debug and assign_test: - actions_in = assign_test - else: - actions_in = '' - actions_out = '' - # If the variable is conditionally allocated, assign pointer - if not conditional == '.true.': - actions_in += ' {p} => {t}{d}\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p", - t=var.target.replace(dim_string_target_name, ''), - d=dim_string) - # If the variable is conditionally allocated, nullify pointer - if not conditional == '.true.': - actions_out += ' nullify({p})\n'.format(p=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - if actions_in: - # Add the conditionals for the "before" operations - actions_before += ''' - if ({conditional}) then -{actions_in} - end if -'''.format(conditional=conditional, actions_in=actions_in.rstrip('\n')) - if actions_out: - # Add the conditionals for the "after" operations - actions_after += ''' - if ({conditional}) then -{actions_out} - end if -'''.format(conditional=conditional, actions_out=actions_out.rstrip('\n')) - - # Add to argument list - if conditional == '.true.': - arg = '{local_name}={var_name}{dim_string},'.format(local_name=var.local_name, - var_name=local_vars[var_standard_name]['name'].replace(dim_string_target_name, ''), dim_string=dim_string) - else: - arg = '{local_name}={ptr_name},'.format(local_name=var.local_name, - ptr_name=f"{tmpptr.local_name}_array({CCPP_INTERNAL_VARIABLES[CCPP_THREAD_NUMBER]})%p") - - else: - arg = '' - args += arg - length += len(arg) - # Split args so that lines don't get too long - if length > 70 and not var_standard_name == arguments[scheme_name][subroutine_name][-1]: - args += ' &\n ' - length = 0 - args = args.rstrip(',') - subroutine_call = ''' -{actions_before} - - call {subroutine_name}({args}) - -{actions_after} -'''.format(subroutine_name=subroutine_name, args=args, actions_before=actions_before.rstrip('\n'), actions_after=actions_after.rstrip('\n')) - error_check = '''if ({target_name_flag}/=0) then - {target_name_msg} = "An error occured in {subroutine_name}: " // trim({target_name_msg}) - ierr={target_name_flag} - return - end if -'''.format(target_name_flag=ccpp_error_code_target_name, target_name_msg=ccpp_error_msg_target_name, subroutine_name=subroutine_name) - subcycle_body += ''' - {subroutine_call} - {error_check} - '''.format(subroutine_call=subroutine_call, error_check=error_check) - - module_use += ' use {m}, only: {s}\n'.format(m=module_name, s=subroutine_name) - - # If this subcycle calls any schemes, i.e. has any variables registered - # that need to be passed to the group for this stage, then handle the - # subcycle loops by prepending/appending the necessary code to subcycle_body - subcycle_body_prefix = ''' - ! Start of next subcycle -''' - subcycle_body_suffix = '' - if self.parents[ccpp_stage]: - # Set subcycle loop extent - if ccpp_stage == 'run': - subcycle_body_prefix += ''' - ! Set loop extent variable for the following subcycle - {loop_extent_var_name} = {loop_cnt_max} -'''.format(loop_extent_var_name=ccpp_loop_extent_target_name, - loop_cnt_max=subcycle.loop) - else: - subcycle_body_prefix += ''' - ! Set loop extent variable for the following subcycle - {loop_extent_var_name} = 1 -'''.format(loop_extent_var_name=ccpp_loop_extent_target_name) - # Create subcycle (Fortran do loop) if needed - if subcycle.loop > 1 and ccpp_stage == 'run': - subcycle_body_prefix += ''' - associate(cnt => {loop_var_name}) - do cnt=1,{loop_cnt_max}\n\n'''.format(loop_var_name=ccpp_loop_counter_target_name, - loop_cnt_max=subcycle.loop) - subcycle_body_suffix += ''' - end do - end associate -''' - else: - subcycle_body_prefix += ''' - {loop_var_name} = 1\n'''.format(loop_var_name=ccpp_loop_counter_target_name) - - # Add this subcycle's Fortran body to the group body - if subcycle_body: - body += subcycle_body_prefix + subcycle_body + subcycle_body_suffix - - #For the init stage, for the case when the suite doesn't have any schemes with init phases, - #we still need to add the host-supplied ccpp_t variable to the init group caps so that it is - #available for setting the initialized flag for the particular instance being called. Otherwise, - #the initialized_set_block for the init phase tries to reference the unavailable ccpp_t variable. - if (ccpp_stage == 'init' and not self.parents[ccpp_stage]): - ccpp_var.intent = 'in' - self.parents[ccpp_stage].update({ccpp_var.standard_name:ccpp_var}) - - # Get list of arguments, module use statement and variable definitions for this subroutine (=stage for the group) - (self.arguments[ccpp_stage], sub_module_use, sub_var_defs) = create_arguments_module_use_var_defs( - self.parents[ccpp_stage], metadata_define, - tmpvars.values(), tmpptrs.values()) - sub_argument_list = create_argument_list_wrapped(self.arguments[ccpp_stage]) - - # Remove duplicates from additional manual variable definitions - var_defs_manual = list(set(var_defs_manual)) - - # Write cap - shorten certain ccpp_stages to stay under the 63 character limit for Fortran function names - subroutine = self._suite + '_' + self._name + '_' + CCPP_STAGES[ccpp_stage] + '_cap' - self._subroutines.append(subroutine) - # Test and set blocks for initialization status - check that at least - # the mandatory CCPP error handling arguments are present (i.e. there is - # at least one subroutine that gets called from this group), or skip. - if self.arguments[ccpp_stage]: - initialized_test_block = Group.initialized_test_blocks[ccpp_stage].format( - ccpp_var_name = ccpp_var.local_name, - target_name_flag=ccpp_error_code_target_name, - target_name_msg=ccpp_error_msg_target_name, - name=self._name) - else: - initialized_test_block = '' - initialized_set_block = Group.initialized_set_blocks[ccpp_stage].format( - ccpp_var_name = ccpp_var.local_name, - target_name_flag=ccpp_error_code_target_name, - target_name_msg=ccpp_error_msg_target_name, - name=self._name) - # Create subroutine - local_subs += Group.sub.format(subroutine=subroutine, - argument_list=sub_argument_list, - module_use='\n '.join(sub_module_use), - initialized_test_block=initialized_test_block, - initialized_set_block=initialized_set_block, - var_defs='\n '.join(sub_var_defs + var_defs_manual), - body=body) - - # Write output to stdout or file - if (self.filename is not sys.stdout): - filepath = os.path.split(self.filename)[0] - if filepath and not os.path.isdir(filepath): - os.makedirs(filepath) - # If the file exists, write to temporary file first and compare them: - # - if identical, delete the temporary file and keep the existing one - # and set the group cap update flag to false - # - if different, replace existing file with temporary file and set - # the group cap update flag to true (default value) - # If the file does not exist, write the cap an set the flag to true - if os.path.isfile(self.filename): - write_to_test_file = True - test_filename = self.filename + '.test' - f = open(test_filename, 'w') - else: - write_to_test_file = False - f = open(self.filename, 'w') - else: - f = sys.stdout - f.write(Group.header.format(group=self._name, - module=self._module, - module_use=module_use, - subroutines=', &\n '.join(self._subroutines), - num_instances=CCPP_NUM_INSTANCES)) - f.write(local_subs) - f.write(Group.footer.format(module=self._module)) - if (f is not sys.stdout): - f.close() - # See comment above on updating the group cap or not - if write_to_test_file: - if filecmp.cmp(self.filename, test_filename): - # Files are equal, delete the test cap - # and set update flag to False - os.remove(test_filename) - self.update_cap = False - else: - # Files are different, replace existing cap - # with test cap and set flag to True - # Python 3 only: os.replace(test_filename, self.filename) - os.remove(self.filename) - os.rename(test_filename, self.filename) - self.update_cap = True - else: - self.update_cap = True - return - - @property - def name(self): - '''Get the name of the group.''' - return self._name - - @name.setter - def name(self, value): - self._name = value - - @property - def filename(self): - '''Get the filename of write the output to.''' - return self._filename - - @filename.setter - def filename(self, value): - self._filename = value - - @property - def update_cap(self): - '''Get the update_cap flag.''' - return self._update_cap - - @update_cap.setter - def update_cap(self, value): - self._update_cap = value - - @property - def init(self): - '''Get the init flag.''' - return self._init - - @init.setter - def init(self, value): - if not type(value) == types.BooleanType: - raise Exception("Invalid type {0} of argument value, boolean expected".format(type(value))) - self._init = value - - @property - def finalize(self): - '''Get the finalize flag.''' - return self._finalize - - @finalize.setter - def finalize(self, value): - if not type(value) == types.BooleanType: - raise Exception("Invalid type {0} of argument value, boolean expected".format(type(value))) - self._finalize = value - - @property - def suite(self): - '''Get the suite name.''' - return self._suite - - @property - def module(self): - '''Get the module name.''' - return self._module - - @property - def subcycles(self): - '''Get the subcycles.''' - return self._subcycles - - @property - def subroutines(self): - '''Get the subroutine names.''' - return self._subroutines - - def print_debug(self): - '''Basic debugging output about the group.''' - print(self._name) - for subcycle in self._subcycles: - subcycle.print_debug() - - @property - def pset(self): - '''Get the unique physics set of this group.''' - return self._pset - - @pset.setter - def pset(self, value): - self._pset = value - - @property - def parents(self): - '''Get the parent variables for the group.''' - return self._parents - - @parents.setter - def parents(self, value): - self._parents = value - - @property - def arguments(self): - '''Get the argument list of the group.''' - return self._arguments - - @arguments.setter - def arguments(self, value): - self._arguments = value - - -class Subcycle(object): - - def __init__(self, **kwargs): - self._filename = 'sys.stdout' - self._schemes = None - for key, value in kwargs.items(): - setattr(self, "_"+key, value) - - @property - def loop(self): - '''Get the list of loop.''' - return self._loop - - @loop.setter - def loop(self, value): - if not type(value) is int: - raise Exception("Invalid type {0} of argument value, integer expected".format(type(value))) - self._loop = value - - @property - def schemes(self): - '''Get the list of schemes.''' - return self._schemes - - @schemes.setter - def schemes(self, value): - if not type(value) is list: - raise Exception("Invalid type {0} of argument value, list expected".format(type(value))) - self._schemes = value - - def print_debug(self): - '''Basic debugging output about the subcycle.''' - print(self._loop) - for scheme in self._schemes: - print(scheme) - - -############################################################################### -if __name__ == "__main__": - main() diff --git a/scripts/parse_tools/__init__.py b/scripts/parse_tools/__init__.py deleted file mode 100644 index 4d7ec792..00000000 --- a/scripts/parse_tools/__init__.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Public API for the parse_tools library -""" -import sys -import os.path -sys.path.insert(0, os.path.dirname(__file__)) - -# pylint: disable=wrong-import-position -from parse_source import ParseContext, ParseSource -from parse_source import ParseSyntaxError, ParseInternalError -from parse_source import CCPPError, context_string, type_name -from parse_source import unique_standard_name, reset_standard_name_counter -from parse_object import ParseObject -from parse_checkers import check_fortran_id, FORTRAN_ID -from parse_checkers import FORTRAN_DP_RE -from parse_checkers import FORTRAN_SCALAR_REF, FORTRAN_SCALAR_REF_RE -from parse_checkers import check_fortran_ref, check_fortran_literal -from parse_checkers import check_fortran_intrinsic, check_local_name -from parse_checkers import check_diagnostic_id, check_diagnostic_fixed -from parse_checkers import check_fortran_type, check_balanced_paren -from parse_checkers import fortran_list_match -from parse_checkers import registered_fortran_ddt_name -from parse_checkers import register_fortran_ddt_name -from parse_checkers import registered_fortran_ddt_names -from parse_checkers import check_units, check_dimensions, check_cf_standard_name -from parse_checkers import check_default_value, check_valid_values, check_molar_mass -from parse_log import init_log, set_log_level, flush_log -from parse_log import set_log_to_stdout, set_log_to_null -from parse_log import set_log_to_file, verbose -from preprocess import PreprocStack -from xml_tools import find_schema_file, find_schema_version -from xml_tools import read_xml_file, validate_xml_file -from xml_tools import expand_nested_suites, write_xml_file -from fortran_conditional import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX -# pylint: enable=wrong-import-position - -__all__ = [ - 'CCPPError', - 'check_balanced_paren', - 'check_cf_standard_name', - 'check_default_value', - 'check_diagnostic_id', - 'check_diagnostic_fixed', - 'check_dimensions', - 'check_fortran_id', - 'check_fortran_intrinsic', - 'check_fortran_literal', - 'check_fortran_ref', - 'check_fortran_type', - 'check_local_name', - 'check_valid_values', - 'check_molar_mass', - 'context_string', - 'expand_nested_suites', - 'find_schema_file', - 'find_schema_version', - 'flush_log', - 'FORTRAN_DP_RE', - 'FORTRAN_ID', - 'FORTRAN_SCALAR_REF', - 'FORTRAN_SCALAR_REF_RE', - 'init_log', - 'ParseContext', - 'ParseInternalError', - 'ParseSource', - 'ParseSyntaxError', - 'ParseObject', - 'PreprocStack', - 'read_xml_file', - 'register_fortran_ddt_name', - 'registered_fortran_ddt_name', - 'registered_fortran_ddt_names', - 'reset_standard_name_counter', - 'set_log_level', - 'set_log_to_file', - 'set_log_to_null', - 'set_log_to_stdout', - 'type_name', - 'unique_standard_name', - 'validate_xml_file', - 'write_xml_file', - 'FORTRAN_CONDITIONAL_REGEX_WORDS', - 'FORTRAN_CONDITIONAL_REGEX' -] diff --git a/scripts/parse_tools/fortran_conditional.py b/scripts/parse_tools/fortran_conditional.py deleted file mode 100755 index 17ae6859..00000000 --- a/scripts/parse_tools/fortran_conditional.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python3 -# - -"""Definitions to convert a conditional statement in the metadata, expressed in standard names, -into a Fortran conditional (used in an if statement), expressed in local names. -""" - -import re - -FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '(', ')', '==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.', - '.true.', '.false.', '.lt.', '.le.', '.eq.', '.ge.', '.gt.', '.ne.', - '.not.', '.and.', '.or.', '.xor.'] -FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([word.replace('(',r'\(').replace(')', r'\)') for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) diff --git a/scripts/parse_tools/parse_checkers.py b/scripts/parse_tools/parse_checkers.py deleted file mode 100755 index 9a688a13..00000000 --- a/scripts/parse_tools/parse_checkers.py +++ /dev/null @@ -1,1120 +0,0 @@ -#!/usr/bin/env python3 - -"""Helper functions to validate parsed input""" - -# Python library imports -import re -import sys -import os.path -sys.path.insert(0, os.path.dirname(__file__)) -# CCPP framework imports -from parse_source import CCPPError, ParseInternalError - -######################################################################## - -_UNITLESS_REGEX = "1" -_NON_LEADING_ZERO_NUM = r"[1-9]\d*" -_CHAR_WITH_UNDERSCORE = "([a-zA-Z]+_[a-zA-Z]+)+" -_NEGATIVE_NON_LEADING_ZERO_NUM = f"[-]{_NON_LEADING_ZERO_NUM}" -_POSITIVE_NON_LEADING_ZERO_NUM = f"[+]{_NON_LEADING_ZERO_NUM}" -_UNIT_EXPONENT = f"({_NEGATIVE_NON_LEADING_ZERO_NUM}|{_POSITIVE_NON_LEADING_ZERO_NUM}|{_NON_LEADING_ZERO_NUM})" -_UNIT_REGEX = f"[a-zA-Z]+{_UNIT_EXPONENT}?" -_UNITS_REGEX = rf"^({_CHAR_WITH_UNDERSCORE}|{_UNIT_REGEX}(\s{_UNIT_REGEX})*|{_UNITLESS_REGEX})$" -_UNITS_RE = re.compile(_UNITS_REGEX) -_MAX_MOLAR_MASS = 10000.0 - -def check_units(test_val, prop_dict, error): - """Return if a valid unit, otherwise, None - if is True, raise an Exception if is not valid. - >>> check_units('m s-1', None, True) - 'm s-1' - >>> check_units('kg m-3', None, True) - 'kg m-3' - >>> check_units('m2 s-2', None, True) - 'm2 s-2' - >>> check_units('m+2 s-2', None, True) - 'm+2 s-2' - >>> check_units('1', None, True) - '1' - >>> check_units('', None, False) - - >>> check_units('', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '' is not a valid unit - >>> check_units(' ', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '' is not a valid unit - >>> check_units(['foo'], None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: ['foo'] is invalid; not a string - """ - if isinstance(test_val, str): - if _UNITS_RE.match(test_val.strip()) is None: - if error: - raise CCPPError("'{}' is not a valid unit".format(test_val)) - else: - test_val = None - # end if - # end if - else: - if error: - raise CCPPError("'{}' is invalid; not a string".format(test_val)) - else: - test_val = None - # end if - # end if - return test_val - -def check_dimensions(test_val, prop_dict, error, max_len=0): - """Return if a valid dimensions list, otherwise, None - If > 0, each string in must not be longer than - . - if is True, raise an Exception if is not valid. - >>> check_dimensions(["dim1", "dim2name"], None, False) - ['dim1', 'dim2name'] - >>> check_dimensions([":", ":"], None, False) - [':', ':'] - >>> check_dimensions([":", "dim2"], None, False) - [':', 'dim2'] - >>> check_dimensions(["dim1", ":"], None, False) - ['dim1', ':'] - >>> check_dimensions(["8", "::"], None, False) - ['8', '::'] - >>> check_dimensions(['start1:end1', 'start2:end2'], None, False) - ['start1:end1', 'start2:end2'] - >>> check_dimensions(['start1:', 'start2:end2'], None, False) - ['start1:', 'start2:end2'] - >>> check_dimensions(['start1 :end1', 'start2: end2'], None, False) - ['start1 :end1', 'start2: end2'] - >>> check_dimensions(['size(foo)'], None, False) - ['size(foo)'] - >>> check_dimensions(['size(foo,1) '], None, False) - ['size(foo,1) '] - >>> check_dimensions(['size(foo,1'], None, False) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Invalid dimension component, size(foo,1 - >>> check_dimensions(["dim1", "dim2name"], None, False, max_len=5) - - >>> check_dimensions(["dim1", "dim2name"], None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'dim2name' is too long (> 5 chars) - >>> check_dimensions("hi_mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is invalid; not a list - >>> check_dimensions(["1:dim1", "dim2name"], None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '1:dim1 is an invalid dimension name; integer dimension indices not supported - >>> check_dimensions(["ccpp_constant_one:1", "dim2name"], None, True) - ['ccpp_constant_one:1', 'dim2name'] - """ - info_msg = None - if not isinstance(test_val, list): - if error: - raise CCPPError("'{}' is invalid; not a list".format(test_val)) - else: - test_val = None - # end if - else: - for item in test_val: - isplit = item.split(':') - # Check for too many colons - if (len(isplit) > 3): - if error: - errmsg = "'{}' is an invalid dimension range" - raise CCPPError(errmsg.format(item)) - else: - test_val = None - # end if - break - # end if - # Check possible dim styles (a, a:b, a:, :b, :, ::, a:b:c, a::c) - tdims = [x.strip() for x in isplit if len(x) > 0] - starts_at_one = False - if len(tdims) > 0 and tdims[0] == 'ccpp_constant_one': - starts_at_one = True - # end if - is_int = False - for tdim in tdims: - # Check numeric value first - try: - is_int = isinstance(int(tdim), int) - # Allow integer dimensions, but not indices - if is_int: - valid = starts_at_one or len(tdims) == 1 - if not valid: - info_msg = 'integer dimension indices not supported' - # end if - else: - valid = False - # end if - except ValueError as ve: - # Not an integer, try a Fortran ID - valid = check_fortran_id(tdim, None, - error, max_len=max_len) is not None - if not valid: - # Check for size entry -- simple check - tcheck = tdim.strip().lower() - if tcheck[0:4] == 'size': - ploc = check_balanced_paren(tdim[4:]) - if -1 in ploc: - emsg = 'Invalid dimension component, {}' - raise CCPPError(emsg.format(tdim)) - else: - valid = tdim - # end if - # end if - # end if - # End try - if not valid: - if error: - if info_msg: - errmsg = f"'{item}' is an invalid dimension name; {info_msg}" - else: - errmsg = f"'{item}' is an invalid dimension name" - # end if - raise CCPPError(errmsg) - else: - test_val = None - # end if - break - # end if - # end for - # end for - # end if - return test_val - -######################################################################## - -# CF_ID is a string representing the regular expression for CF Standard Names -CF_ID = r"(?i)[a-z][a-z0-9_]*" -__CFID_RE = re.compile(CF_ID+r"$") - -def check_cf_standard_name(test_val, prop_dict, error): - """Return if a valid CF Standard Name, otherwise, None - http://cfconventions.org/Data/cf-standard-names/docs/guidelines.html - if is True, raise an Exception if is not valid. - >>> check_cf_standard_name("hi_mom", None, False) - 'hi_mom' - >>> check_cf_standard_name("hi mom", None, False) - - >>> check_cf_standard_name("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is not a valid CF Standard Name - >>> check_cf_standard_name("", None, False) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: CCPP Standard Name cannot be blank - >>> check_cf_standard_name("_hi_mom", None, False) - - >>> check_cf_standard_name("2pac", None, False) - - >>> check_cf_standard_name("Agood4tranID", None, False) - 'agood4tranid' - >>> check_cf_standard_name("agoodcfid", None, False) - 'agoodcfid' - """ - if len(test_val) == 0: - raise CCPPError("CCPP Standard Name cannot be blank") - else: - match = __CFID_RE.match(test_val) - # end if - if match is None: - if error: - errmsg = "'{}' is not a valid CCPP Standard Name" - raise CCPPError(errmsg.format(test_val)) - else: - test_val = None - # end if - else: - test_val = test_val.lower() - # end if - return test_val - -######################################################################## - -### Fortran-specific parsing helper variables and functions - -######################################################################## - -# FORTRAN_ID is a string representing the regular expression for Fortran names -FORTRAN_ID = r"([A-Za-z][A-Za-z0-9_]*)" -__FID_RE = re.compile(FORTRAN_ID+r"$") -# Note that the scalar array reference expressions below are not really for -# scalar references because a colon can be a placeholder, unlike in Fortran code -__FORTRAN_AID = r"(?:[A-Za-z][A-Za-z0-9_]*)" -__FORT_INT = r"[0-9]+" -__FORT_DIM = r"(?:"+__FORTRAN_AID+r"|[:]|"+__FORT_INT+r")" -__REPEAT_DIM = r"(?:,\s*"+__FORT_DIM+r"\s*)" -__FORTRAN_SCALAR_ARREF = r"[(]\s*("+__FORT_DIM+r"\s*"+__REPEAT_DIM+r"{0,6})[)]" -# FORTRAN_SCALAR_REF: Pattern of a valid Fortran array reference -# NB: Only allows symbols, no expressions and/or function calls -FORTRAN_SCALAR_REF = r"(?:"+FORTRAN_ID+r"\s*"+__FORTRAN_SCALAR_ARREF+r")" -FORTRAN_SCALAR_REF_RE = re.compile(FORTRAN_SCALAR_REF+r"$") -# FORTRAN_FUNCTION_REF: A Fortran function reference -# NB: Currenly does not support function arguments -FORTRAN_FUNCTION_REF = r"(?:"+FORTRAN_ID+r"\s*[(]\s*[)])" -FORTRAN_FUNCTION_REF_RE = re.compile(FORTRAN_FUNCTION_REF) -FORTRAN_INTRINSIC_TYPES = ["integer", "real", "logical", "complex", - "double precision", "character"] -FORTRAN_DP_RE = re.compile(r"(?i)double\s*precision") -FORTRAN_TYPE_RE = re.compile(r"(?i)type\s*\(\s*("+FORTRAN_ID+r")\s*\)") - -_REGISTERED_FORTRAN_DDT_NAMES = ["ccpp_constituent_prop_ptr_t"] - -######################################################################## - -def check_fortran_id(test_val, prop_dict, error, max_len=0): - """Return if a valid Fortran identifier, otherwise, None - If > 0, must not be longer than . - if is True, raise an Exception if is not valid. - >>> check_fortran_id("hi_mom", None, False) - 'hi_mom' - >>> check_fortran_id("hi_mom", None, False, max_len=5) - - >>> check_fortran_id("hi_mom", None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is too long (> 5 chars) - >>> check_fortran_id("hi mom", None, False) - - >>> check_fortran_id("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is not a valid Fortran identifier - >>> check_fortran_id("", None, False) - - >>> check_fortran_id("_hi_mom", None, False) - - >>> check_fortran_id("2pac", None, False) - - >>> check_fortran_id("Agood4tranID", None, False) - 'Agood4tranID' - """ - match = __FID_RE.match(test_val) - if match is None: - if error: - raise CCPPError("'{}' is not a valid Fortran identifier".format(test_val)) - else: - test_val = None - # end if - elif (max_len > 0) and (len(test_val) > max_len): - if error: - raise CCPPError("'{}' is too long (> {} chars)".format(test_val, max_len)) - else: - test_val = None - # end if - # end if - return test_val - -######################################################################## - -def fortran_list_match(test_str): - """Check if could be a list of Fortran expressions. - The list must be enclosed in parentheses and separated by commas. - If the list appears okay, return the items (for further checking) - >>> fortran_list_match('(ccpp_constant_one:dim1)') - ['ccpp_constant_one:dim1'] - >>> fortran_list_match('(foo, bar)') - ['foo', 'bar'] - >>> fortran_list_match('()') - [''] - >>> fortran_list_match('(foo, ,)') - - >>> fortran_list_match('foo, bar') - - >>> fortran_list_match('(foo, bar') - - """ - parens, parene = check_balanced_paren(test_str) - if (parens >= 0) and (parene > parens): - litems = [x.strip() for x in test_str[parens+1:parene].split(',')] - if (len(litems) > 1) and (min([len(x) for x in litems]) == 0): - litems = None - # end if - else: - litems = None - # end if - return litems - -######################################################################## - -def check_fortran_ref(test_val, prop_dict, error, max_len=0): - """Return if a valid simple Fortran variable reference, - otherwise, None. A simple Fortran variable reference is defined as - a scalar id or a scalar array reference. - if is True, raise an Exception if is not valid. - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(1) - 'foo' - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2) - 'bar, baz ' - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2).split(',')[0].strip() - 'bar' - >>> FORTRAN_SCALAR_REF_RE.match("foo( :, baz )").group(2).split(',')[0].strip() - ':' - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2).split(',')[1].strip() - 'baz' - >>> check_fortran_ref("hi_mom", None, False) - 'hi_mom' - >>> check_fortran_ref("hi_mom", None, False, max_len=5) - - >>> check_fortran_ref("hi_mom", None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is too long (> 5 chars) - >>> check_fortran_ref("hi mom", None, False) - - >>> check_fortran_ref("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is not a valid Fortran identifier - >>> check_fortran_ref("", None, False) - - >>> check_fortran_ref("_hi_mom", None, False) - - >>> check_fortran_ref("2pac", None, False) - - >>> check_fortran_ref("Agood4tranID", None, False) - 'Agood4tranID' - >>> check_fortran_ref("foo(bar)", None, False) - 'foo(bar)' - >>> check_fortran_ref("foo( bar, baz )", None, False) - 'foo( bar, baz )' - >>> check_fortran_ref("foo( :, baz )", None, False) - 'foo( :, baz )' - >>> check_fortran_ref("foo( bar, )", None, False) - - >>> check_fortran_ref("foo( bar, )", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'foo( bar, )' is not a valid Fortran scalar reference - >>> check_fortran_ref("foo()", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'foo()' is not a valid Fortran scalar reference - >>> check_fortran_ref("foo(bar, bazz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'bazz' is too long (> 3 chars) in foo(bar, bazz) - >>> check_fortran_ref("foo(barr, baz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'bazr' is too long (> 3 chars) in foo(barr, baz) - >>> check_fortran_ref("fooo(bar, baz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'foo' is too long (> 3 chars) in fooo(bar, baz) - """ - idval = check_fortran_id(test_val, prop_dict, False, max_len=max_len) - if idval is None: - match = FORTRAN_SCALAR_REF_RE.match(test_val) - if match is None: - if error: - emsg = "'{}' is not a valid Fortran scalar reference" - raise CCPPError(emsg.format(test_val)) - else: - test_val = None - # end if - elif max_len > 0: - tokens = test_val.strip().rstrip(')').split('(') - tokens = [tokens[0].strip()] + [x.strip() - for x in tokens[1].split(',')] - for token in tokens: - if len(token) > max_len: - if error: - emsg = "'{}' is too long (> {} chars) in {}" - raise CCPPError(emsg.format(token, max_len, test_val)) - else: - test_val = None - break - # end if - # end if - # end for - # end if - # end if - return test_val - -######################################################################## - -def check_local_name(test_val, prop_dict, error, max_len=0): - """Return if a valid simple Fortran variable reference, - or Fortran constant, otherwise, None. - A simple Fortran variable reference is defined as a scalar id or a - scalar array reference. - A constant is only valid if is not None, the 'protected' - property is present and True, and the 'type' property matches the - type of . - if is True, raise an Exception if is not valid. - >>> check_local_name("hi_mom", None, error=False) - 'hi_mom' - >>> check_local_name('122', {'protected':True,'type':'integer'}, error=False) - '122' - >>> check_local_name('122', None, error=False) - - >>> check_local_name('122', {}, error=False) - - >>> check_local_name('122', {'protected':False,'type':'integer'}, error=False) - - >>> check_local_name('122', {'protected':True,'type':'real'}, error=False) - - >>> check_local_name('-122.e4', {'protected':True,'type':'real'}, error=False) - '-122.e4' - >>> check_local_name('-122.', {'protected':True,'type':'real','kind':'kp'}, error=False) - - >>> check_local_name('-122._kp', {'protected':True,'type':'real','kind':'kp'}, error=False) - '-122._kp' - >>> check_local_name('q(:,:,index_of_water_vapor_specific_humidity)', {}, error=False) - 'q(:,:,index_of_water_vapor_specific_humidity)' - """ - valid_val = None - # First check for a constant - if (prop_dict is not None) and ('protected' in prop_dict): - protected = prop_dict['protected'] - else: - protected = False - # end if - if (prop_dict is not None) and ('type' in prop_dict): - vtype = prop_dict['type'] - else: - vtype = "" - # end if - if (prop_dict is not None) and ('kind' in prop_dict): - kind = prop_dict['kind'] - else: - kind = "" - # end if - if protected and vtype and check_fortran_literal(test_val, vtype, kind): - valid_val = test_val - # end if - if valid_val is None: - valid_val = check_fortran_ref(test_val, prop_dict, error, max_len=max_len) - # end if - return valid_val - - -######################################################################## - -def check_fortran_intrinsic(typestr, error=False): - """Return if a valid Fortran intrinsic type, otherwise, None - if is True, raise an Exception if is not valid. - >>> check_fortran_intrinsic("real", error=False) - 'real' - >>> check_fortran_intrinsic("complex") - 'complex' - >>> check_fortran_intrinsic("integer") - 'integer' - >>> check_fortran_intrinsic("InteGer") - 'InteGer' - >>> check_fortran_intrinsic("logical") - 'logical' - >>> check_fortran_intrinsic("character") - 'character' - >>> check_fortran_intrinsic("double precision") - 'double precision' - >>> check_fortran_intrinsic("double precision") - 'double precision' - >>> check_fortran_intrinsic("doubleprecision") - 'doubleprecision' - >>> check_fortran_intrinsic("char", error=True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'char' is not a valid Fortran type - >>> check_fortran_intrinsic("int") - - >>> check_fortran_intrinsic("char", error=False) - - >>> check_fortran_intrinsic("type") - - >>> check_fortran_intrinsic("complex(kind=r8)") - - """ - chk_type = typestr.strip().lower() - match = chk_type in FORTRAN_INTRINSIC_TYPES - if (not match) and (chk_type[0:6] == 'double'): - # Special case for double precision - match = FORTRAN_DP_RE.match(chk_type) is not None - # End if - if not match: - if error: - raise CCPPError("'{}' is not a valid Fortran type".format(typestr)) - else: - typestr = None - # end if - # end if - return typestr - -######################################################################## - -def check_fortran_type(typestr, prop_dict, error): - """Return if a valid Fortran type, otherwise, None - if is True, raise an Exception if is not valid. - >>> check_fortran_type("real", None, False) - 'real' - >>> check_fortran_type("integer", None, False) - 'integer' - >>> check_fortran_type("InteGer", None, False) - 'InteGer' - >>> check_fortran_type("character", None, False) - 'character' - >>> check_fortran_type("double precision", None, False) - 'double precision' - >>> check_fortran_type("double precision", None, False) - 'double precision' - >>> check_fortran_type("doubleprecision", None, False) - 'doubleprecision' - >>> check_fortran_type("complex", None, False) - 'complex' - >>> check_fortran_type("char", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'char' is not a valid Fortran type - >>> check_fortran_type("int", None, False) - - >>> check_fortran_type("char", {}, False) - - >>> check_fortran_type("type", None, False) - - >>> check_fortran_type("type", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'type' is not a valid derived Fortran type - >>> check_fortran_type("type(hi mom)", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'type(hi mom)' is not a valid derived Fortran type - """ - dt = "" - match = check_fortran_intrinsic(typestr, error=False) - if match is None: - match = registered_fortran_ddt_name(typestr) - dt = " derived" - # end if - if match is None: - if error: - emsg = "'{}' is not a valid{} Fortran type" - raise CCPPError(emsg.format(typestr, dt)) - else: - typestr = None - # end if - # end if - return typestr - -######################################################################## - -def check_fortran_literal(value, typestr, kind): - """Return True iff is a valid Fortran literal of type, . - Note: no attempt is made to handle the older D syntax for real literals. - To promote clean coding, real values MUST have a decimal point, however, - this check is not available for the complex type so we just require - the two components to either both be integers or both be reals. - If is not an empty string, it is required to be present (i.e., if - == 'kind_phys', should be of the form, 123.4_kind_phys) - >>> check_fortran_literal("123", "integer", "") - True - >>> check_fortran_literal("123", "INTEGER", "") - True - >>> check_fortran_literal("-123", "integer", "") - True - >>> check_fortran_literal("+123", "integer", "") - True - >>> check_fortran_literal("+123", "integer", "kind_int") - False - >>> check_fortran_literal("+123_kind_int", "integer", "kind_int") - True - >>> check_fortran_literal("+123_int", "integer", "kind_int") - False - >>> check_fortran_literal("123", "real", "") - False - >>> check_fortran_literal("123.", "real", "") - True - >>> check_fortran_literal("123.45", "real", "kind_phys") - False - >>> check_fortran_literal("123.45_8", "real", "kind_phys") - False - >>> check_fortran_literal("123.45_kind_phys", "real", "kind_phys") - True - >>> check_fortran_literal("123", "double precision", "") - False - >>> check_fortran_literal("123.", "doubleprecision", "") - True - >>> check_fortran_literal("123.45", "double precision", "kind_phys") - False - >>> check_fortran_literal("123.45_8", "doubleprecision", "kind_phys") - False - >>> check_fortran_literal("123.45_kp", "doubleprecision", "kp") - True - >>> check_fortran_literal("123", "logical", "") - False - >>> check_fortran_literal(".true.", "logical", "") - True - >>> check_fortran_literal(".false.", "logical", "") - True - >>> check_fortran_literal("T", "logical", "") - False - >>> check_fortran_literal("F", "logical", "") - False - >>> check_fortran_literal(".TRUE.", "logical", "kind_log") - False - >>> check_fortran_literal(".TRUE._kind_log", "logical", "kind_log") - True - >>> check_fortran_literal("(123.,456.)", "complex", "") - True - >>> check_fortran_literal("(123. , 456.)", "complex", "") - True - >>> check_fortran_literal("(123.,456", "complex", "") - False - >>> check_fortran_literal("(123. , 456.)", "complex", "kp") - False - >>> check_fortran_literal("(123._kp , 456)", "complex", "kp") - False - >>> check_fortran_literal("(123._kp , 456._kp)", "complex", "kp") - True - >>> check_fortran_literal("'hi mom'", "character", "") - True - >>> check_fortran_literal("'hi mom", "character", "") - False - >>> check_fortran_literal('"hi mom"', "character", "") - True - >>> check_fortran_literal('"hi""mom"', "character", "") - True - >>> check_fortran_literal('"hi" "mom"', "character", "") - False - >>> check_fortran_literal("'hi''there''mom'", "character", "") - True - >>> check_fortran_literal("'hi mom'", "character", "kc") - False - >>> check_fortran_literal("kc_'hi mom'", "character", "kc") - True - >>> check_fortran_literal("123._kp", "float", "kp") #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: ERROR: 'float' is not a Fortran intrinsic type - """ - valid = True - if FORTRAN_DP_RE.match(typestr.strip()) is not None: - vtype = 'real' - else: - vtype = typestr.lower() - # end if - # Check complex first - if vtype == 'complex': - cvals = value.strip().split(',') - if len(cvals) == 2: - tp = 'integer' - if ('.' in cvals[0]) and ('.' in cvals[1]): - tp = 'real' - elif ('.' in cvals[0]) or ('.' in cvals[1]): - valid = False - # end if - if (cvals[0][0] == '(') and (cvals[1][-1] == ')'): - valid = valid and check_fortran_literal(cvals[0][1:], tp, kind) - valid = valid and check_fortran_literal(cvals[1][:-1], tp, kind) - else: - valid = False - # end if - else: - valid = False - elif valid: - vparts = value.strip().split('_') - if vtype == 'character': - if len(vparts) > 1: - val = vparts[-1] - vkind = '_'.join(vparts[0:-1]) - else: - val = vparts[0] - vkind = '' - # end if - else: - val = vparts[0] - if len(vparts) > 1: - vkind = '_'.join(vparts[1:]) - else: - vkind = '' - # end if - # end if - if vkind != kind.lower(): - valid = False - # end if, kind is okay, check value - if valid and (vtype == 'integer'): - try: - vtest = int(val) - except ValueError as ve: - valid = False - # End try - elif valid and (vtype == 'real'): - if '.' not in val: - valid = False - else: - try: - vtest = float(val) - except ValueError as ve: - valid = False - # End try - # end if - elif valid and (vtype == 'logical'): - valid = (val.upper() == '.TRUE.') or (val.upper() == '.FALSE.') - elif valid and (vtype == 'character'): - sep = val[0] - cparts = val.split(sep) - # We must have balanced delimiters - if len(cparts)%2 == 0: - valid = False - else: - for index in range(len(cparts)): - if (index%2 == 0) and (len(cparts[index]) > 0): - valid = False - break - # end if - # end for - # end if (else okay) - elif valid: - errmsg = "ERROR: '{}' is not a Fortran intrinsic type" - raise ParseInternalError(errmsg.format(typestr)) - # end if (no else) - # end if - return valid - -def check_default_value(test_val, prop_dict, error): - """Return if a valid default value for a CCPP field, - otherwise, None. - If is True, raise an Exception if is not valid. - A valid value is determined by the 'type' of the variable. It is an - error for there to be no 'type' property in . - >>> check_default_value('314', {'type':'integer'}, False) - '314' - >>> check_default_value('314', {'type':'integer'}, True) - '314' - >>> check_default_value('314', {'type':'integer', 'kind':'ikind'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 314 is not a valid Fortran integer of kind, ikind - >>> check_default_value('314_ikind', {'type':'integer', 'kind':'ikind'}, True) - '314_ikind' - >>> check_default_value('314', {'type':'real'}, False) - - >>> check_default_value('314', {'type':'real'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 314 is not a valid Fortran real - >>> check_default_value('3.14', {'type':'real'}, False) - '3.14' - >>> check_default_value('314', {'tipe':'integer'}, False) - - >>> check_default_value('314', {'local_name':'foo'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: foo does not have a 'type' attribute - >>> check_default_value('314', {'tipe':'integer'}, False) - - >>> check_default_value('314', None, True) - '314' - """ - valid = None - if prop_dict and ('type' in prop_dict): - valid = test_val - var_type = prop_dict['type'].lower().strip() - if 'kind' in prop_dict: - vkind = prop_dict['kind'].lower().strip() - else: - vkind = '' - # end if - if not check_fortran_literal(test_val, var_type, vkind): - valid = None - if error: - emsg = '{} is not a valid Fortran {}' - if vkind: - emsg += ' of kind, {}' - raise CCPPError(emsg.format(test_val, var_type, vkind)) - # end if - # end if (no else, is okay) - elif prop_dict is None: - # Special case for checks during parsing, always pass - valid = test_val - elif error: - emsg = "{} does not have a 'type' attribute" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(lname)) - # end if - return valid - -def check_valid_values(test_val, prop_dict, error): - """Return if a valid 'valid_values' attribute value, - otherwise, None. - If is True, raise an Exception if is not valid. - """ - raise ParseInternalError("NOT IMPLEMENTED") - -def check_diagnostic_fixed(test_val, prop_dict, error): - """Return if a valid descriptor for a CCPP diagnostic, - otherwise, None. - If is True, raise an Exception if is not valid. - A fixed diagnostic name is any Fortran identifier, however, it is - an error to specify both 'diagnostic_name' and 'diagnostic_name_fixed'. - >>> check_diagnostic_fixed("foo", {'diagnostic_name_fixed' : 'foo'}, False) - 'foo' - >>> check_diagnostic_fixed("foo", {'diagnostic_name_fixed' : 'foo'}, True) - 'foo' - >>> check_diagnostic_fixed("foo", {'diagnostic_name' : 'foo'}, False) - - >>> check_diagnostic_fixed("foo", {'diagnostic_name':'','local_name':'hi','standard_name':'mom'}, True) - 'foo' - >>> check_diagnostic_fixed("foo", {'diagnostic_name':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: hi (mom) cannot have both 'diagnostic_name' and 'diagnostic_name_fixed' attributes - >>> check_diagnostic_fixed("2foo", {'diagnostic_name_fixed':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '2foo' (hi) is not a valid fixed diagnostic name - """ - valid = test_val - if (prop_dict and ('diagnostic_name' in prop_dict) and - prop_dict['diagnostic_name']): - valid = None - if error: - emsg = "{} ({}) cannot have both 'diagnostic_name' and " - emsg += "'diagnostic_name_fixed' attributes" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - if 'standard_name' in prop_dict: - sname = prop_dict['standard_name'] - else: - sname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(lname, sname)) - # end if - elif check_fortran_id(test_val, prop_dict, False) is None: - valid = None - if error: - emsg = "'{}' ({}) is not a valid fixed diagnostic name" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(test_val, lname)) - # end if - # end if - return valid - -######################################################################## - -_DIAG_PRE = r"("+FORTRAN_ID+")?" -_DIAG_SUFF = r"([_0-9A-Za-z]+)?" -_DIAG_PROP = r"((\${process}|\${scheme_name})"+_DIAG_SUFF+r")" -_DIAG_RE = re.compile(_DIAG_PRE+_DIAG_PROP+r"?$") - -def check_diagnostic_id(test_val, prop_dict, error): - """Return if a valid descriptor for a CCPP diagnostic, - otherwise, None. - If is True, raise an Exception if is not valid. - A diagnostic name is a Fortran identifier with the optional - addition of one variable substitution. - A variable substitution is a substring of the form of either: - ${process}: The scheme process name will be substituted for this - substring. If this substring is included, it is an error for - there to be no process specified by the scheme (although this - error cannot be detected by this routine). - ${scheme_name}: The scheme name will be substituted for this substring. - It is an error to specify both 'diagnostic_name' and - 'diagnostic_name_fixed'. - >>> check_diagnostic_id("foo", {'diagnostic_name' : 'foo'}, False) - 'foo' - >>> check_diagnostic_id("foo", {'diagnostic_name' : 'foo'}, True) - 'foo' - >>> check_diagnostic_id("foo", {'diagnostic_name_fixed' : 'foo'}, False) - - >>> check_diagnostic_id("foo_${process}", {}, False) - 'foo_${process}' - >>> check_diagnostic_id("foo_${process}_2bad", {}, False) - 'foo_${process}_2bad' - >>> check_diagnostic_id("${process}_2bad", {}, False) - '${process}_2bad' - >>> check_diagnostic_id("foo_${scheme_name}", {}, False) - 'foo_${scheme_name}' - >>> check_diagnostic_id("foo_${scheme_name}_2bad", {}, False) - 'foo_${scheme_name}_2bad' - >>> check_diagnostic_id("${scheme_name}_suff", {}, False) - '${scheme_name}_suff' - >>> check_diagnostic_id("pref_${scheme}_suff", {}, False) - - >>> check_diagnostic_id("pref_${scheme_name_suff", {}, False) - - >>> check_diagnostic_id("pref_$scheme_name}_suff", {}, False) - - >>> check_diagnostic_id("pref_{scheme_name}_suff", {}, False) - - >>> check_diagnostic_id("foo", {'diagnostic_name_fixed':'','local_name':'hi','standard_name':'mom'}, True) - 'foo' - >>> check_diagnostic_id("foo", {'diagnostic_name_fixed':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: hi (mom) cannot have both 'diagnostic_name' and 'diagnostic_name_fixed' attributes - >>> check_diagnostic_id("2foo", {'diagnostic_name':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '2foo' (hi) is not a valid diagnostic name - """ - if (prop_dict and ('diagnostic_name_fixed' in prop_dict) and - prop_dict['diagnostic_name_fixed']): - valid = None - if error: - emsg = "{} ({}) cannot have both 'diagnostic_name' and " - emsg += "'diagnostic_name_fixed' attributes" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - if 'standard_name' in prop_dict: - sname = prop_dict['standard_name'] - else: - sname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(lname, sname)) - # end if - else: - match = _DIAG_RE.match(test_val) - if match is None: - valid = None - if error: - emsg = "'{}' is not a valid diagnostic_name value" - raise CCPPError(emsg.format(test_val)) - # end if - else: - valid = test_val - # end if - # end if - return valid - -######################################################################## - -def check_molar_mass(test_val, prop_dict, error): - """Return if valid molar mass, otherwise, None - if is True, raise an Exception if is not valid. - >>> check_molar_mass('1', None, True) - 1.0 - >>> check_molar_mass('1.0', None, True) - 1.0 - >>> check_molar_mass('1.0', None, False) - 1.0 - >>> check_molar_mass('-1', None, False) - - >>> check_molar_mass('-1.0', None, False) - - >>> check_molar_mass('string', None, False) - - >>> check_molar_mass(10001, None, False) - - >>> check_molar_mass('-1', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '-1' is not a valid molar mass - >>> check_molar_mass('-1.0', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '-1.0' is not a valid molar mass - >>> check_molar_mass('string', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '-1.0' is not a valid molar mass - >>> check_molar_mass(10001, None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '10001' is not a valid molar mass - """ - # Check if input value is an int or float - try: - test_val = float(test_val) - if test_val < 0.0 or test_val > _MAX_MOLAR_MASS: - if error: - raise CCPPError(f"{test_val} is not a valid molar mass") - else: - test_val = None - # end if - # end if - except: - # not an int or float, conditionally throw error - if error: - raise CCPPError(f"{test_val} is invalid; not a float or int") - else: - test_val=None - # end if - # end try - return test_val - -######################################################################## - -def check_balanced_paren(string, start=0, error=False): - """Return indices delineating a balance set of parentheses. - Parentheses in character context do not count. - Left parenthesis search begins at . - Return start and end indices if found - If no parentheses are found, return (-1, -1). - If a left parenthesis is found but no balancing right, return (begin, -1) - where begin is the index where the left parenthesis was found. - If error is True, raise a CCPPError. - >>> check_balanced_paren("foo") - (-1, -1) - >>> check_balanced_paren("(foo, bar)") - (0, 9) - >>> check_balanced_paren("( (foo, bar) )", start=1) - (2, 11) - >>> check_balanced_paren("(size(foo,1), qux)") - (0, 17) - >>> check_balanced_paren("(foo('bar()'))") - (0, 13) - >>> check_balanced_paren("(foo('bar()')") - (0, -1) - >>> check_balanced_paren("(foo('bar()')", error=True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: ERROR: Unbalanced parenthesis in '(foo('bar()')' - """ - index = start - begin = -1 - end = -1 - depth = 0 - inchar = None - str_len = len(string) - while index < str_len: - if (string[index] == '"') or (string[index] == "'"): - if inchar == string[index]: - inchar = None - elif inchar is None: - inchar = string[index] - # else in character context, keep going - # end if - elif inchar is not None: - # In character context, keep going - pass - elif string[index] == '(': - if depth == 0: - begin = index - # end if - depth = depth + 1 - if depth == 0: - break - # end if - elif string[index] == ')': - depth = depth - 1 - if depth == 0: - end = index - break - # end if - # else just keep going - # end if - index = index + 1 - # End while - if (begin >= 0) and (end < 0) and error: - raise CCPPError("ERROR: Unbalanced parenthesis in '{}'".format(string)) - # end if - return begin, end - -######################################################################## - -def registered_fortran_ddt_names(): - return _REGISTERED_FORTRAN_DDT_NAMES - -######################################################################## - -def registered_fortran_ddt_name(name): - if name in _REGISTERED_FORTRAN_DDT_NAMES: - return name - else: - return None - -######################################################################## - -def register_fortran_ddt_name(name): - if name not in _REGISTERED_FORTRAN_DDT_NAMES: - _REGISTERED_FORTRAN_DDT_NAMES.append(name) - -######################################################################## - -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - import doctest - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/parse_tools/parse_log.py b/scripts/parse_tools/parse_log.py deleted file mode 100644 index f85a5d09..00000000 --- a/scripts/parse_tools/parse_log.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -"""Shared logger for parse processes""" - -# Python library imports -import logging -# CCPP framework imports - -def init_log(name, level=None): - """Initialize a new logger object""" - logger = logging.getLogger(name) - # Turn logging to WARNING if not set - llevel = logger.getEffectiveLevel() - if (level is None) and (llevel == logging.NOTSET): - logger.setLevel(logging.WARNING) - elif level: - logger.setLevel(level) - # End if - set_log_to_stdout(logger) - return logger - -def set_log_level(logger, level): - """Set the logging level of to """ - logger.setLevel(level) - -def remove_handlers(logger): - """Remove all handlers from """ - for handler in list(logger.handlers): - logger.removeHandler(handler) - -def set_log_to_stdout(logger): - """Set to log to standard out""" - remove_handlers(logger) - logger.addHandler(logging.StreamHandler()) - -def set_log_to_null(logger): - """Set to log to NULL""" - remove_handlers(logger) - logger.addHandler(logging.NullHandler()) - -def set_log_to_file(logger, filename): - """Set to log to """ - remove_handlers(logger) - logger.addHandler(logging.StreamHandler()) - -def flush_log(logger): - """Flush all pending output from """ - for handler in list(logger.handlers): - handler.flush() - -def verbose(logger): - """Return true if debug is enabled for this logger""" - return logger.isEnabledFor(logging.DEBUG) diff --git a/scripts/parse_tools/parse_object.py b/scripts/parse_tools/parse_object.py deleted file mode 100644 index 2c5f72f5..00000000 --- a/scripts/parse_tools/parse_object.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -"""A module for the base, ParseObject class""" - -# CCPP framework imports -from parse_source import ParseContext, CCPPError, context_string - -######################################################################## - -class ParseObject(ParseContext): - """ParseObject is a simple class that keeps track of an object's - place in a file and safely produces lines from an array of lines - >>> ParseObject('foobar.F90', []) #doctest: +ELLIPSIS - - >>> ParseObject('foobar.F90', []).filename - 'foobar.F90' - >>> ParseObject('foobar.F90', ["##hi mom",], line_start=1).curr_line() - (None, 1) - >>> ParseObject('foobar.F90', ["first line","## hi mom"], line_start=1).curr_line() - ('## hi mom', 1) - >>> ParseObject('foobar.F90', ["##hi mom",], line_start=1).next_line() - (None, 1) - >>> ParseObject('foobar.F90', ["##first line","## hi mom"], line_start=1).next_line() - ('## hi mom', 1) - >>> ParseObject('foobar.F90', ["## hi \\\\","mom"], line_start=0).next_line() - ('## hi mom', 0) - >>> ParseObject('foobar.F90', ["line1","##line2","## hi mom"], line_start=2).next_line() - ('## hi mom', 2) - >>> ParseObject('foobar.F90', ["## hi \\\\","there \\\\","mom"], line_start=0).next_line() - ('## hi there mom', 0) - >>> ParseObject('foobar.F90', ["!! line1","!! hi mom"], line_start=1).next_line() - ('!! hi mom', 1) - """ - - _max_errors = 32 - - def __init__(self, filename, lines_in, line_start=0): - """Initialize this ParseObject""" - self.__lines = lines_in - self.__line_start = line_start - self.__line_end = line_start - self.__line_next = line_start - self.__num_lines = len(self.__lines) - self.__error_message = "" - self.__num_errors = 0 - super().__init__(linenum=line_start, filename=filename) - - @property - def first_line_num(self): - """Return the first line parsed""" - return self.__line_start - - @property - def last_line_num(self): - """Return the last line parsed""" - return self.__line_end - - def valid_line(self): - """Return True if the current line is valid""" - return (self.line_num >= 0) and (self.line_num < self.__num_lines) - - @property - def error_message(self): - """Return this object's error message""" - return self.__error_message - - def curr_line(self): - """Return the current line (if valid) and the current line number. - If the current line is invalid, return None""" - valid_line = self.valid_line() - _curr_line = None - _my_curr_lineno = self.line_num - if valid_line: - try: - _curr_line = self.__lines[self.line_num].rstrip() - self.__line_next = self.line_num + 1 - self.__line_end = self.__line_next - except CCPPError: - self.add_syntax_err("line", self.line_num) - valid_line = False - # end if - # We allow continuation self.__lines (ending with a single backslash) - if valid_line and _curr_line.endswith('\\'): - next_line, _ = self.next_line() - if next_line is None: - # We ran out of lines, just strip the backslash - _curr_line = _curr_line[0:len(_curr_line)-1] - else: - _curr_line = _curr_line[0:len(_curr_line)-1] + next_line - # end if - # end if - # curr_line should not change the line number - self.line_num = _my_curr_lineno - return _curr_line, self.line_num - - def next_line(self): - """Return the next line in our file (if valid) and the next line's - line number. If the next line is not valid, return None""" - self.line_num = self.__line_next - return self.curr_line() - - def peek_line(self, line_num): - """Return the text of without advancing to that line. - if is out of bounds, return None.""" - if (line_num >= 0) and (line_num < len(self.__lines)): - return self.__lines[line_num] - # end if - return None - - def add_syntax_err(self, token_type, token=None, skip_context=False): - """Add a ParseSyntaxError-type message to this object's error - log, separating it from any previous messages with a newline.""" - if self.__error_message: - if self.__num_errors == self._max_errors: - self.__error_message += '\nMaximum number of errors exceeded' - self.line_num = self.__num_lines # Intentionally walk off end - self.__line_next = self.line_num - elif self.__num_errors > self._max_errors: - # Oops, something went wrong, panic! - raise CCPPError(self.error_message) - # end if - self.__error_message += '\n' - # end if - if self.__num_errors < self._max_errors: - if skip_context: - cstr = "" - else: - cstr = context_string(self) - # end if - if token is None: - self.__error_message += "{}{}".format(token_type, cstr) - else: - self.__error_message += "Invalid {}, '{}'{}".format(token_type, - token, cstr) - # end if - # end if - self.__num_errors += 1 - - def reset_pos(self, line_start=0): - """Attempt to set the current file position to . - If is out of bounds, raise an exception.""" - if (line_start < 0) or (line_start >= self.__num_lines): - emsg = 'Attempt to reset_pos to non-existent line, {}' - raise CCPPError(emsg.format(line_start)) - # end if - self.line_num = line_start - self.__line_next = line_start - - def write_line(self, line_num, line): - """Overwrite line, with . - If is out of bounds, raise an exception.""" - if (line_num < 0) or (line_num >= len(self.__lines)): - emsg = 'Attempt to write non-existent line, {}' - raise CCPPError(emsg.format(line_num)) - # end if - self.__lines[line_num] = line - - def __del__(self): - """Attempt to cleanup memory used by this object""" - try: - del self.__lines - del self.regions - except Exception: - pass # Python does not guarantee much about __del__ conditions - # end try - -######################################################################## diff --git a/scripts/parse_tools/parse_source.py b/scripts/parse_tools/parse_source.py deleted file mode 100644 index 1a4082cb..00000000 --- a/scripts/parse_tools/parse_source.py +++ /dev/null @@ -1,411 +0,0 @@ -#!/usr/bin/env python3 - -"""Classes to aid the parsing process""" - - -# Python library imports -from collections.abc import Iterable -# end if -import copy -import sys -import os.path -import logging -# CCPP framework imports - -class _StdNameCounter(): - """Class to hold a global counter to avoid using global keyword""" - __SNAME_NUM = 0 # Counter for unique standard names - - @classmethod - def new_stdname_number(cls): - """Increment and return the global counter.""" - _StdNameCounter.__SNAME_NUM += 1 - return _StdNameCounter.__SNAME_NUM - - @classmethod - def reset_stdname_counter(cls, reset_val=0): - """Reset the global counter to """ - _StdNameCounter.__SNAME_NUM = reset_val - -############################################################################### -def unique_standard_name(): -############################################################################### - """ - Return a unique standard name. - """ - return 'enter_standard_name_{}'.format(_StdNameCounter.new_stdname_number()) - -############################################################################### -def reset_standard_name_counter(): -############################################################################### - """ - Reset the unique_standard_name counter so that future calls to - unique_standard name will restart. - """ - _StdNameCounter.reset_stdname_counter() - -############################################################################### -def context_string(context=None, with_comma=True, nodir=False): -############################################################################### - """Return a context string if is not None otherwise, return - an empty string. - if with_comma is True, prepend string with ', at ' or ', in '. - >>> context_string() - '' - >>> context_string(with_comma=True) - '' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False) - 'dir/source.F90:33' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90"), with_comma=True) - ', at dir/source.F90:33' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90")) - ', at dir/source.F90:33' - >>> context_string(context= ParseContext(filename="dir/source.F90"), with_comma=False) - 'dir/source.F90' - >>> context_string(context= ParseContext(filename="dir/source.F90"), with_comma=True) - ', in dir/source.F90' - >>> context_string(context= ParseContext(filename="dir/source.F90")) - ', in dir/source.F90' - >>> context_string(nodir=True) - '' - >>> context_string(with_comma=True, nodir=True) - '' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False, nodir=True) - 'source.F90:33' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90"), with_comma=True, nodir=True) - ', at source.F90:33' - >>> context_string(context= ParseContext(linenum=32, filename="dir/source.F90"), nodir=True) - ', at source.F90:33' - >>> context_string(context= ParseContext(filename="dir/source.F90"), with_comma=False, nodir=True) - 'source.F90' - >>> context_string(context= ParseContext(filename="dir/source.F90"), with_comma=True, nodir=True) - ', in source.F90' - >>> context_string(context= ParseContext(filename="dir/source.F90"), nodir=True) - ', in source.F90' - """ - if context is None: - where_str = '' - elif context.line_num < 0: - where_str = 'in ' - else: - where_str = 'at ' - # End if - if (context is not None) and with_comma: - comma = ', ' - else: - comma = '' - where_str = '' # Override previous setting - # End if - if context is None: - spec = '' - elif nodir: - spec = '{ctx:nodir}' - else: - spec = '{ctx}' - # End if - if context is None: - cstr = "" - else: - cstr = '{comma}{where_str}' + spec - # End if - return cstr.format(comma=comma, where_str=where_str, ctx=context) - -############################################################################### -def type_name(obj): -############################################################################### - """Return the name of the type of """ - return type(obj).__name__ - -############################################################################### -class CCPPError(ValueError): - """Class so programs can log user errors without backtrace""" - def __init__(self, message): - """Initialize this exception""" - logging.shutdown() - super(CCPPError, self).__init__(message) - -######################################################################## - -class ParseSyntaxError(CCPPError): - """Exception that is aware of parsing context""" - def __init__(self, token_type, token=None, context=None): - """Initialize this exception""" - logging.shutdown() - cstr = context_string(context) - if token is None: - message = "{}{}".format(token_type, cstr) - else: - message = "Invalid {}, '{}'{}".format(token_type, token, cstr) - # End if - super(ParseSyntaxError, self).__init__(message) - -######################################################################## - -class ParseInternalError(Exception): - """Exception for internal parser use errors - Note that this error will not be trapped by programs such as ccpp_capgen - """ - def __init__(self, errmsg, context=None): - """Initialize this exception""" - logging.shutdown() - message = "{}{}".format(errmsg, context_string(context)) - super(ParseInternalError, self).__init__(message) - -######################################################################## - -class ParseContextError(CCPPError): - """Exception for errors using ParseContext""" - def __init__(self, errmsg, context): - """Initialize this exception""" - logging.shutdown() - message = "{}{}".format(errmsg, context_string(context)) - super(ParseContextError, self).__init__(message) - -######################################################################## - -class ContextRegion(Iterable): - """Class to imitate the LIFO nature of program language blocks""" - - def __init__(self): - """Initialize this ContextRegion""" - self._lifo = list() - - def push(self, rtype, rname): - """Push a new region onto the stack""" - self._lifo.append([rtype, rname]) - - def pop(self): - """Remove the top item from the stack""" - return self._lifo.pop() - - def type_list(self): - """Return just the types in the list""" - return [x[0] for x in self._lifo] - - def __iter__(self): - """Local version of iterator""" - for item in self._lifo: - yield item[0] - - def __len__(self): - """Local implementation of len builtin""" - return len(self._lifo) - - def __getitem__(self, index): - """Special item getter for a ContextRegion""" - return self._lifo[index] - -######################################################################## - -class ParseContext(): - """A class for keeping track of a parsing position - >>> ParseContext(32, "source.F90") #doctest: +ELLIPSIS - - >>> ParseContext("source.F90", 32) - Traceback (most recent call last): - parse_tools.parse_source.CCPPError: ParseContext linenum must be an int - >>> ParseContext(32, 90) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: ParseContext filenum must be a string - >>> "{}".format(ParseContext(32, "source.F90")) - 'source.F90:33' - >>> "{}".format(ParseContext()) - '' - >>> ParseContext(linenum=32, filename="source.F90").increment(13) - - """ - - def __init__(self, linenum=None, filename=None, context=None): - """Initialize this ParseContext""" - # Set regions first in case of exception - if context is not None: - self.__regions = copy.deepcopy(context.regions) - else: - self.__regions = ContextRegion() - # End if - if context is not None: - # If context is passed, ignore linenum - linenum = context.line_num - elif linenum is None: - linenum = -1 - elif not isinstance(linenum, int): - raise CCPPError('ParseContext linenum must be an int') - # No else, everything is okay - # End if - if context is not None: - # If context is passed, ignore filename - filename = context.filename - elif filename is None: - filename = "" - elif not isinstance(filename, str): - raise CCPPError('ParseContext filename must be a string') - # No else, everything is okay - # End if - self.__linenum = linenum - self.__filename = filename - - def default_module_name(self): - """Return a default module for this file""" - return os.path.splitext(os.path.basename(self.filename))[0] - - @property - def line_num(self): - """Return the current line""" - return self.__linenum - - @line_num.setter - def line_num(self, newnum): - """Set a new line number for this context""" - self.__linenum = newnum - - @property - def filename(self): - """Return the object's filename""" - return self.__filename - - @property - def regions(self): - """Return the object's region list""" - return self.__regions - - def __format__(self, spec): - """Return a string representing the location in a file - Note that self.__linenum is zero based. - can be 'dir' (show filename directory) or 'nodir' filename only. - Any other spec entry is ignored. - """ - if spec == 'dir': - fname = self.__filename - elif spec == 'nodir': - fname = os.path.basename(self.__filename) - else: - fname = self.__filename - # End if - if self.__linenum >= 0: - fmt_str = "{}:{}".format(fname, self.__linenum+1) - else: - fmt_str = "{}".format(fname) - # End if - return fmt_str - - def __str__(self): - """Return a string representing the location in a file - Note that self.__linenum is zero based. - """ - if self.__linenum >= 0: - retstr = "{}:{}".format(self.__filename, self.__linenum+1) - else: - retstr = "{}".format(self.__filename) - # End if - return retstr - - def increment(self, inc=1): - """Increment the location within a file""" - if self.__linenum < 0: - self.__linenum = 0 - # End if - self.__linenum = self.__linenum + inc - - def enter_region(self, region_type, region_name=None, nested_ok=True): - """Mark the entry of a region (e.g., DDT, module, function). - If nested_ok == False, throw an exception if the context is already - inside a region with the same type.""" - if (region_type not in self.__regions.type_list()) or nested_ok: - self.__regions.push(region_type, region_name) - else: - emsg = "Cannot enter a nested {} region" - raise ParseContextError(emsg.format(region_type), self) - # End if - - def leave_region(self, region_type, region_name=None): - """Mark the exit from a region. Check region name if possible""" - if self.__regions: - curr_type, curr_name = self.__regions.pop() - if curr_type != region_type: - emsg = "Trying to exit {} region while currently in {} region" - raise ParseContextError(emsg.format(region_type, curr_type), - self) - # End if - if (region_name is not None) and (curr_name is not None): - if region_name != curr_name: - emsg = "Trying to exit {} {} while currently in {} {}" - raise ParseContextError(emsg.format(region_type, - region_name, - curr_type, - curr_name), self) - # End if - elif (region_name is not None) and (curr_name is None): - emsg = "Trying to exit {} {} while currently in unnamed {} region" - raise ParseContextError(emsg.format(region_type, region_name, - curr_type), self) - # End if - else: - raise ParseContextError("Cannot exit, not currently in any region", - self) - # End if - - def curr_region(self): - """Return the innermost current region""" - curr = None - if self.__regions: - curr = self.__regions[-1] - # No else, will return None - # End if - return curr - - def in_region(self, region_type, region_name=None): - """Return True iff we are currently in """ - return self.curr_region() == [region_type, region_name] - - def region_str(self): - """Create a string describing the current region""" - rgn_str = "" - for index in len(self.__regions): - rtype, rname = self.__regions[index] - if rgn_str: - rgn_str += " ==> " - # End if - rgn_str += "{}".format(rtype) - if rname is not None: - rgn_str += " {}".format(rname) - # End if - # End for - return rgn_str - -######################################################################## - -class ParseSource(): - """ - A simple object for providing source information - >>> ParseSource("myname", "mytype", ParseContext(13, "foo.F90")) #doctest: +ELLIPSIS - - >>> ParseSource("myname", "mytype", ParseContext(13, "foo.F90")).ptype - 'mytype' - >>> ParseSource("myname", "mytype", ParseContext(13, "foo.F90")).name - 'myname' - >>> print("{}".format(ParseSource("myname", "mytype", ParseContext(13, "foo.F90")).context)) - foo.F90:14 - """ - - def __init__(self, name_in, type_in, context_in): - """Initialize this ParseSource object.""" - self.__name = name_in - self.__type = type_in - self.__context = context_in - - @property - def ptype(self): - """Return this source's type""" - return self.__type - - @property - def name(self): - """Return this source's name""" - return self.__name - - @property - def context(self): - """Return this source's context""" - return self.__context - -######################################################################## diff --git a/scripts/parse_tools/preprocess.py b/scripts/parse_tools/preprocess.py deleted file mode 100755 index 06b94147..00000000 --- a/scripts/parse_tools/preprocess.py +++ /dev/null @@ -1,425 +0,0 @@ -#! /usr/bin/env python3 -""" -Classes to parse C preprocessor lines and to maintain a stack to allow -inclusion and exclusion of lines based on preprocessor symbol definitions. -""" - -# Python library imports -import re -import ast -# CCPP Framewor imports -from parse_source import ParseSyntaxError - -__defined_re__ = re.compile(r"defined\s+([A-Za-z0-9_]+)") - -############################################################################### - -class PreprocError(ValueError): - """Class to report preprocessor line errors""" - def __init__(self, message): - super(PreprocError, self).__init__(message) - -######################################################################## - -def preproc_bool(value): - """Turn a preprocessor value into a boolean""" - if isinstance(value, bool): - line_val = value - else: - try: - ival = int(value) - line_val = ival != 0 - except ValueError: - line_val = value != "0" - # end try - # end if - return line_val - -######################################################################## - -def preproc_item_value(item, preproc_defs): - """Find the value of a preproc (part of a parsed - preprocessor line)""" - value = False - if isinstance(item, ast.Expr): - value = preproc_item_value(item.value, preproc_defs) - elif isinstance(item, ast.Call): - func = item.func.id - # The only 'function' we know how to process is "defined" - if func == "defined": - args = item.args - if len(args) != 1: - raise PreprocError("Invalid defined statement, {}".format(ast.dump(item))) - # end if - symbol = args[0].id - # defined is True as long as we know about the symbol - value = symbol in preproc_defs - elif func == "notdefined": - args = item.args - if len(args) != 1: - raise PreprocError("Invalid defined statement, {}".format(ast.dump(item))) - # end if - symbol = args[0].id - # notdefined is True as long as we do not know about the symbol - value = symbol not in preproc_defs - else: - raise PreprocError("Cannot parse function {}".format(func)) - # end if - elif isinstance(item, ast.BoolOp): - left_val = preproc_item_value(item.values[0], preproc_defs) - right_val = preproc_item_value(item.values[1], preproc_defs) - oper = item.op - if isinstance(oper, ast.And): - value = preproc_bool(left_val) and preproc_bool(right_val) - elif isinstance(oper, ast.Or): - value = preproc_bool(left_val) or preproc_bool(right_val) - else: - raise PreprocError("Unknown binary operator, {}".format(oper)) - # end if - elif isinstance(item, ast.UnaryOp): - val = preproc_item_value(item.operand, preproc_defs) - oper = item.op - if isinstance(oper, ast.Not): - value = not preproc_bool(val) - else: - raise PreprocError("Unknown unary operator, {}".format(oper)) - # end if - elif isinstance(item, ast.Compare): - left_val = preproc_item_value(item.left, preproc_defs) - value = True - for index in range(len(item.ops)): - oper = item.ops[index] - rcomp = item.comparators[index] - right_val = preproc_item_value(rcomp, preproc_defs) - if isinstance(oper, ast.Eq): - value = value and (left_val == right_val) - elif isinstance(oper, ast.NotEq): - value = value and (left_val != right_val) - else: - # What remains are numerical comparisons, use integers - try: - ilval = int(left_val) - irval = int(right_val) - if isinstance(oper, ast.Gt): - value = value and (ilval > irval) - elif isinstance(oper, ast.GtE): - value = value and (ilval >= irval) - elif isinstance(oper, ast.Lt): - value = value and (ilval < irval) - elif isinstance(oper, ast.LtE): - value = value and (ilval <= irval) - else: - emsg = "Unknown comparison operator, {}" - raise PreprocError(emsg.format(oper)) - # end if - except ValueError: - value = False - # end try - # end if - # end for - elif isinstance(item, ast.Name): - id_key = item.id - if id_key in preproc_defs: - value = preproc_defs[id_key] - else: - value = id_key - # end if - elif isinstance(item, ast.Num): - value = item.n - else: - raise PreprocError("Cannot parse {}".format(item)) - # end if - return value - -######################################################################## - -def parse_preproc_line(line, preproc_defs): - """Parse a preprocessor line into a tree that can be evaluated""" - # Scan line and translate to python syntax - inchar = None # Character context - line_len = len(line) - pline = "" - index = 0 - while index < line_len: - if (line[index] == '"') or (line[index] == "'"): - if inchar == line[index]: - inchar = None - elif inchar is None: - inchar = line[index] - # Else in character context, just copy - # end if - pline = pline + line[index] - elif inchar is not None: - # In character context, just copy current character - pline = pline + line[index] - elif line[index:index+2] == '&&': - pline = pline + 'and' - index = index + 1 - elif line[index:index+2] == '||': - pline = pline + 'or' - index = index + 1 - elif line[index] == '!': - pline = pline + "not" - else: - match = __defined_re__.match(line[index:]) - if match is None: - # Just copy current character - pline = pline + line[index] - else: - mlen = len(match.group(0)) - pline = pline + "defined ({})".format(match.group(1)) - index = index + mlen - 1 - # end if - # end if - index = index + 1 - # end while - try: - ast_line = ast.parse(pline) - # We should only have one 'statement' - if len(ast_line.body) != 1: - line_val = False - success = False - else: - value = preproc_item_value(ast_line.body[0], preproc_defs) - line_val = preproc_bool(value) - success = True - # end if - except SyntaxError: - line_val = False - success = False - # end try - return line_val, success - -######################################################################## - -class PreprocStack(object): - """Class to handle preprocess regions""" - - ifdef_re = re.compile(r"#\s*ifdef\s+(.*)") - ifndef_re = re.compile(r"#\s*ifndef\s+(.*)") - if_re = re.compile(r"#\s*if([^dn].*)") - elif_re = re.compile(r"#\s*elif\s(.*)") - ifelif_re = re.compile(r"#\s*(?:el)?if\s(.*)") - else_re = re.compile(r"#\s*else") - end_re = re.compile(r"#\s*endif") - define_re = re.compile(r"#\s*define\s+([A-Za-z0-9_]+)\s+([^\s]*)") - undef_re = re.compile(r"#\s*undef\s+([A-Za-z0-9_]+)") - - def __init__(self): - """Initialize our region stack""" - self._region_stack = list() - - @staticmethod - def process_if_line(line, preproc_defs): - """Decide if (el)?if represents a True or False condition. - Return True iff the line evaluates to a True condition. - is a dictionary where each key is a symbol which - can be tested (e.g., 'FOO' in #ifdef FOO). The value is that - symbol's preprocessor value, if provided (e.g., 3 for -DFOO=3), - otherwise, it is None. - Return second logical value of False if we are unable to process - >>> PreprocStack().process_if_line("#if 0", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#if 1", {'CCPP':1}) - (True, True) - >>> PreprocStack().process_if_line("#elif 0", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#elif 1", {'CCPP':1}) - (True, True) - >>> PreprocStack().process_if_line("#if ( WRF_CHEM == 1 )", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#if ( WRF_CHEM == 1 )", {'WRF_CHEM':1}) - (True, True) - >>> PreprocStack().process_if_line("#if ( WRF_CHEM == 1 )", {'WRF_CHEM':0}) - (False, True) - >>> PreprocStack().process_if_line("#if (WRF_CHEM == 0)", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#if (WRF_CHEM == 0)", {'WRF_CHEM':1}) - (False, True) - >>> PreprocStack().process_if_line("#if (WRF_CHEM == 0)", {'WRF_CHEM':0}) - (True, True) - >>> PreprocStack().process_if_line("#if defined(CCPP)", {'CCPP':1}) - (True, True) - >>> PreprocStack().process_if_line("#if defined(CCPP)", {'CCPP':0}) - (True, True) - >>> PreprocStack().process_if_line("#if defined(CCPP)", {}) - (False, True) - >>> PreprocStack().process_if_line("#if ( defined WACCM_PHYS )", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#if (defined WACCM_PHYS)", {'WACCM_PHYS':1}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined WACCM_PHYS)", {'WACCM_PHYS':0}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) && (! defined(STUBMPI)))", {}) - (False, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) && (! defined(STUBMPI)))", {'DM_PARALLEL':1}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) && (! defined(STUBMPI)))", {'DM_PARALLEL':1, 'STUBMPI':0}) - (False, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) && (! defined(STUBMPI)))", {'STUBMPI':0}) - (False, True) - >>> PreprocStack().process_if_line("# if (defined(DM_PARALLEL) || (! defined(STUBMPI)))", {}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) || (! defined(STUBMPI)))", {'DM_PARALLEL':1}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) || (! defined(STUBMPI)))", {'DM_PARALLEL':1, 'STUBMPI':0}) - (True, True) - >>> PreprocStack().process_if_line("#if (defined(DM_PARALLEL) || (! defined(STUBMPI)))", {'STUBMPI':0}) - (False, True) - >>> PreprocStack().process_if_line("#elif ( WRF_CHEM == 1 )", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("#elif ( WRF_CHEM == 1 )", {'WRF_CHEM':1}) - (True, True) - >>> PreprocStack().process_if_line("#elif ( WRF_CHEM == 1 )", {'WRF_CHEM':0}) - (False, True) - >>> PreprocStack().process_if_line("#elif (WRF_CHEM == 0)", {'CCPP':1}) - (False, True) - >>> PreprocStack().process_if_line("# elif (WRF_CHEM == 0)", {'WRF_CHEM':1}) - (False, True) - >>> PreprocStack().process_if_line("#elif (WRF_CHEM == 0)", {'WRF_CHEM':0}) - (True, True) - >>> PreprocStack().process_if_line("#if defined(CCPP) &&", {'CCPP':1}) - (False, False) - """ - match = PreprocStack.ifelif_re.match(line) - if match is None: - return False, False # This is not a preproc line - # end if - value, okay = parse_preproc_line(match.group(1).strip(), preproc_defs) - return value, okay - - def process_line(self, line, preproc_defs, pobj, logger): - """Read and return if it is a preprocessor line. - In addition, if it is a preprocessor line enter an appropriate region - if indicated by .""" - sline = line.strip() - is_preproc_line = PreprocStack.is_preproc_line(line) - if is_preproc_line and (preproc_defs is not None): - match = PreprocStack.ifdef_re.match(sline) - if match is not None: - start_region = match.group(1) in preproc_defs - if start_region and (logger is not None): - lmsg = "Preproc: Starting True region ({}) on line {}" - logger.debug(lmsg.format(match.group(1), pobj)) - # end if - self.enter_region(start_region) - # end if - if match is None: - match = PreprocStack.ifndef_re.match(sline) - if match is not None: - start_region = match.group(1) not in preproc_defs - if (not start_region) and (logger is not None): - lmsg = "Preproc: Starting False region ({}) on line {}" - logger.debug(lmsg.format(match.group(1), pobj)) - # end if - self.enter_region(start_region) - # end if - # end if - if match is None: - match = PreprocStack.if_re.match(sline) - if match is not None: - line_val, success = self.process_if_line(sline, - preproc_defs) - self.enter_region(line_val) - if (not success) and (logger is not None): - lmsg = "WARNING: Preprocessor #if statement not handled, at {}" - logger.warning(lmsg.format(pobj)) - # end if - # end if - # end if - if match is None: - match = PreprocStack.elif_re.match(sline) - if match is not None: - line_val, success = self.process_if_line(sline, - preproc_defs) - self.modify_region(line_val) - if (not success) and (logger is not None): - lmsg = "WARNING: Preprocessor #elif statement not handled, at {}" - logger.warning(lmsg.format(pobj)) - # end if - # end if - # end if - if match is None: - match = PreprocStack.else_re.match(sline) - if match is not None: - # Always try to use True for else, modify_region will set - # correct value - self.modify_region(True) - # end if - # end if - if match is None: - match = PreprocStack.end_re.match(sline) - if match is not None: - self.exit_region(pobj) - # end if - # end if - if (match is None) and self.in_true_region(): - match = PreprocStack.define_re.match(sline) - if match is not None: - # Add (or replace) a symbol to our defs - preproc_defs[match.group(1)] = match.group(2) - # end if - # end if - if (match is None) and self.in_true_region(): - match = PreprocStack.undef_re.match(sline) - if (match is not None) and (match.group(1) in preproc_defs): - # Remove a symbol from our defs - del preproc_defs[match.group(1)] - # end if - # end if - # Ignore all other lines - # end if - return is_preproc_line - - def enter_region(self, valid): - """Enter a new region (if, ifdef, ifndef) which may - currently be valid""" - self._region_stack.append([valid, valid]) - - def exit_region(self, pobj): - """Leave the current (innermost) region""" - if not self._region_stack: - emsg = "#endif found with no matching #if, #ifdef, or #ifndef" - raise ParseSyntaxError(emsg, context=pobj) - # end if - self._region_stack.pop() - - def modify_region(self, valid): - """Possibly modify the current (innermost) region. - A region can be modified from False to True. - Any attempt to modify a region which has been True results in False - because after a region has been True, any #elif or #else must skipped. - """ - curr_region = self._region_stack.pop() - if curr_region[0]: - self._region_stack.append([curr_region[0], False]) - else: - self._region_stack.append([curr_region[0], valid]) - # end if - - def in_true_region(self): - """Return True iff the current line should be processed""" - true_region = True - for region in self._region_stack: - if not region[1]: - true_region = False - break - # end if - # end for - return true_region - - @staticmethod - def is_preproc_line(line): - """Return True iff line appears to be a preprocessor line""" - return line.lstrip()[0] == '#' - -######################################################################## - -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/parse_tools/xml_tools.py b/scripts/parse_tools/xml_tools.py deleted file mode 100644 index d9210038..00000000 --- a/scripts/parse_tools/xml_tools.py +++ /dev/null @@ -1,572 +0,0 @@ -#!/usr/bin/env python3 - -""" -Parse a host-model registry XML file and return the captured variables. -""" - -# Python library imports -import os -import re -import shutil -import subprocess -import sys -import xml.etree.ElementTree as ET -import xml.dom.minidom -sys.path.insert(0, os.path.dirname(__file__)) -# CCPP framework imports -from parse_source import CCPPError -from parse_log import init_log, set_log_to_null - -# Global data -_INDENT_STR = " " -beg_tag_re = re.compile(r"([<][^/][^<>]*[^/][>])") -end_tag_re = re.compile(r"([<][/][^<>/]+[>])") -simple_tag_re = re.compile(r"([<][^/][^<>/]+[/][>])") - -# Find python version -PYSUBVER = sys.version_info[1] -_LOGGER = None - -############################################################################### -class XMLToolsInternalError(ValueError): -############################################################################### - """Error class for reporting internal errors""" - def __init__(self, message): - """Initialize this exception""" - super().__init__(message) - -############################################################################### -def find_schema_version(root): -############################################################################### - """ - Find the version of the host registry file represented by root - >>> find_schema_version(ET.fromstring('')) - [1, 0] - >>> find_schema_version(ET.fromstring('')) - [2, 0] - >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Illegal version string, '1.a' - Format must be . - >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Illegal version string, '0.0' - Major version must be at least 1 - >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Illegal version string, '0.0' - Minor version must be at least 0 - """ - verbits = None - if 'version' not in root.attrib: - raise CCPPError("version attribute required") - # end if - version = root.attrib['version'] - versplit = version.split('.') - try: - if len(versplit) != 2: - raise CCPPError('oops') - # end if (no else needed) - try: - verbits = [int(x) for x in versplit] - except ValueError as verr: - raise CCPPError(verr) from verr - # end try - if verbits[0] < 1: - raise CCPPError('Major version must be at least 1') - # end if - if verbits[1] < 0: - raise CCPPError('Minor version must be non-negative') - # end if - except CCPPError as verr: - errstr = """Illegal version string, '{}' - Format must be .""" - ve_str = str(verr) - if ve_str: - errstr = ve_str + '\n' + errstr - # end if - raise CCPPError(errstr.format(version)) from verr - # end try - return verbits - -############################################################################### -def find_schema_file(schema_root, version, schema_path=None): -############################################################################### - """Find and return the schema file based on and - or return None. - If is present, use that as the directory to find the - appropriate schema file. Otherwise, just look in the current directory.""" - - verstring = '_'.join([str(x) for x in version]) - schema_filename = "{}_v{}.xsd".format(schema_root, verstring) - if schema_path: - schema_file = os.path.join(schema_path, schema_filename) - else: - schema_file = schema_filename - # end if - if os.path.exists(schema_file): - return schema_file - # end if - return None - -############################################################################### -def validate_xml_file(filename, schema_root, version, logger, schema_path=None): -############################################################################### - """ - Find the appropriate schema and validate the XML file, , - against it using xmllint - """ - # Check the filename - if not os.path.isfile(filename): - raise CCPPError("validate_xml_file: Filename, '{}', does not exist".format(filename)) - # end if - if not os.access(filename, os.R_OK): - raise CCPPError("validate_xml_file: Cannot open '{}'".format(filename)) - # end if - if os.path.isfile(schema_root): - # We already have a file, just use it - schema_file = schema_root - else: - if not schema_path: - # Find the schema, based on the model version - thispath = os.path.abspath(__file__) - pdir = os.path.dirname(os.path.dirname(os.path.dirname(thispath))) - schema_path = os.path.join(pdir, 'schema') - # end if - schema_file = find_schema_file(schema_root, version, schema_path) - if not (schema_file and os.path.isfile(schema_file)): - verstring = '.'.join([str(x) for x in version]) - emsg = f"""validate_xml_file: Cannot find schema for version {verstring}, - {schema_file} does not exist""" - raise CCPPError(emsg) - # end if - # end if - if not os.access(schema_file, os.R_OK): - emsg = "validate_xml_file: Cannot open schema, '{}'" - raise CCPPError(emsg.format(schema_file)) - # end if - - # Find xmllint - xmllint = shutil.which('xmllint') # Blank if not installed - if not xmllint: - msg = "xmllint not found, could not validate file {}" - raise CCPPError("validate_xml_file: " + msg.format(filename)) - # end if - - # Validate XML file against schema - logger.debug("Checking file {} against schema {}".format(filename, - schema_file)) - cmd = [xmllint, '--noout', '--schema', schema_file, filename] - cproc = subprocess.run(cmd, check=False, capture_output=True) - if cproc.returncode == 0: - # We got a pass return code but some versions of xmllint do not - # correctly return an error code on non-validation so double check - # the result - result = b'validates' in cproc.stdout or b'validates' in cproc.stderr - else: - result = False - # end if - if result: - logger.debug(cproc.stdout) - logger.debug(cproc.stderr) - return result - else: - cmd = ' '.join(cmd) - outstr = f"Execution of '{cmd}' failed with code: {cproc.returncode}\n" - if cproc.stdout: - outstr += f"{cproc.stdout.decode('utf-8', errors='replace').strip()}\n" - if cproc.stderr: - outstr += f"{cproc.stderr.decode('utf-8', errors='replace').strip()}\n" - raise CCPPError(outstr) - -############################################################################### -def read_xml_file(filename, logger=None): -############################################################################### - """Read the XML file, , and return its tree and root - - Parameters: - filename (str): The path to an XML file to read and search. - logger (logging.Logger, optional): Logger for warnings/errors. - - Returns: - tree (xml.etree.ElementTree): The element tree from the input file. - root (xml.etree.ElementTree.Element): The root element of tree. - - Raises: - CCPPError: If the file cannot be found or read. - """ - if os.path.isfile(filename) and os.access(filename, os.R_OK): - file_open = (lambda x: open(x, 'r', encoding='utf-8')) - with file_open(filename) as file_: - try: - tree = ET.parse(file_) - root = tree.getroot() - except ET.ParseError as perr: - emsg = "read_xml_file: Cannot read {}, {}" - raise CCPPError(emsg.format(filename, perr)) from perr - elif not os.access(filename, os.R_OK): - raise CCPPError("read_xml_file: Cannot open '{}'".format(filename)) - else: - emsg = "read_xml_file: Filename, '{}', does not exist" - raise CCPPError(emsg.format(filename)) - # end if - if logger: - logger.debug(f"Reading XML file {filename}") - # end if - return tree, root - -############################################################################### -def load_suite_by_name(suite_name, group_name, file, logger=None): -############################################################################### - """ - Load a suite by its name, or a group of a suite by the suite and group names. - - Parameters: - suite_name (str): The name of the suite to find. - group_name (str or None): The name of the group to find within the suite. - file (str): The path to an XML file to read and search. - logger (logging.Logger, optional): Logger for warnings/errors. - - Returns: - xml.etree.ElementTree.Element: The matching suite or group element. - - Raises: - CCPPError: If the suite or group is not found, or if the schema is invalid. - - Examples: - >>> import tempfile - >>> import xml.etree.ElementTree as ET - >>> logger = init_log('xml_tools') - >>> set_log_to_null(logger) - >>> # Create temporary files for the nested suites - >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> # Write XML contents to temporary file - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... - ... - ... ''') - >>> load_suite_by_name("physics_suite", None, file1_path, logger).tag - 'suite' - >>> load_suite_by_name("physics_suite", "dynamics", file1_path, logger).attrib['name'] - 'dynamics' - >>> load_suite_by_name("physics_suite", "missing_group", file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Nested suite physics_suite, group missing_group, not found - >>> load_suite_by_name("missing_suite", None, file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Nested suite missing_suite not found - >>> tmpdir.cleanup() - """ - _, root = read_xml_file(file, logger) - schema_version = find_schema_version(root) - res = validate_xml_file(file, 'suite', schema_version, logger) - if not res: - raise CCPPError(f"Invalid suite definition file, '{file}'") - suite = root - if suite.attrib.get("name") == suite_name: - if group_name: - for group in suite.findall("group"): - if group.attrib.get("name") == group_name: - return group - else: - return suite - emsg = f"Nested suite {suite_name}" \ - + (f", group {group_name}," if group_name else "") \ - + " not found" + (f" in file {file}" if file else "") - raise CCPPError(emsg) - -############################################################################### -def replace_nested_suite(element, nested_suite, default_path, logger): -############################################################################### - """ - Replace a tag with the actual suite or group it references. - - This function looks up a referenced suite or suite group from an external - file, deep copies its children, and replaces the element - in the parent `element` with the copied contents. - - Parameters: - element (xml.etree.ElementTree.Element): The parent element containing the nested suite. - nested_suite (xml.etree.ElementTree.Element): The element to be replaced. - default_path (str): The default path to look for nested SDFs if file is not a absolute path. - logger (logging.Logger or None): Logger to record debug information. - - Returns: - str: The name of the suite that was replaced - - Example: - >>> import tempfile - >>> import xml.etree.ElementTree as ET - >>> from types import SimpleNamespace - >>> logger = init_log('xml_tools') - >>> set_log_to_null(logger) - >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... my_scheme - ... - ... - ... ''') - >>> # Import nested suite at suite level - >>> xml = f''' - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> nested = top_suite.find("nested_suite") - >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> # Import group from nested suite at group level - >>> xml = f''' - ... - ... - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> top_group = top_suite.find("group") - >>> nested = top_group.find("nested_suite") - >>> replace_nested_suite(top_group, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> # Import group from nested suite at suite level - >>> xml = f''' - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> nested = top_suite.find("nested_suite") - >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> tmpdir.cleanup() - """ - suite_name = nested_suite.attrib.get("name") - group_name = nested_suite.attrib.get("group") - file = nested_suite.attrib.get("file") - if not os.path.isabs(file): - file = os.path.join(default_path, file) - referenced_suite = load_suite_by_name(suite_name, group_name, file, - logger=logger) - imported_content = [ET.fromstring(ET.tostring(child)) - for child in referenced_suite] - # Swap nested suite with imported content - for item in imported_content: - # If we are inserting a nested suite at the suite level (element.tag is suite), - # but we only want one group (group_name is not none), then we need to wrap - # the item in a group element. If on the other hand we insert an entire suite - # (all groups) at the suite level, or a specific group at the group level, - # then we can insert the item as is. - if element.tag == 'suite' and group_name: - item_to_insert = ET.Element("group", attrib={"name": group_name}) - item_to_insert.append(item) - else: - item_to_insert = item - element.insert(list(element).index(nested_suite), item_to_insert) - element.remove(nested_suite) - if logger: - msg = f"Expanded nested suite '{suite_name}'" \ - + (f", group '{group_name}'," if group_name else "") \ - + (f" in file '{file}'" if file else "") - logger.debug(msg.rstrip(',')) - # Return the name of the suite that we just replaced - return suite_name - -############################################################################### -def expand_nested_suites(suite, default_path, logger=None): -############################################################################### - """ - Recursively expand all elements within the XML element. - - This function finds elements within or elements, - and replaces them with the corresponding content from another suite. - - This operation is recursive and will continue expanding until no - elements remain. - - Parameters: - suite (xml.etree.ElementTree.Element): The root element. - logger (logging.Logger, optional): Logger for debug messages. - - Returns: - None. The XML tree is modified in place. - - Example: - >>> import tempfile - >>> import xml.etree.ElementTree as ET - >>> logger = init_log('xml_tools') - >>> set_log_to_null(logger) - >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> file2_path = os.path.join(tmpdir.name, "file2.xml") - >>> file3_path = os.path.join(tmpdir.name, "file3.xml") - >>> file4_path = os.path.join(tmpdir.name, "file4.xml") - >>> file5_path = os.path.join(tmpdir.name, "file5.xml") - >>> # Write mock XML contents for the nested suites - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... cloud_scheme - ... - ... - ... ''') - >>> with open(file2_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... pbl_scheme - ... - ... - ... ''') - >>> with open(file3_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... rrtmg_lw_scheme - ... - ... - ... rrtmg_sw_scheme - ... - ... - ... ''') - >>> with open(file4_path, "w") as f: - ... _ = f.write(f''' - ... - ... - ... - ... ''') - >>> with open(file5_path, "w") as f: - ... _ = f.write(f''' - ... - ... - ... - ... ''') - >>> # Parent suite - >>> xml_content = f''' - ... - ... - ... - ... - ... - ... - ... - ... ''' - >>> suite = ET.fromstring(xml_content) - >>> expand_nested_suites(suite, tmpdir.name, logger) - >>> ET.dump(suite) - - - cloud_scheme - - pbl_scheme - - rrtmg_lw_scheme - - rrtmg_sw_scheme - - >>> # Test infite recursion - >>> xml_content = f''' - ... - ... - ... - ... - ... - ... - ... ''' - >>> suite = ET.fromstring(xml_content) - >>> expand_nested_suites(suite, tmpdir.name, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Exceeded number of iterations while expanding nested suites - >>> tmpdir.cleanup() - """ - # To avoid infinite recursion, we simply count the number - # of iterations and stop at a certain limit. If someone is - # smart enough to come up with nested suite constructs that - # require more iterations, than he/she should be able to - # track down this variable and adjust it! - max_iterations = 10 - # Collect the names of the expanded suites - suite_names = [] - # Iteratively expand nested suites until they are all gone - keep_expanding = True - for num_iterations in range(max_iterations): - keep_expanding = False - # First, search all groups for nested_suite elements - groups = suite.findall("group") - for group in groups: - nested_suites = group.findall("nested_suite") - for nested in nested_suites: - suite_names.append(replace_nested_suite(group, nested, default_path, logger)) - # Trigger another pass over the root element - keep_expanding = True - # Second, search all suites for nested_suite elements - nested_suites = suite.findall("nested_suite") - for nested in nested_suites: - suite_names.append(replace_nested_suite(suite, nested, default_path, logger)) - # Trigger another pass over the root element - keep_expanding = True - if not keep_expanding: - return - raise CCPPError("Exceeded number of iterations while expanding nested suites. " + \ - "Check for infinite recursion or adjust limit max_iterations. " + \ - f"Suites expanded so far: {suite_names}") - -############################################################################### -def write_xml_file(root, file_path, logger=None): -############################################################################### - """Pretty-prints element root to an ASCII file using xml.dom.minidom""" - - def remove_whitespace_nodes(node): - """Helper function to recursively remove all text nodes that contain - only whitespace, which eliminates blank lines in the output.""" - for child in list(node.childNodes): - if child.nodeType == child.TEXT_NODE and not child.data.strip(): - node.removeChild(child) - elif child.hasChildNodes(): - remove_whitespace_nodes(child) - - # Convert ElementTree to a byte string - byte_string = ET.tostring(root, 'us-ascii') - - # Parse string using minidom for pretty printing - reparsed = xml.dom.minidom.parseString(byte_string) - - # Clean whitespace-only text nodes - remove_whitespace_nodes(reparsed) - - # Generate pretty-printed XML string - pretty_xml = reparsed.toprettyxml(indent=" ") - - # Write to file - with open(file_path, 'w', errors='xmlcharrefreplace') as f: - f.write(pretty_xml) - - # Tell everyone! - if logger: - logger.debug(f"Writing XML file {file_path}") - -############################################################################## diff --git a/scripts/state_machine.py b/scripts/state_machine.py deleted file mode 100755 index 9c26afd1..00000000 --- a/scripts/state_machine.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python3 -# - -"""Classes and methods to implement a simple state machine.""" - -# Python library imports -import re -from collections import OrderedDict -# CCPP framework imports -from parse_tools import FORTRAN_ID - -############################################################################### - -class StateMachine: - """Class and methods to implement a simple state machine. - Note, a collections.UserDict would be nice here but it is not in python 2. - >>> StateMachine() - StateMachine() - >>> StateMachine([('ab','a','b','a')]) - StateMachine(ab) - >>> StateMachine([('ab','a','b','a'),('cd','c','d','c')]) - StateMachine(ab, cd) - >>> StateMachine([('ab','a','b','a')]).add_transition('cd','c','d','c') - - >>> StateMachine([('ab','a','b','a')])['cd'] = ('c','d','c') - >>> StateMachine([('ab','a','b','a'),('cd','c','d','c')]).transitions() - ['ab', 'cd'] - >>> StateMachine([('ab','a','b','a')]).initial_state('ab') - 'a' - >>> StateMachine([('ab','a','b','a')]).final_state('ab') - 'b' - >>> StateMachine([('ab','a','b','a')]).transition_regex('ab') - re.compile('a$', re.IGNORECASE) - >>> StateMachine([('ab','a','b','a')]).function_match('foo_a', transition='ab') - ('foo', 'a', 'ab') - >>> StateMachine([('ab','a','b',r'ax?')]).function_match('foo_a', transition='ab') - ('foo', 'a', 'ab') - >>> StateMachine([('ab','a','b',r'ax?')]).function_match('foo_ax', transition='ab') - ('foo', 'ax', 'ab') - >>> StateMachine([('ab','a','b','a')]).function_match('foo_ab', transition='ab') - (None, None, None) - >>> StateMachine([('ab','a','b','a'),('cd','c','d','c')]).function_match('foo_c') - ('foo', 'c', 'cd') - >>> StateMachine([('ab','a','b',r'ax?')]).transition_match('a') - 'ab' - >>> StateMachine([('ab','a','b',r'ax?')]).transition_match('ax') - 'ab' - >>> StateMachine([('ab','a','b',r'ax?')]).transition_match('axx') - - >>> StateMachine([('ab','a','b','a')]).transition_match('ab') - - >>> StateMachine([('ab','a','b','a'),('cd','c','d','c')]).transition_match('c') - 'cd' - >>> StateMachine((('ab','a','b','a'),)).add_transition('ab','c','d','c') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ValueError: ERROR: transition, 'ab', already exists - >>> StateMachine((('ab','a','b','a'))) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ValueError: Invalid initial_data transition ('ab'), should be of the form (name, inital_state, final_state, regex). - >>> StateMachine([('ab','a','b','a')])['cd'] = ('c','d') #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ValueError: Invalid transition (('c', 'd')), should be of the form (inital_state, final_state, regex). - """ - - def __init__(self, initial_data=None): - """Implement a finite state machine. - is an iterable where each item has four elements: - (transition_name, , , ) - is a string representing allowable names for - functions which form part of the transition action. - """ - # Implement the State Transition Table as a tuple and use accessors - self.__stt__ = OrderedDict() - if initial_data is not None: - # Note that we need to add states with longer regular expressions - # before short ones so that we match correctly. - for trans in sorted(initial_data, key=lambda x: len(x[3]) if len(x) > 3 else 0, reverse=True): - if len(trans) != 4: - raise ValueError("Invalid initial_data transition ({}), should be of the form (name, inital_state, final_state, regex).".format(trans)) - # end if - self.add_transition(trans[0], trans[1], trans[2], trans[3]) - # end for - # end if - - def add_transition(self, name, init_state, final_state, regex): - """Add a transition to this state machine. - See __setitem__ for implementation details.""" - self[name] = (init_state, final_state, regex) - - def transitions(self): - """Return a list of transition names""" - return list(self.__stt__.keys()) - - def initial_state(self, transition): - """Return the initial (before) state for """ - return self.__stt__[transition][0] - - def final_state(self, transition): - """Return the final (after) state for """ - return self.__stt__[transition][1] - - def transition_regex(self, transition): - """Return the compiled regex for """ - return self.__stt__[transition][2] - - def function_regex(self, transition): - """Return the compiled functino regex for """ - return self.__stt__[transition][3] - - def transition_match(self, test_str, transition=None): - """Return the matched transition, if found. - """ - match_trans = None - if transition is None: - trans_list = self.transitions() - else: - trans_list = [transition] - # end if - for trans in trans_list: - regex = self.transition_regex(trans) - match = regex.match(test_str) - if match is not None: - match_trans = trans - break - # end if - # end for - return match_trans - - def function_match(self, test_str, transition=None): - """Return a function ID, transition identifier, and matched - transition if found. - If is None, look for a match in any transition, - otherwise, only look for a specific match to that transition. - """ - if transition is None: - trans_list = self.transitions() - else: - trans_list = [transition] - # end if - func_id = None - trans_id = None - match_trans = None - for trans in trans_list: - regex = self.function_regex(trans) - match = regex.match(test_str) - if match is not None: - func_id = match.group(1) - trans_id = match.group(2) - match_trans = trans - break - # end if - # end for - return func_id, trans_id, match_trans - - def __getitem__(self, key): - return self.__stt__[key] - - def __setitem__(self, key, value): - if key in self.__stt__: - raise ValueError("ERROR: transition, '{}', already exists".format(key)) - # end if - if len(value) != 3: - raise ValueError("Invalid transition ({}), should be of the form (inital_state, final_state, regex).".format(value)) - # end if - regex = re.compile(value[2] + r"$", flags=re.IGNORECASE) - function = re.compile(FORTRAN_ID + r"_(" + value[2] + r")$", flags=re.IGNORECASE) - self.__stt__[key] = (value[0], value[1], regex, function) - - def __delitem__(self, key): - del self.__stt__[key] - - def __iter__(self): - return iter(self.__stt__) - - def __len__(self): - return len(self.__stt__) - - def __str__(self): - return "StateMachine({})".format(", ".join(self.transitions())) - - def __repr__(self): - return str(self) - -############################################################################### -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/suite_objects.py b/scripts/suite_objects.py deleted file mode 100755 index 3c846ed5..00000000 --- a/scripts/suite_objects.py +++ /dev/null @@ -1,2024 +0,0 @@ -#!/usr/bin/env python3 -# - -"""Classes and methods to create a Fortran suite-implementation file -to implement calls to a set of suites for a given host model.""" - -# Python library imports -import logging -import re -import xml.etree.ElementTree as ET -# CCPP framework imports -from ccpp_state_machine import CCPP_STATE_MACH, RUN_PHASE_NAME -from code_block import CodeBlock -from constituents import ConstituentVarDict -from framework_env import CCPPFrameworkEnv -from metavar import Var, VarDictionary, VarLoopSubst -from metavar import CCPP_CONSTANT_VARS, CCPP_LOOP_VAR_STDNAMES -from parse_tools import ParseContext, ParseSource, context_string -from parse_tools import ParseInternalError, CCPPError -from parse_tools import init_log, set_log_to_null -from var_props import is_horizontal_dimension, find_horizontal_dimension -from var_props import find_vertical_dimension -from var_props import VarCompatObj - -# pylint: disable=too-many-lines - -############################################################################### -# Module (global) variables -############################################################################### - -_OBJ_LOC_RE = re.compile(r"(0x[0-9A-Fa-f]+)>") -_BLANK_DIMS_RE = re.compile(r"[(][:](,:)*[)]$") - -# Source for internally generated variables. -_API_SOURCE_NAME = "CCPP_API" -# Use the constituent source type for consistency -_API_SUITE_VAR_NAME = ConstituentVarDict.constitutent_source_type() -_API_GROUP_VAR_NAME = "group" -_API_SCHEME_VAR_NAME = "scheme" -_API_LOCAL_VAR_NAME = "local" -_API_LOCAL_VAR_TYPES = [_API_LOCAL_VAR_NAME, _API_SUITE_VAR_NAME] -_API_CONTEXT = ParseContext(filename="ccpp_suite.py") -_API_SOURCE = ParseSource(_API_SOURCE_NAME, _API_SCHEME_VAR_NAME, _API_CONTEXT) -_API_LOCAL = ParseSource(_API_SOURCE_NAME, _API_LOCAL_VAR_NAME, _API_CONTEXT) -_API_LOGGING = init_log('ccpp_suite') -set_log_to_null(_API_LOGGING) -_API_DUMMY_RUN_ENV = CCPPFrameworkEnv(_API_LOGGING, - ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - -############################################################################### -def new_suite_object(item, context, parent, run_env, loop_count=0): -############################################################################### - "'Factory' method to create the appropriate suite object from XML" - new_item = None - if item.tag == 'subcycle': - new_item = Subcycle(item, context, parent, run_env, loop_count=loop_count) - elif item.tag == 'scheme': - new_item = Scheme(item, context, parent, run_env) - else: - emsg = "Unknown CCPP suite element type, '{}'" - raise CCPPError(emsg.format(item.tag)) - # end if - return new_item - -############################################################################### - -class CallList(VarDictionary): - """A simple class to hold a routine's call list (dummy arguments)""" - - def __init__(self, name, run_env, routine=None): - """Initialize this call list. - is the name of this dictionary. - is a pointer to the routine for which this is a call list - or None for a routine that is not a SuiteObject. - """ - self.__routine = routine - super().__init__(name, run_env) - - def add_vars(self, call_list, run_env, gen_unique=False): - """Add new variables from another CallList ()""" - for var in call_list.variable_list(): - stdname = var.get_prop_value('standard_name') - self.add_variable(var, run_env, gen_unique=gen_unique, adjust_intent=True, exists_ok=True) - # end for - - def add_variable(self, newvar, run_env, exists_ok=False, gen_unique=False, - adjust_intent=False): - """Add as for VarDictionary but make sure that the variable - has an intent with the default being intent(in). - """ - # We really need an intent on a dummy argument - if newvar.get_prop_value("intent") is None: - subst_dict = {'intent' : 'in'} - oldvar = newvar - newvar = oldvar.clone(subst_dict, source_name=self.name, - source_type=_API_GROUP_VAR_NAME, - context=oldvar.context) - # end if - super().add_variable(newvar, run_env, exists_ok=exists_ok, - gen_unique=gen_unique, adjust_intent=adjust_intent) - - def call_string(self, cldicts=None, is_func_call=False, subname=None, sub_lname_list=None): - """Return a dummy argument string for this call list. - may be a list of VarDictionary objects to search for - local_names (default is to use self). - should be set to True to construct a call statement. - If is False, construct a subroutine dummy argument - list. - may be a list of local_name substitutions. - """ - arg_str = "" - arg_sep = "" - for var in self.variable_list(): - # Do not include constants - stdname = var.get_prop_value('standard_name') - if stdname not in CCPP_CONSTANT_VARS: - # Find the dummy argument name - dummy = var.get_prop_value('local_name') - # Now, find the local variable name - if cldicts is not None: - for cldict in cldicts: - dvar = cldict.find_variable(standard_name=stdname, - any_scope=False) - if dvar is not None: - break - # end if - # end for - if dvar is None: - if subname is not None: - errmsg = "{}: ".format(subname) - else: - errmsg = "" - # end if - errmsg += "'{}', not found in call list for '{}'" - clnames = [x.name for x in cldicts] - raise CCPPError(errmsg.format(stdname, clnames)) - # end if - lname = dvar.get_prop_value('local_name') - # Optional variables in the caps are associated with - # local pointers of _ptr - if dvar.get_prop_value('optional'): - lname = dummy+'_ptr' - # end if - else: - cldict = None - aref = var.array_ref(local_name=dummy) - if aref is not None: - lname = aref.group(1) - else: - lname = dummy - # end if - # end if - # Modify Scheme call_list to handle local_name change for this var. - # Are there any variable transforms for this scheme? - # If so, change Var's local_name need to local dummy array containing - # transformed argument, var_trans_local. - if sub_lname_list: - for (var_trans_local, var_lname, sname, rindices, lindices, compat_obj) in sub_lname_list: - if (sname == stdname): - lname = var_trans_local - # end if - # end for - # end if - if is_func_call: - if cldicts is not None: - use_dicts = cldicts - else: - use_dicts = [self] - # end if - run_phase = self.routine.run_phase() - # We only need dimensions for suite variables in run phase - need_dims = SuiteObject.is_suite_variable(dvar) and run_phase - vdims = var.call_dimstring(var_dicts=use_dicts, - explicit_dims=need_dims, - loop_subst=run_phase) - if _BLANK_DIMS_RE.match(vdims) is None: - lname = lname + vdims - # end if - # end if - if is_func_call: - arg_str += "{}{}={}".format(arg_sep, dummy, lname) - else: - arg_str += "{}{}".format(arg_sep, lname) - # end if - arg_sep = ", " - # end if - # end for - return arg_str - - @property - def routine(self): - """Return the routine for this call list (or None)""" - return self.__routine - -############################################################################### - -class SuiteObject(VarDictionary): - """Base class for all CCPP Suite objects (e.g., Scheme, Subcycle) - SuiteObjects have an internal dictionary for variables created for - execution of the SuiteObject. These variables will be allocated and - managed at the Group level (unless cross-group usage or persistence - requires handling at the Suite level). - SuiteObjects also have a call list which is a list of variables which - are passed to callable SuiteObjects (e.g., Scheme). - """ - - def __init__(self, name, context, parent, run_env, - active_call_list=False, variables=None, phase_type=None): - # pylint: disable=too-many-arguments - self.__name = name - self.__context = context - self.__parent = parent - self.__run_env = run_env - if active_call_list: - self.__call_list = CallList(name + '_call_list', run_env, - routine=self) - else: - self.__call_list = None - # end if - self.__parts = list() - self.__needs_horizontal = None - self.__phase_type = phase_type - # Initialize our dictionary - super().__init__(self.name, run_env, - variables=variables, parent_dict=parent) - - def declarations(self): - """Return a list of local variables to be declared in parent Group - or Suite. By default, this list is the object's embedded VarDictionary. - """ - return self.variable_list() - - def add_part(self, item, replace=False): - """Add an object (e.g., Scheme, Subcycle) to this SuiteObject. - if is True, replace in its current position in self. - """ - if replace: - if item in self.__parts: - index = self.__parts.index(item) - else: - emsg = 'Cannot replace {} in {}, not a member' - raise ParseInternalError(emsg.format(item.name, self.name)) - # end if - else: - if item in self.__parts: - emsg = 'Cannot add {} to {}, already a member' - raise ParseInternalError(emsg.format(item.name, self.name)) - # end if - index = len(self.__parts) - # end if - # Just add - self.__parts.insert(index, item) - item.reset_parent(self) - - def schemes(self): - """Return a flattened list of schemes for this SuiteObject""" - schemes = list() - for item in self.__parts: - schemes.extend(item.schemes()) - # end for - return schemes - - def reset_parent(self, new_parent): - """Reset the parent of this SuiteObject (which has been moved)""" - self.__parent = new_parent - - def phase(self): - """Return the CCPP state phase_type for this SuiteObject""" - trans = self.phase_type - if trans is None: - if self.parent is not None: - trans = self.parent.phase() - else: - trans = False - # end if - # end if - return trans - - def run_phase(self): - """Return True iff this SuiteObject is in a run phase group""" - return self.phase() == RUN_PHASE_NAME - - def timestep_phase(self): - '''Return True iff this SuiteObject is in a timestep initial or - timestep final phase group''' - phase = self.phase() - return (phase is not None) and ('timestep' in phase) - - def register_action(self, vaction): - """Register (i.e., save information for processing during write stage) - and return True or pass up to the parent of - . Return True if any level registers , False otherwise. - The base class will not register any action, it must be registered in - an override of this method. - """ - if self.parent is not None: - return self.parent.register_action(vaction) - # end if - return False - - @classmethod - def is_suite_variable(cls, var): - """Return True iff belongs to our Suite""" - return var and (var.source.ptype == _API_SUITE_VAR_NAME) - - def is_local_variable(self, var): - """Return the local variable matching if one is found belonging - to this object or any of its SuiteObject parents.""" - stdname = var.get_prop_value('standard_name') - lvar = None - obj = self - while (not lvar) and (obj is not None) and isinstance(obj, SuiteObject): - lvar = obj.find_variable(standard_name=stdname, any_scope=False, - search_call_list=False) - if not lvar: - obj = obj.parent - # end if - # end while - return lvar - - def add_call_list_variable(self, newvar, exists_ok=False, - gen_unique=False, subst_dict=None): - """Add to this SuiteObject's call_list. If this SuiteObject - does not have a call list, recursively try the SuiteObject's parent - If is not None, create a clone using that as a dictionary - of substitutions. - Do not add if it exists as a local variable. - Do not add if it is a suite variable""" - stdname = newvar.get_prop_value('standard_name') - if self.parent: - pvar = self.parent.find_variable(standard_name=stdname, - source_var=newvar, - any_scope=False) - else: - pvar = None - # end if - if SuiteObject.is_suite_variable(pvar): - pass # Do not add suite variable to a call list - elif self.is_local_variable(newvar): - pass # Do not add to call list, it is owned by a SuiteObject - elif self.call_list is not None: - if (stdname in CCPP_LOOP_VAR_STDNAMES) and (not self.run_phase()): - errmsg = 'Attempting to use loop variable {} in {} phase' - raise CCPPError(errmsg.format(stdname, self.phase())) - # end if - # Do we need a clone? - if isinstance(self, Group): - stype = _API_GROUP_VAR_NAME - else: - stype = None - # end if - if stype or subst_dict: - oldvar = newvar - if subst_dict is None: - subst_dict = {} - # end if - # Make sure that this variable has an intent - if ((oldvar.get_prop_value("intent") is None) and - ("intent" not in subst_dict)): - subst_dict["intent"] = "in" - # end if - newvar = oldvar.clone(subst_dict, source_name=self.name, - source_type=stype, context=self.context) - # end if - self.call_list.add_variable(newvar, self.run_env, - exists_ok=exists_ok, - gen_unique=gen_unique, - adjust_intent=True) - # We need to make sure that this variable's dimensions are available - for vardim in newvar.get_dim_stdnames(include_constants=False): - # Unnamed dimensions are ok for allocatable variables - if vardim == '' and newvar.get_prop_value('allocatable'): - continue - elif vardim == '': - emsg = f"{self.name}: Cannot have unnamed/empty string dimension" - raise ParseInternalError(emsg) - # end if - dvar = self.find_variable(standard_name=vardim, - any_scope=True) - if dvar is None: - emsg = "{}: Could not find dimension {} in {}" - raise ParseInternalError(emsg.format(self.name, - vardim, stdname)) - # end if - elif self.parent is None: - errmsg = 'No call_list found for {}'.format(newvar) - raise ParseInternalError(errmsg) - elif pvar: - # Check for call list incompatibility - if pvar is not None: - compat, reason = pvar.compatible(newvar, self.run_env) - if not compat: - emsg = 'Attempt to add incompatible variable to call list:' - emsg += '\n{} from {} is not compatible with {} from {}' - nlreason = newvar.get_prop_value(reason) - plreason = pvar.get_prop_value(reason) - emsg += '\nreason = {} ({} != {})'.format(reason, - nlreason, - plreason) - nlname = newvar.get_prop_value('local_name') - plname = pvar.get_prop_value('local_name') - raise CCPPError(emsg.format(nlname, newvar.source.name, - plname, pvar.source.name)) - # end if - # end if (no else, variable already in call list) - else: - self.parent.add_call_list_variable(newvar, exists_ok=exists_ok, - gen_unique=gen_unique, - subst_dict=subst_dict) - # end if - - def add_variable_to_call_tree(self, var, vmatch=None, subst_dict=None): - """Add to 's call_list (or a parent if does not - have an active call_list). - If is not None, also add the loop substitution variables - which must be present. - If is not None, create a clone using that as a dictionary - of substitutions. - """ - found_dims = False - if var is not None: - self.add_call_list_variable(var, exists_ok=True, - gen_unique=True, subst_dict=subst_dict) - found_dims = True - # end if - if vmatch is not None: - svars = vmatch.has_subst(self, any_scope=True) - if svars is None: - found_dims = False - else: - found_dims = True - for svar in svars: - self.add_call_list_variable(svar, exists_ok=True) - # end for - # Register the action (probably at Group level) - self.register_action(vmatch) - # end if - # end if - return found_dims - - def horiz_dim_match(self, ndim, hdim, nloop_subst): - """Find a match between and , if they are both - horizontal dimensions. - If == , return . - If is not None and its required standard names exist - in our extended dictionary, return them. - Otherwise, return None. - NB: Loop substitutions are only allowed during the run phase but in - other phases, horizontal_dimension and horizontal_loop_extent - are the same. - """ - dim_match = None - nis_hdim = is_horizontal_dimension(ndim) - his_hdim = is_horizontal_dimension(hdim) - if nis_hdim and his_hdim: - if ndim == hdim: - dim_match = ndim - elif self.run_phase() and (nloop_subst is not None): - svars = nloop_subst.has_subst(self, any_scope=True) - match = svars is not None - if match: - if isinstance(self, Scheme): - obj = self.parent - else: - obj = self - # end if - for svar in svars: - obj.add_call_list_variable(svar, exists_ok=True) - # end for - dim_match = ':'.join(nloop_subst.required_stdnames) - # end if - elif not self.run_phase(): - if ((hdim == 'ccpp_constant_one:horizontal_dimension') and - (ndim == 'ccpp_constant_one:horizontal_loop_extent')): - dim_match = hdim - elif ((hdim == 'ccpp_constant_one:horizontal_dimension') and - (ndim == 'horizontal_loop_begin:horizontal_loop_end')): - dim_match = hdim - # end if (no else, there is no non-run-phase match) - # end if (no else, there is no match) - # end if (no else, there is no match) - return dim_match - - @staticmethod - def dim_match(need_dim, have_dim): - """Test whether matches . - If they match, return the matching dimension (which may be - modified by, e.g., a loop substitution). - If they do not match, return None. - """ - match = None - # First, try for all the marbles - if need_dim == have_dim: - match = need_dim - # end if - # Is one side missing a one start? - if not match: - ndims = need_dim.split(':') - hdims = have_dim.split(':') - if len(ndims) > len(hdims): - if ndims[0].lower == 'ccpp_constant_one': - ndims = ndims[1:] - elif hdims[0].lower == 'ccpp_constant_one': - hdims = hdims[1:] - # end if (no else) - # Last try - match = ndims == hdims - # end if - # end if - - return match - - def match_dimensions(self, need_dims, have_dims): - """Compare dimensions between and . - Return 6 items: - 1) Return True if all dims match. - If has a vertical dimension and does not - but all other dimensions match, return False but include the - missing dimension index as the third return value. - 2) Return modified, if necessary to - reflect the available limits. - 3) Return have_dims modified, if necessary to reflect - any loop substitutions. If no substitutions, return None - This is done so that the correct dimensions are used in the host cap. - 4) Return the name of the missing vertical index, or None - 5) Return a permutation array if the dimension ordering is - different (or None if the ordering is the same). Each element of the - permutation array is the index in for that dimension of - . - 6) Finally, return a 'reason' string. If match (first return value) is - False, this string will contain information about the reason for - the match failure. - >>> SuiteObject('foo', _API_CONTEXT, None, _API_DUMMY_RUN_ENV).match_dimensions(['horizontal_loop_extent'], ['horizontal_loop_extent']) - (True, ['horizontal_loop_extent'], ['horizontal_loop_extent'], None, '') - >>> SuiteObject('foo', _API_CONTEXT,None,_API_DUMMY_RUN_ENV,variables=[Var({'local_name':'beg','standard_name':'horizontal_loop_begin','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL,_API_DUMMY_RUN_ENV),Var({'local_name':'end','standard_name':'horizontal_loop_end','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV)],active_call_list=True,phase_type='initialize').match_dimensions(['ccpp_constant_one:horizontal_loop_extent'], ['ccpp_constant_one:horizontal_dimension']) - (True, ['ccpp_constant_one:horizontal_dimension'], ['ccpp_constant_one:horizontal_dimension'], None, '') - >>> SuiteObject('foo', _API_CONTEXT,None,_API_DUMMY_RUN_ENV,variables=[Var({'local_name':'beg','standard_name':'horizontal_loop_begin','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV),Var({'local_name':'end','standard_name':'horizontal_loop_end','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV)],active_call_list=True,phase_type=RUN_PHASE_NAME).match_dimensions(['ccpp_constant_one:horizontal_loop_extent'], ['horizontal_loop_begin:horizontal_loop_end']) - (True, ['horizontal_loop_begin:horizontal_loop_end'], ['horizontal_loop_begin:horizontal_loop_end'], None, '') - >>> SuiteObject('foo', _API_CONTEXT,None,_API_DUMMY_RUN_ENV,variables=[Var({'local_name':'beg','standard_name':'horizontal_loop_begin','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV),Var({'local_name':'end','standard_name':'horizontal_loop_end','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV),Var({'local_name':'lev','standard_name':'vertical_layer_dimension','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV)],active_call_list=True,phase_type=RUN_PHASE_NAME).match_dimensions(['ccpp_constant_one:horizontal_loop_extent','ccpp_constant_one:vertical_layer_dimension'], ['horizontal_loop_begin:horizontal_loop_end','ccpp_constant_one:vertical_layer_dimension']) - (True, ['horizontal_loop_begin:horizontal_loop_end', 'ccpp_constant_one:vertical_layer_dimension'], ['horizontal_loop_begin:horizontal_loop_end', 'ccpp_constant_one:vertical_layer_dimension'], None, '') - >>> SuiteObject('foo', _API_CONTEXT,None,_API_DUMMY_RUN_ENV,variables=[Var({'local_name':'beg','standard_name':'horizontal_loop_begin','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV),Var({'local_name':'end','standard_name':'horizontal_loop_end','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV),Var({'local_name':'lev','standard_name':'vertical_layer_dimension','units':'count','dimensions':'()','type':'integer'}, _API_LOCAL, _API_DUMMY_RUN_ENV)],active_call_list=True,phase_type=RUN_PHASE_NAME).match_dimensions(['ccpp_constant_one:horizontal_loop_extent','ccpp_constant_one:vertical_layer_dimension'], ['ccpp_constant_one:vertical_layer_dimension','horizontal_loop_begin:horizontal_loop_end']) - (True, ['horizontal_loop_begin:horizontal_loop_end', 'ccpp_constant_one:vertical_layer_dimension'], ['ccpp_constant_one:vertical_layer_dimension', 'horizontal_loop_begin:horizontal_loop_end'], [1, 0], '') - """ - new_need_dims = [] - new_have_dims = list(have_dims) - perm = [] - match = True - reason = '' - nlen = len(need_dims) - hlen = len(have_dims) - _, nvdim_index = find_vertical_dimension(need_dims) - _, hvdim_index = find_vertical_dimension(have_dims) - _, nhdim_index = find_horizontal_dimension(need_dims) - _, hhdim_index = find_horizontal_dimension(have_dims) - if hhdim_index < 0 <= nhdim_index: - match = False - nlen = 0 # To skip logic below - hlen = 0 # To skip logic below - reason = '{hname}{hctx} is missing a horizontal dimension ' - reason += 'required by {nname}{nctx}' - # end if - for nindex in range(nlen): - neddim = need_dims[nindex] - if nindex == nhdim_index: - # Look for a horizontal dimension match - vmatch = VarDictionary.loop_var_match(neddim) - hmatch = self.horiz_dim_match(neddim, have_dims[hhdim_index], - vmatch) - if hmatch: - perm.append(hhdim_index) - new_need_dims.append(hmatch) - new_have_dims[hhdim_index] = hmatch - found_ndim = True - else: - found_ndim = False - # end if - else: - # Find the first dimension in have_dims that matches neddim - found_ndim = False - if nvdim_index < 0 <= hvdim_index: - skip = hvdim_index - else: - skip = -1 - # end if - hdim_indices = [x for x in range(hlen) - if (x not in perm) and (x != skip)] - for hindex in hdim_indices: - if (hindex != hvdim_index) or (nvdim_index >= 0): - hmatch = self.dim_match(neddim, have_dims[hindex]) - if hmatch: - perm.append(hindex) - new_need_dims.append(hmatch) - new_have_dims[hindex] = hmatch - found_ndim = True - break - # end if - # end if - # end if - # end for - if not found_ndim: - match = False - reason = 'Could not find dimension, ' + neddim + ', in ' - reason += '{hname}{hctx}. Needed by {nname}{nctx}' - break - # end if (no else, we are still okay) - # end for - perm_test = list(range(hlen)) - # If no permutation is found, reset to None - if perm == perm_test: - perm = None - elif (not match): - perm = None - # end if (else, return perm as is) - if new_have_dims == have_dims: - have_dims = None # Do not make any substitutions - # end if - return match, new_need_dims, new_have_dims, perm, reason - - def find_variable(self, standard_name=None, source_var=None, - any_scope=True, clone=None, - search_call_list=False, loop_subst=False): - """Find a matching variable to , create a local clone (if - is True), or return None. - First search the SuiteObject's internal dictionary, then its - call list (unless is True, then any parent - dictionary (if is True). - can be a Var object or a standard_name string. - is not used by this version of . - """ - # First, search our local dictionary - if standard_name is None: - if source_var is None: - emsg = "One of or must be passed." - raise ParseInternalError(emsg) - # end if - standard_name = source_var.get_prop_value('standard_name') - elif source_var is not None: - stest = source_var.get_prop_value('standard_name') - if stest != standard_name: - emsg = (" and must match if " + - "both are passed.") - raise ParseInternalError(emsg) - # end if - # end if - scl = search_call_list - stdname = standard_name - # Don't clone yet, might find the variable further down - found_var = super().find_variable(standard_name=stdname, - source_var=source_var, - any_scope=False, clone=None, - search_call_list=scl, - loop_subst=loop_subst) - if (not found_var) and (self.call_list is not None) and scl: - # Don't clone yet, might find the variable further down - found_var = self.call_list.find_variable(standard_name=stdname, - source_var=source_var, - any_scope=False, - clone=None, - search_call_list=scl, - loop_subst=loop_subst) - # end if - loop_okay = VarDictionary.loop_var_okay(stdname, self.run_phase()) - if not loop_okay: - loop_subst = False - # end if - if (found_var is None) and any_scope and (self.parent is not None): - # We do not have the variable, look to parents. - found_var = self.parent.find_variable(standard_name=stdname, - source_var=source_var, - any_scope=True, - clone=clone, - search_call_list=scl, - loop_subst=loop_subst) - # end if - return found_var - - def match_variable(self, var, run_env): - """Try to find a source for in this SuiteObject's dictionary - tree. Several items are returned: - found_var: True if a match was found - vert_dim: The vertical dimension in , or None - call_dims: How this variable should be called (or None if no match) - perm: Permutation (XXgoldyXX: Not yet implemented) - """ - vstdname = var.get_prop_value('standard_name') - vdims = var.get_dimensions() - if (not vdims) and self.run_phase(): - vmatch = VarDictionary.loop_var_match(vstdname) - else: - vmatch = None - # end if - - found_var = False - new_vdims = list() - var_vdim = var.has_vertical_dimension(dims=vdims) - compat_obj = None - dict_var = None - if var.get_prop_value('type') == 'ccpp_constituent_properties_t': - if self.phase() == 'register': - found_var = True - new_vdims = [':'] - return found_var, dict_var, var_vdim, new_vdims, compat_obj - else: - errmsg = "Variables of type ccpp_constituent_properties_t only allowed in register phase: " - sname = var.get_prop_value('standard_name') - errmsg += f"'{sname}' found in {self.phase()} phase" - raise CCPPError(errmsg) - # end if - # end if - - # Does this variable exist in the calling tree? - dict_var = self.find_variable(source_var=var, any_scope=True) - if dict_var is None: - # No existing variable but add loop var match to call tree - found_var = self.parent.add_variable_to_call_tree(dict_var, - vmatch=vmatch) - new_vdims = vdims - elif dict_var.source.ptype in _API_LOCAL_VAR_TYPES: - # We cannot change the dimensions of locally-declared variables - # Using a loop substitution is invalid because the loop variable - # value has not yet been set. - # Therefore, we have to use the declaration dimensions in the call. - found_var = True - new_vdims = dict_var.get_dimensions() - else: - # Check dimensions - dict_dims = dict_var.get_dimensions() - if vdims: - args = self.parent.match_dimensions(vdims, dict_dims) - match, new_vdims, new_dict_dims, perm, err = args - if perm is not None: - errmsg = "Permuted indices are not yet supported" - lname = var.get_prop_value('local_name') - dstr = ', '.join(vdims) - ctx = context_string(var.context) - errmsg += ", var = {}({}){}".format(lname, dstr, ctx) - raise CCPPError(errmsg) - # end if - else: - new_vdims = list() - new_dict_dims = dict_dims - match = True - # end if - # If variable is defined as "inactive" by the host, ensure that - # this variable is declared as "optional" by the scheme. If - # not satisfied, return error. - host_var_active = dict_var.get_prop_value('active') - scheme_var_optional = var.get_prop_value('optional') - if (not scheme_var_optional and host_var_active.lower() != '.true.'): - errmsg = "Non optional scheme arguments for conditionally allocatable variables" - sname = dict_var.get_prop_value('standard_name') - errmsg += ", {}".format(sname) - raise CCPPError(errmsg) - # end if - # Add the variable to the parent call tree - if dict_dims == new_dict_dims: - sdict = {} - else: - sdict = {'dimensions':new_dict_dims} - # end if - found_var = self.parent.add_variable_to_call_tree(var, - subst_dict=sdict) - if not match: - found_var = False - nctx = context_string(var.context) - nname = var.get_prop_value('local_name') - hctx = context_string(dict_var.context) - hname = dict_var.get_prop_value('local_name') - raise CCPPError(err.format(nname=nname, nctx=nctx, - hname=hname, hctx=hctx)) - # end if - # end if - # We have a match! - # Are the Scheme's and Host's compatible? - # If so, create compatibility object, containing any necessary - # forward/reverse transforms to/from and . - if dict_var is not None: - dict_var = self.parent.find_variable(source_var=var, any_scope=True) - compat_obj = var.compatible(dict_var, run_env) - # end if - return found_var, dict_var, var_vdim, new_vdims, compat_obj - - def part(self, index, error=True): - """Return one of this SuiteObject's parts raise an exception, or, - if is False, just return None""" - plen = len(self.__parts) - if (0 <= index < plen) or (abs(index) <= plen): - return self.__parts[index] - # end if - if error: - errmsg = 'No part {} in {} {}'.format(index, - self.__class__.__name__, - self.name) - raise ParseInternalError(errmsg) - # end if - return None - - def has_item(self, item_name): - """Return True iff item, , is already in this SuiteObject""" - has = False - for item in self.__parts: - if item.name == item_name: - has = True - else: - has = item.has_item(item_name) - # end if - if has: - break - # end if - # end for - return has - - @property - def name(self): - """Return the name of the element""" - return self.__name - - @name.setter - def name(self, value): - """Set the name of the element if it has not been set""" - if self.__name is None: - self.__name = value - else: - errmsg = 'Attempt to change name of {} to {}' - raise ParseInternalError(errmsg.format(self, value)) - # end if - - @property - def parent(self): - """This SuiteObject's parent (or none)""" - return self.__parent - - @property - def call_list(self): - """Return the SuiteObject's call_list""" - return self.__call_list - - @property - def phase_type(self): - """Return the phase_type of this suite_object""" - return self.__phase_type - - @property - def parts(self): - """Return a copy the component parts of this SuiteObject. - Returning a copy allows for the part list to be changed during - processing of the return value""" - return self.__parts[:] - - @property - def context(self): - """Return the context of this SuiteObject""" - return self.__context - - @property - def run_env(self): - """Return the CCPPFrameworkEnv runtime object for this SuiteObject""" - return self.__run_env - - def __repr__(self): - """Create a unique readable string for this Object""" - so_repr = super().__repr__() - olmatch = _OBJ_LOC_RE.search(so_repr) - if olmatch is not None: - loc = ' at {}'.format(olmatch.group(1)) - else: - loc = "" - # end if - return '<{} {}{}>'.format(self.__class__.__name__, self.name, loc) - - def __format__(self, spec): - """Return a string representing the SuiteObject, including its children. - is used between subitems. - is the indent level for multi-line output. - """ - if spec: - sep = spec[0] - else: - sep = '\n' - # end if - try: - ind_level = int(spec[1:]) - except (ValueError, IndexError): - ind_level = 0 - # end try - if sep == '\n': - indent = " " - else: - indent = "" - # end if - if self.name == self.__class__.__name__: - # This object does not have separate name - nstr = self.name - else: - nstr = "{}: {}".format(self.__class__.__name__, self.name) - # end if - output = "{}<{}>".format(indent*ind_level, nstr) - subspec = "{}{}".format(sep, ind_level + 1) - substr = "{o}{s}{p:" + subspec + "}" - subout = "" - for part in self.parts: - subout = substr.format(o=subout, s=sep, p=part) - # end for - if subout: - output = "{}{}{}{}".format(output, subout, sep, - indent*ind_level, - self.__class__.__name__) - else: - output = "{}".format(output, self.__class__.__name__) - # end if - return output - -############################################################################### - -class Scheme(SuiteObject): - """A single scheme in a suite (e.g., init method)""" - - def __init__(self, scheme_xml, context, parent, run_env): - """Initialize this physics Scheme""" - name = scheme_xml.text - self.__subroutine_name = None - self.__context = context - self.__version = scheme_xml.get('version', None) - self.__lib = scheme_xml.get('lib', None) - self.__has_vertical_dimension = False - self.__group = None - self.__forward_transforms = list() - self.__reverse_transforms = list() - self._has_run_phase = True - self.__optional_vars = list() - super().__init__(name, context, parent, run_env, active_call_list=True) - - def update_group_call_list_variable(self, var): - """If is in our group's call list, update its intent. - Add to our group's call list unless: - - is in our group's call list - - is in our group's dictionary, - - is a suite variable""" - stdname = var.get_prop_value('standard_name') - my_group = self.__group - gvar = my_group.call_list.find_variable(standard_name=stdname, - any_scope=False) - if gvar: - gvar.adjust_intent(var) - else: - gvar = my_group.find_variable(standard_name=stdname, - any_scope=False) - if gvar is None: - # Check for suite variable - gvar = my_group.find_variable(standard_name=stdname, - any_scope=True) - if gvar and (not SuiteObject.is_suite_variable(gvar)): - gvar = None - # end if - if gvar is None: - my_group.add_call_list_variable(var, gen_unique=True) - # end if - # end if - - def is_local_variable(self, var): - """Return None as we never consider to be in our local - dictionary. - This is an override of the SuiteObject version""" - return None - - def analyze(self, phase, group, scheme_library, suite_vars, level): - """Analyze the scheme's interface to prepare for writing""" - self.__group = group - my_header = None - if self.name in scheme_library: - func = scheme_library[self.name] - if phase in func: - my_header = func[phase] - self.__subroutine_name = my_header.title - else: - self._has_run_phase = False - return set() - # end if - else: - estr = 'No schemes found for {}' - raise ParseInternalError(estr.format(self.name), - context=self.__context) - # end if - if my_header is None: - estr = 'No {} header found for scheme, {}' - raise ParseInternalError(estr.format(phase, self.name), - context=self.__context) - # end if - if my_header.module is None: - estr = 'No module found for subroutine, {}' - raise ParseInternalError(estr.format(self.subroutine_name), - context=self.__context) - # end if - scheme_mods = set() - scheme_mods.add((my_header.module, self.subroutine_name)) - for var in my_header.variable_list(): - vstdname = var.get_prop_value('standard_name') - def_val = var.get_prop_value('default_value') - vdims = var.get_dimensions() - vintent = var.get_prop_value('intent') - args = self.match_variable(var, self.run_env) - found, dict_var, vert_dim, new_dims, compat_obj = args - if found: - # Hack to get the missing dimensions promoted to the right place - # Add variable allocation checks for group, suite and host variables - if dict_var: - self.handle_downstream_variables(dict_var) - # end if - if not self.has_vertical_dim: - self.__has_vertical_dimension = vert_dim is not None - # end if - # We have a match, make sure var is in call list - if new_dims == vdims: - self.add_call_list_variable(var, exists_ok=True, gen_unique=True) - self.update_group_call_list_variable(var) - else: - subst_dict = {'dimensions':new_dims} - clone = var.clone(subst_dict) - self.add_call_list_variable(clone, exists_ok=True) - self.update_group_call_list_variable(clone) - # end if - else: - if vintent == 'out': - if self.__group is None: - errmsg = 'Group not defined for {}'.format(self.name) - raise ParseInternalError(errmsg) - # end if - # The Group will manage this variable - self.__group.manage_variable(var) - self.add_call_list_variable(var) - elif def_val and (vintent != 'out'): - if self.__group is None: - errmsg = 'Group not defined for {}'.format(self.name) - raise ParseInternalError(errmsg) - # end if - # The Group will manage this variable - self.__group.manage_variable(var) - # We still need it in our call list (the group uses a clone) - self.add_call_list_variable(var) - else: - errmsg = 'Input argument for {}, {}, not found.' - if self.find_variable(source_var=var) is not None: - # The variable exists, maybe it is dim mismatch - lname = var.get_prop_value('local_name') - emsg = '\nCheck for dimension mismatch in {}' - errmsg += emsg.format(lname) - # end if - if ((not self.run_phase()) and - (vstdname in CCPP_LOOP_VAR_STDNAMES)): - emsg = '\nLoop variables not allowed in {} phase.' - errmsg += emsg.format(self.phase()) - # end if - raise CCPPError(errmsg.format(self.subroutine_name, - vstdname)) - # end if - # end if - # Are there any forward/reverse transforms for this variable? - has_transform = False - if compat_obj is not None and (compat_obj.has_vert_transforms or - compat_obj.has_unit_transforms or - compat_obj.has_kind_transforms): - self.add_var_transform(var, compat_obj, vert_dim) - has_transform = True - # end if - - # Is this a conditionally allocated variable? - # If so, declare localpointer variable. This is needed to - # pass inactive (not present) status through the caps. - if var.get_prop_value('optional'): - newvar_ptr = var.clone(var.get_prop_value('local_name')+'_ptr') - self.__optional_vars.append([dict_var, var, newvar_ptr, has_transform]) - # end if - - # end for - return scheme_mods - - def handle_downstream_variables(self, var): - """Ensure all dimension and optional variable arguments are available""" - # Get the basic attributes that decide whether we need - # to check the variable when we write the group - standard_name = var.get_prop_value('standard_name') - dimensions = var.get_dimensions() - active = var.get_prop_value('active') - var_dicts = [ self.__group.call_list ] + self.__group.suite_dicts() - - # If the variable isn't active, skip it - if active.lower() =='.false.': - return - # Also, if the variable is one of the CCPP error handling messages, skip it - # since it is defined as intent(out) and we can't do meaningful checks on it - elif standard_name == 'ccpp_error_code' or standard_name == 'ccpp_error_message': - return - # To perform allocation checks, we need to know all variables - # that are part of the 'active' attribute conditional and add - # it to the group's call list. - else: - (_, vars_needed) = var.conditional(var_dicts) - for var_needed in vars_needed: - self.update_group_call_list_variable(var_needed) - - # For arrays, we need to get information on the dimensions and add it to - # the group's call list so that we can test for the correct size later on - if dimensions: - for dim in dimensions: - if not ':' in dim: - dim_var = self.find_variable(standard_name=dim) - if not dim_var: - # To allow for numerical dimensions in metadata. - if not dim.isnumeric(): - raise Exception(f"No dimension with standard name '{dim}'") - # end if - else: - self.update_group_call_list_variable(dim_var) - # end if - else: - (ldim, udim) = dim.split(":") - ldim_var = self.find_variable(standard_name=ldim) - if not ldim_var: - # To allow for numerical dimensions in metadata. - if not ldim.isnumeric(): - raise Exception(f"No dimension with standard name '{ldim}'") - # end if - # end if - self.update_group_call_list_variable(ldim_var) - udim_var = self.find_variable(standard_name=udim) - if not udim_var: - # To allow for numerical dimensions in metadata. - if not udim.isnumeric(): - raise Exception(f"No dimension with standard name '{udim}'") - # end if - else: - self.update_group_call_list_variable(udim_var) - # end if - - def associate_optional_var(self, dict_var, var, var_ptr, has_transform, cldicts, indent, outfile): - """Write local pointer association for optional variables.""" - if (dict_var): - (conditional, _) = dict_var.conditional(cldicts) - if (has_transform): - lname = var.get_prop_value('local_name')+'_local' - else: - lname = var.get_prop_value('local_name') - # end if - lname_ptr = var_ptr.get_prop_value('local_name') - outfile.write(f"if {conditional} then", indent) - outfile.write(f"{lname_ptr} => {lname}", indent+1) - outfile.write(f"end if", indent) - # end if - - def assign_pointer_to_var(self, dict_var, var, var_ptr, has_transform, cldicts, indent, outfile): - """Assign local pointer to variable.""" - if (dict_var): - intent = var.get_prop_value('intent') - if (intent == 'out' or intent == 'inout'): - (conditional, _) = dict_var.conditional(cldicts) - if (has_transform): - lname = var.get_prop_value('local_name')+'_local' - else: - lname = var.get_prop_value('local_name') - # end if - lname_ptr = var_ptr.get_prop_value('local_name') - outfile.write(f"if {conditional} then", indent) - outfile.write(f"{lname} = {lname_ptr}", indent+1) - outfile.write(f"end if", indent) - # end if - # end if - - def add_var_transform(self, var, compat_obj, vert_dim): - """Register any variable transformation needed by for this Scheme. - For any transformation identified in , create dummy variable - from to perform the transformation. Determine the indices needed - for the transform and save for use during write stage""" - - # Add local variable (_local) needed for transformation. - # Do not let the Group manage this variable. Handle local var - # when writing Group. - prop_dict = var.copy_prop_dict() - prop_dict['local_name'] = var.get_prop_value('local_name')+'_local' - # This is a local variable. - if 'intent' in prop_dict: - del prop_dict['intent'] - # end if - local_trans_var = Var(prop_dict, - ParseSource(_API_SOURCE_NAME, - _API_LOCAL_VAR_NAME, var.context), - self.run_env) - found = self.__group.find_variable(source_var=local_trans_var, any_scope=False) - if not found: - lmsg = "Adding new local variable, '{}', for variable transform" - self.run_env.logger.info(lmsg.format(local_trans_var.get_prop_value('local_name'))) - self.__group.transform_locals.append(local_trans_var) - # end if - - # Create indices (default) for transform. - lindices = [':']*var.get_rank() - rindices = [':']*var.get_rank() - - # If needed, modify vertical dimension for vertical orientation flipping - _, vdim = find_vertical_dimension(var.get_dimensions()) - if vdim >= 0: - vdims = vert_dim.split(':') - vdim_name = vdims[-1] - group_vvar = self.__group.call_list.find_variable(vdim_name) - if group_vvar is None: - raise CCPPError(f"add_var_transform: Cannot find dimension variable, {vdim_name}") - # end if - vname = group_vvar.get_prop_value('local_name') - if len(vdims) == 2: - sdim_name = vdims[0] - group_vvar = self.find_variable(sdim_name) - if group_vvar is None: - raise CCPPError(f"add_var_transform: Cannot find dimension variable, {sdim_name}") - # end if - sname = group_vvar.get_prop_value('local_name') - else: - sname = '1' - # end if - lindices[vdim] = sname+':'+vname - if compat_obj.has_vert_transforms: - rindices[vdim] = vname+':'+sname+':-1' - else: - rindices[vdim] = sname+':'+vname - # end if - # end if - - # If needed, modify horizontal dimension for loop substitution. - # NOT YET IMPLEMENTED - #hdim = find_horizontal_dimension(var.get_dimensions()) - #if compat_obj.has_dim_transforms: - - # Register any reverse (pre-Scheme) transforms. Also, save local_name used in - # transform (used in write stage). - if (var.get_prop_value('intent') != 'out'): - lmsg = "Automatic unit conversion from '{}' to '{}' for '{}' before entering '{}'" - self.run_env.logger.info(lmsg.format(compat_obj.v2_units, - compat_obj.v1_units, - compat_obj.v2_stdname, - self.__subroutine_name)) - self.__reverse_transforms.append([local_trans_var.get_prop_value('local_name'), - var.get_prop_value('local_name'), - var.get_prop_value('standard_name'), - rindices, lindices, compat_obj]) - # end if - # Register any forward (post-Scheme) transforms. - if (var.get_prop_value('intent') != 'in'): - lmsg = "Automatic unit conversion from '{}' to '{}' for '{}' after returning '{}'" - self.run_env.logger.info(lmsg.format(compat_obj.v1_units, - compat_obj.v2_units, - compat_obj.v1_stdname, - self.__subroutine_name)) - self.__forward_transforms.append([var.get_prop_value('local_name'), - var.get_prop_value('standard_name'), - local_trans_var.get_prop_value('local_name'), - lindices, rindices, compat_obj]) - # end if - def write_var_transform(self, var, dummy, rindices, lindices, compat_obj, - outfile, indent, forward): - """Write variable transformation needed to call this Scheme in . - is the variable that needs transformation before and after calling Scheme. - is the local variable needed for the transformation.. - are the LHS indices of for reverse transforms (before Scheme). - are the RHS indices of for reverse transforms (before Scheme). - are the LHS indices of for forward transforms (after Scheme). - are the RHS indices of for forward transforms (after Scheme). - """ - # - # Write reverse (pre-Scheme) transform. - # - if not forward: - # dummy(lindices) = var(rindices) - stmt = compat_obj.reverse_transform(lvar_lname=dummy, - rvar_lname=var, - lvar_indices=lindices, - rvar_indices=rindices) - # - # Write forward (post-Scheme) transform. - # - else: - # var(lindices) = dummy(rindices) - stmt = compat_obj.forward_transform(lvar_lname=var, - rvar_lname=dummy, - lvar_indices=rindices, - rvar_indices=lindices) - # end if - outfile.write(stmt, indent) - - def write(self, outfile, errcode, errmsg, indent): - # Unused arguments are for consistent write interface - # pylint: disable=unused-argument - """Write code to call this Scheme to """ - # Dictionaries to try are our group, the group's call list, - # or our module - cldicts = [self.__group, self.__group.call_list] - cldicts.extend(self.__group.suite_dicts()) - my_args = self.call_list.call_string(cldicts=cldicts, - is_func_call=True, - subname=self.subroutine_name, - sub_lname_list = self.__reverse_transforms) - # - outfile.write('', indent) - outfile.write('if ({} == 0) then'.format(errcode), indent) - # - # Write any reverse (pre-Scheme) transforms. - if len(self.__reverse_transforms) > 0: - outfile.comment('Compute reverse (pre-scheme) transforms', indent+1) - # end if - for rcnt, (dummy, var_lname, var_sname, rindices, lindices, compat_obj) in enumerate(self.__reverse_transforms): - # Any transform(s) were added during the Group's analyze phase, but - # the local_name(s) of the assoicated with the transform(s) - # may have since changed. Here we need to use the standard_name - # from and replace its local_name with the local_name from the - # Group's call_list. - lvar = self.__group.call_list.find_variable(standard_name=var_sname) - lvar_lname = lvar.get_prop_value('local_name') - tstmt = self.write_var_transform(lvar_lname, dummy, rindices, lindices, compat_obj, outfile, indent+1, False) - # end for - outfile.write('',indent+1) - # - # Associate any conditionally allocated variables. - # - if self.__optional_vars: - outfile.write('! Associate conditional variables', indent+1) - # end if - for (dict_var, var, var_ptr, has_transform) in self.__optional_vars: - tstmt = self.associate_optional_var(dict_var, var, var_ptr, has_transform, cldicts, indent+1, outfile) - # end for - # - # Write the scheme call. - # - if self._has_run_phase: - stmt = 'call {}({})' - outfile.write('',indent+1) - outfile.write('! Call scheme', indent+1) - outfile.write(stmt.format(self.subroutine_name, my_args), indent+1) - outfile.write('',indent+1) - # end if - # - # Copy any local pointers. - # - first_ptr_declaration=True - for (dict_var, var, var_ptr, has_transform) in self.__optional_vars: - if first_ptr_declaration: - outfile.write('! Copy any local pointers to dummy/local variables', indent+1) - first_ptr_declaration=False - # end if - tstmt = self.assign_pointer_to_var(dict_var, var, var_ptr, has_transform, cldicts, indent+1, outfile) - # end for - outfile.write('',indent+1) - # - # Write any forward (post-Scheme) transforms. - # - if len(self.__forward_transforms) > 0: - outfile.comment('Compute forward (post-scheme) transforms', indent+1) - # end if - for fcnt, (var_lname, var_sname, dummy, lindices, rindices, compat_obj) in enumerate(self.__forward_transforms): - # Any transform(s) were added during the Group's analyze phase, but - # the local_name(s) of the assoicated with the transform(s) - # may have since changed. Here we need to use the standard_name - # from and replace its local_name with the local_name from the - # Group's call_list. - lvar = self.__group.call_list.find_variable(standard_name=var_sname) - lvar_lname = lvar.get_prop_value('local_name') - tstmt = self.write_var_transform(lvar_lname, dummy, rindices, lindices, compat_obj, outfile, indent+1, True) - # end for - outfile.write('', indent) - outfile.write('end if', indent) - - def schemes(self): - """Return self as a list for consistency with subcycle""" - return [self] - - def variable_list(self, recursive=False, - std_vars=True, loop_vars=True, consts=True): - """Return a list of all variables for this Scheme. - Because Schemes do not have any variables, return a list - of this object's CallList variables instead. - Note that because of this, is not allowed.""" - if recursive: - raise ParseInternalError("recursive=True not allowed for Schemes") - # end if - return self.call_list.variable_list(recursive=recursive, - std_vars=std_vars, - loop_vars=loop_vars, consts=consts) - - @property - def subroutine_name(self): - """Return this scheme's actual subroutine name""" - return self.__subroutine_name - - @property - def has_vertical_dim(self): - """Return True if at least one of this Scheme's variables has - a vertical dimension (vertical_layer_dimension or - vertical_interface_dimension) - """ - return self.__has_vertical_dimension - - def __str__(self): - """Create a readable string for this Scheme""" - return ''.format(self.name, self.subroutine_name) - -############################################################################### - -class Subcycle(SuiteObject): - """Class to represent a subcycled group of schemes or scheme collections""" - - def __init__(self, sub_xml, context, parent, run_env, loop_count=0): - self._loop_extent = sub_xml.get('loop', "1") # Number of iterations - self._loop = None - # See if our loop variable is an integer or a variable - try: - _ = int(self._loop_extent) - self._loop = self._loop_extent - self._loop_var_int = True - name = f"loop{loop_count}" - super().__init__(name, context, parent, run_env, active_call_list=False) - loop_count = loop_count + 1 - except ValueError: - self._loop_var_int = False - lvar = parent.find_variable(standard_name=self._loop_extent, any_scope=True) - if lvar is None: - emsg = "Subcycle, {}, specifies {} iterations, variable not found" - raise CCPPError(emsg.format(name, self._loop_extent)) - else: - self._loop_var_int = False - self._loop = lvar.get_prop_value('local_name') - # end if - name = f"loop{loop_count}_{self._loop_extent}"[0:63] - super().__init__(name, context, parent, run_env, active_call_list=True) - parent.add_call_list_variable(lvar, exists_ok=True) - loop_count = loop_count + 1 - # end try - for item in sub_xml: - new_item = new_suite_object(item, context, self, run_env, loop_count=loop_count) - self.add_part(new_item) - # end for - - def analyze(self, phase, group, scheme_library, suite_vars, level): - """Analyze the Subcycle's interface to prepare for writing""" - if self.name is None: - self.name = "subcycle_index{}".format(level) - # end if - # Create a Group variable for the subcycle index. - newvar = Var({'local_name':self.name, 'standard_name':self.name, - 'type':'integer', 'units':'count', 'dimensions':'()'}, - _API_LOCAL, self.run_env) - group.manage_variable(newvar) - # Handle all the suite objects inside of this subcycle - scheme_mods = set() - for item in self.parts: - smods = item.analyze(phase, group, scheme_library, - suite_vars, level+1) - for smod in smods: - scheme_mods.add(smod) - # end for - # end for - return scheme_mods - - def write(self, outfile, errcode, errmsg, indent): - """Write code for the subcycle loop, including contents, to """ - outfile.write('do {} = 1, {}'.format(self.name, self._loop), indent) - # Note that 'scheme' may be a sybcycle or other construct - for item in self.parts: - item.write(outfile, errcode, errmsg, indent+1) - # end for - outfile.write('end do', 2) - - @property - def loop(self): - """Return the loop value or variable local_name""" - return self._loop - -############################################################################### - -class Group(SuiteObject): - """Class to represent a grouping of schemes in a suite - A Group object is implemented as a subroutine callable by the API. - The main arguments to a group are the host model variables. - Additional output arguments are generated from schemes with intent(out) - arguments. - Additional input or inout arguments are generated for inputs needed by - schemes which are produced (intent(out)) by other groups. - """ - - __subhead = ''' - subroutine {subname}({args}) -''' - - __subend = ''' - end subroutine {subname} - -! ======================================================================== -''' - - __thread_check = CodeBlock([('#ifdef _OPENMP', -1), - ('if (omp_get_thread_num() > 1) then', 1), - ('{errcode} = 1', 2), - (('{errmsg} = "Cannot call {phase} routine ' - 'from a threaded region"'), 2), - ('return', 2), - ('end if', 1), - ('#endif', -1)]) - - - def __init__(self, group_xml, transition, parent, context, run_env): - """Initialize this Group object from . - is the group's phase, is the group's suite. - """ - name = parent.name + '_' + group_xml.get('name') - if transition not in CCPP_STATE_MACH.transitions(): - errmsg = "Bad transition argument to Group, '{}'" - raise ParseInternalError(errmsg.format(transition)) - # end if - # Initialize the dictionary of variables internal to group - super().__init__(name, context, parent, run_env, - active_call_list=True, phase_type=transition) - add_to = self - # Add the sub objects - for item in group_xml: - new_item = new_suite_object(item, context, add_to, run_env) - add_to.add_part(new_item) - # end for - self._local_schemes = set() - self._host_vars = None - self._host_ddts = None - self._loop_var_matches = list() - self._phase_check_stmts = list() - self._set_state = None - self._ddt_library = None - self.transform_locals = list() - - def phase_match(self, scheme_name): - """If scheme_name matches the group phase, return the group and - function ID. Otherwise, return None - """ - fid, tid, _ = CCPP_STATE_MACH.transition_match(scheme_name, - transition=self.phase()) - if tid is not None: - return self, fid - # end if - return None, None - - def move_to_call_list(self, standard_name): - """Move a variable from the group internal dictionary to the call list. - This is done when the variable, , will be allocated by - the suite. - """ - gvar = self.find_variable(standard_name=standard_name, any_scope=False) - if gvar is None: - errmsg = "Group {}, cannot move {}, variable not found" - raise ParseInternalError(errmsg.format(self.name, standard_name)) - # end if - self.add_call_list_variable(gvar, exists_ok=True) - self.remove_variable(standard_name) - - def register_action(self, vaction): - """Register any recognized type for use during self.write. - Return True iff is handled. - """ - if isinstance(vaction, VarLoopSubst): - self._loop_var_matches = vaction.add_to_list(self._loop_var_matches) - # Add the missing dim - vaction.add_local(self, _API_LOCAL, self.run_env) - return True - # end if - return False - - def manage_variable(self, newvar): - """Add to our local dictionary making necessary - modifications to the variable properties so that it is - allocated appropriately""" - # Need new prop dict to eliminate unwanted properties (e.g., intent) - vdims = newvar.get_dimensions() - # Look for dimensions where we have a loop substitution and replace - # with the correct size - if self.run_phase(): - hdims = [x.missing_stdname for x in self._loop_var_matches] - else: - # Do not do loop substitutions in full phases - hdims = list() - # end if - for index, dim in enumerate(vdims): - newdim = None - for subdim in dim.split(':'): - if subdim in hdims: - # We have a loop substitution, find and replace - hindex = hdims.index(subdim) - names = self._loop_var_matches[hindex].required_stdnames - newdim = ':'.join(names) - break - # end if - if ('vertical' in subdim) and ('index' in subdim): - # We have a vertical index, replace with correct dimension - errmsg = "vertical index replace not implemented" - raise ParseInternalError(errmsg) - # end if - # end for - if newdim is not None: - vdims[index] = newdim - # end if - # end for - if self.timestep_phase(): - persist = 'timestep' - else: - persist = 'run' - # end if - # Start with an official copy of 's prop_dict with - # corrected dimensions - subst_dict = {'dimensions':vdims} - prop_dict = newvar.copy_prop_dict(subst_dict=subst_dict) - # Add the allocatable items - prop_dict['allocatable'] = len(vdims) > 0 # No need to allocate scalar - prop_dict['persistence'] = persist - # This is a local variable - if 'intent' in prop_dict: - del prop_dict['intent'] - # end if - # Create a new variable, save the original context - local_var = Var(prop_dict, - ParseSource(_API_SOURCE_NAME, - _API_LOCAL_VAR_NAME, newvar.context), - self.run_env) - self.add_variable(local_var, self.run_env, exists_ok=True, gen_unique=True) - # Finally, make sure all dimensions are accounted for - emsg = self.add_variable_dimensions(local_var, [_API_LOCAL_VAR_NAME], - _API_SUITE_VAR_NAME, - adjust_intent=True, - to_dict=self.call_list) - if emsg: - raise CCPPError(emsg) - # end if - - def analyze(self, phase, suite_vars, scheme_library, ddt_library, - check_suite_state, set_suite_state): - """Analyze the Group's interface to prepare for writing""" - self._ddt_library = ddt_library - # Sanity check for Group - if phase != self.phase(): - errmsg = 'Group {} has phase {} but analyze is phase {}' - raise ParseInternalError(errmsg.format(self.name, - self.phase(), phase)) - # end if - for item in self.parts: - # Items can be schemes, subcycles or other objects - # All have the same interface and return a set of module use - # statements (lschemes) - lschemes = item.analyze(phase, self, scheme_library, suite_vars, 1) - for lscheme in lschemes: - self._local_schemes.add(lscheme) - # end for - # end for - self._phase_check_stmts = check_suite_state - self._set_state = set_suite_state - if (self.run_env.logger and - self.run_env.logger.isEnabledFor(logging.DEBUG)): - self.run_env.logger.debug("{}".format(self)) - # end if - - def allocate_dim_str(self, dims, context): - """Create the dimension string for an allocate statement""" - rdims = list() - for dim in dims: - rdparts = list() - dparts = dim.split(':') - for dpart in dparts: - dvar = self.find_variable(standard_name=dpart, any_scope=False) - if dvar is None: - dvar = self.call_list.find_variable(standard_name=dpart, - any_scope=False) - # end if - if dvar is None: - # Check if it's a module-level variable - dvar = self.find_variable(standard_name=dpart, any_scope=True) - # end if - if dvar is None: - emsg = "Dimension variable, '{}', not found{}" - lvar = self.find_local_name(dpart, any_scope=True) - if lvar is not None: - emsg += "\nBe sure to use standard names!" - # end if - ctx = context_string(context) - raise CCPPError(emsg.format(dpart, ctx)) - # end if - lname = dvar.get_prop_value('local_name') - rdparts.append(lname) - # end for - rdims.append(':'.join(rdparts)) - # end for - return ', '.join(rdims) - - def find_variable(self, standard_name=None, source_var=None, - any_scope=True, clone=None, - search_call_list=False, loop_subst=False): - """Find a matching variable to , create a local clone (if - is True), or return None. - This purpose of this special Group version is to record any constituent - variable found for processing during the write phase. - """ - fvar = super().find_variable(standard_name=standard_name, - source_var=source_var, - any_scope=any_scope, clone=clone, - search_call_list=search_call_list, - loop_subst=loop_subst) - if fvar and fvar.is_constituent(): - if fvar.source.ptype == ConstituentVarDict.constitutent_source_type(): - # We found this variable in the constituent dictionary, - # add it to our call list - self.add_call_list_variable(fvar, exists_ok=True) - # end if - # end if - return fvar - - def write(self, outfile, host_arglist, indent, const_mod, - suite_vars=None, allocate=False, deallocate=False): - """Write code for this subroutine (Group), including contents, - to """ - # Unused arguments are for consistent write interface - # pylint: disable=unused-argument - # group type for (de)allocation - if self.timestep_phase(): - group_type = 'timestep' # Just allocate for the timestep - else: - group_type = 'run' # Allocate for entire run - # end if - # Collect information on local variables - subpart_allocate_vars = {} - subpart_optional_vars = {} - subpart_scalar_vars = {} - allocatable_var_set = set() - optional_var_set = set() - pointer_var_set = list() - inactive_var_set = set() - for item in [self]:# + self.parts: - for var in item.declarations(): - lname = var.get_prop_value('local_name') - sname = var.get_prop_value('standard_name') - if (lname in subpart_allocate_vars) or (lname in subpart_optional_vars) or (lname in subpart_scalar_vars): - if subpart_allocate_vars[lname][0].compatible(var, self.run_env): - pass # We already are going to declare this variable - else: - errmsg = "Duplicate Group variable, {}" - raise ParseInternalError(errmsg.format(lname)) - # end if - else: - opt_var = var.get_prop_value('optional') - dims = var.get_dimensions() - if (dims is not None) and dims: - if opt_var: - if (self.call_list.find_variable(standard_name=sname)): - subpart_optional_vars[lname] = (var, item, opt_var) - optional_var_set.add(lname) - else: - inactive_var_set.add(var) - # end if - else: - subpart_allocate_vars[lname] = (var, item, opt_var) - allocatable_var_set.add(lname) - # end if - else: - subpart_scalar_vars[lname] = (var, item, opt_var) - # end if - # end if - # end for - # All optional dummy variables within group need to have - # an associated pointer array declared. - for cvar in self.call_list.variable_list(): - opt_var = cvar.get_prop_value('optional') - if opt_var: - name = cvar.get_prop_value('local_name')+'_ptr' - kind = cvar.get_prop_value('kind') - dims = cvar.get_dimensions() - if cvar.is_ddt(): - vtype = 'type' - else: - vtype = cvar.get_prop_value('type') - # end if - if dims: - dimstr = '(:' + ',:'*(len(dims) - 1) + ')' - else: - dimstr = '' - # end if - pointer_var_set.append([name,kind,dimstr,vtype]) - # end if - # end for - # Any optional arguments that are not requested by the host need to have - # a local null pointer passed from the group to the scheme. - for ivar in inactive_var_set: - name = ivar.get_prop_value('local_name')+'_ptr' - kind = ivar.get_prop_value('kind') - dims = ivar.get_dimensions() - if ivar.is_ddt(): - vtype = 'type' - else: - vtype = ivar.get_prop_value('type') - # end if - if dims: - dimstr = '(:' + ',:'*(len(dims) - 1) + ')' - else: - dimstr = '' - # end if - pointer_var_set.append([name,kind,dimstr,vtype]) - # end for - # Any arguments used in variable transforms before or after the - # Scheme call? If so, declare local copy for reuse in the Group cap. - for ivar in self.transform_locals: - lname = ivar.get_prop_value('local_name') - opt_var = ivar.get_prop_value('optional') - dims = ivar.get_dimensions() - if (dims is not None) and dims: - subpart_allocate_vars[lname] = (ivar, item, opt_var) - allocatable_var_set.add(lname) - else: - subpart_scalar_vars[lname] = (ivar, item, opt_var) - # end if - # end for - - # end for - # First, write out the subroutine header - subname = self.name - call_list = self.call_list.call_string() - outfile.write(Group.__subhead.format(subname=subname, args=call_list), - indent) - # Write out any use statements - if self._local_schemes: - modmax = max([len(s[0]) for s in self._local_schemes]) - else: - modmax = 0 - # end if - # Write out the scheme use statements - scheme_use = 'use {},{} only: {}' - for scheme in sorted(self._local_schemes): - smod = scheme[0] - sname = scheme[1] - slen = ' '*(modmax - len(smod)) - outfile.write(scheme_use.format(smod, slen, sname), indent+1) - # end for - # Look for any DDT types - call_vars = self.call_list.variable_list() - all_vars = ([x[0] for x in subpart_allocate_vars.values()] + - [x[0] for x in subpart_scalar_vars.values()] + - [x[0] for x in subpart_optional_vars.values()]) - all_vars.extend(call_vars) - self._ddt_library.write_ddt_use_statements(all_vars, outfile, - indent+1, pad=modmax) - outfile.write('', 0) - # Write out dummy arguments - outfile.write('! Dummy arguments', indent+1) - msg = 'Variables for {}: ({})' - if (self.run_env.logger and - self.run_env.logger.isEnabledFor(logging.DEBUG)): - self.run_env.logger.debug(msg.format(self.name, call_vars)) - # end if - self.call_list.declare_variables(outfile, indent+1, dummy=True) - # DECLARE local variables - if subpart_allocate_vars or subpart_scalar_vars or subpart_optional_vars: - outfile.write('\n! Local Variables', indent+1) - # end if - # Scalars - for key in subpart_scalar_vars: - var = subpart_scalar_vars[key][0] - spdict = subpart_scalar_vars[key][1] - target = subpart_scalar_vars[key][2] - var.write_def(outfile, indent+1, spdict, - allocatable=False, target=target) - # end for - # Allocatable arrays - for key in subpart_allocate_vars: - var = subpart_allocate_vars[key][0] - spdict = subpart_allocate_vars[key][1] - target = subpart_allocate_vars[key][2] - var.write_def(outfile, indent+1, spdict, - allocatable=(key in allocatable_var_set), - target=target) - # end for - # Target arrays. - for key in subpart_optional_vars: - var = subpart_optional_vars[key][0] - spdict = subpart_optional_vars[key][1] - target = subpart_optional_vars[key][2] - var.write_def(outfile, indent+1, spdict, - allocatable=(key in optional_var_set), - target=target) - # end for - # Pointer variables - for (name, kind, dim, vtype) in pointer_var_set: - var.write_ptr_def(outfile, indent+1, name, kind, dim, vtype) - # end for - outfile.write('', 0) - # Get error variable names - if self.run_env.use_error_obj: - raise ParseInternalError("Error object not supported") - else: - verrcode = self.call_list.find_variable(standard_name='ccpp_error_code') - if verrcode is not None: - errcode = verrcode.get_prop_value('local_name') - else: - errmsg = "No ccpp_error_code variable for group, {}" - raise CCPPError(errmsg.format(self.name)) - # end if - verrmsg = self.call_list.find_variable(standard_name='ccpp_error_message') - if verrmsg is not None: - errmsg = verrmsg.get_prop_value('local_name') - else: - errmsg = "No ccpp_error_message variable for group, {}" - raise CCPPError(errmsg.format(self.name)) - # end if - # Initialize error variables - outfile.write("! Initialize ccpp error handling", 2) - outfile.write("{} = 0".format(errcode), 2) - outfile.write("{} = ''".format(errmsg), 2) - outfile.write("",2) - # end if - # Output threaded region check (except for run phase) - if not self.run_phase(): - outfile.write("! Output threaded region check ",indent+1) - Group.__thread_check.write(outfile, indent, - {'phase' : self.phase(), - 'errcode' : errcode, - 'errmsg' : errmsg}) - # Check state machine - outfile.write("! Check state machine",indent+1) - self._phase_check_stmts.write(outfile, indent, - {'errcode' : errcode, 'errmsg' : errmsg, - 'funcname' : self.name}) - # Write any loop match calculations - outfile.write("! Set horizontal loop extent",indent+1) - for vmatch in self._loop_var_matches: - action = vmatch.write_action(self, dict2=self.call_list) - if action: - outfile.write(action, indent+1) - # end if - # end for - # Allocate local arrays - outfile.write('\n! Allocate local arrays', indent+1) - alloc_stmt = "allocate({}({}))" - for lname in sorted(allocatable_var_set): - var = subpart_allocate_vars[lname][0] - dims = var.get_dimensions() - alloc_str = self.allocate_dim_str(dims, var.context) - outfile.write(alloc_stmt.format(lname, alloc_str), indent+1) - # end for - for lname in optional_var_set: - var = subpart_optional_vars[lname][0] - dims = var.get_dimensions() - alloc_str = self.allocate_dim_str(dims, var.context) - outfile.write(alloc_stmt.format(lname, alloc_str), indent+1) - # end for - # Allocate suite vars - if allocate: - outfile.write('\n! Allocate suite_vars', indent+1) - for svar in suite_vars.variable_list(): - dims = svar.get_dimensions() - if dims: - timestep_var = svar.get_prop_value('persistence') - if group_type == timestep_var: - alloc_str = self.allocate_dim_str(dims, svar.context) - lname = svar.get_prop_value('local_name') - outfile.write(alloc_stmt.format(lname, alloc_str), - indent+1) - # end if (do not allocate in this phase) - # end if dims (do not allocate scalars) - # end for - # end if - # Write the scheme and subcycle calls - for item in self.parts: - item.write(outfile, errcode, errmsg, indent + 1) - # end for - # Deallocate local arrays - if allocatable_var_set: - outfile.write('\n! Deallocate local arrays', indent+1) - # end if - for lname in sorted(allocatable_var_set): - outfile.write('if (allocated({})) {} deallocate({})'.format(lname,' '*(20-len(lname)),lname), indent+1) - # end for - for lname in optional_var_set: - outfile.write('if (allocated({})) {} deallocate({})'.format(lname,' '*(20-len(lname)),lname), indent+1) - # end for - # Nullify local pointers - if pointer_var_set: - outfile.write('\n! Nullify local pointers', indent+1) - # end if - for (name, kind, dim, vtype) in pointer_var_set: - #cspace = ' '*(15-len(name)) - outfile.write('if (associated({})) {} nullify({})'.format(name,' '*(15-len(name)),name), indent+1) - # end fo - # Deallocate suite vars - if deallocate: - for svar in suite_vars.variable_list(): - dims = svar.get_dimensions() - if dims: - timestep_var = svar.get_prop_value('persistence') - if group_type == timestep_var: - lname = svar.get_prop_value('local_name') - outfile.write('deallocate({})'.format(lname), indent+1) - # end if - # end if (no else, do not deallocate scalars) - # end for - # end if - self._set_state.write(outfile, indent, {}) - # end if - outfile.write(Group.__subend.format(subname=subname), indent) - - @property - def suite(self): - """Return this Group's suite""" - return self.parent - - def suite_dicts(self): - """Return a list of this Group's Suite's dictionaries""" - return self.suite.suite_dicts() - -############################################################################### - -if __name__ == "__main__": - # First, run doctest - # pylint: disable=ungrouped-imports - import doctest - import sys - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if diff --git a/scripts/var_props.py b/scripts/var_props.py deleted file mode 100755 index d732ae5e..00000000 --- a/scripts/var_props.py +++ /dev/null @@ -1,1565 +0,0 @@ -#!/usr/bin/env python3 - -""" -Classes and supporting code to hold all information on the compatibility of -two CCPP metadata variables. -VariableProperty: Class which describes a single variable property -VarCompatObj -""" - -# Python library imports -import keyword -import re -# CCPP framework imports -from conversion_tools import unit_conversion -from framework_env import CCPPFrameworkEnv -from parse_tools import check_local_name, check_fortran_type, context_string -from parse_tools import check_molar_mass -from parse_tools import FORTRAN_DP_RE, FORTRAN_SCALAR_REF_RE, fortran_list_match -from parse_tools import check_units, check_dimensions, check_cf_standard_name -from parse_tools import check_diagnostic_id, check_diagnostic_fixed -from parse_tools import check_default_value, check_valid_values -from parse_tools import ParseContext, ParseSource -from parse_tools import ParseInternalError, ParseSyntaxError, CCPPError - -############################################################################### -_REAL_SUBST_RE = re.compile(r"(.*\d)p(\d.*)") -_HDIM_TEMPNAME = '_CCPP_HORIZ_DIM' - -############################################################################### -# Supported horizontal dimensions (should be defined in CCPP_STANDARD_VARS) -CCPP_HORIZONTAL_DIMENSIONS = ['ccpp_constant_one:horizontal_dimension', - 'ccpp_constant_one:horizontal_loop_extent', - 'horizontal_loop_begin:horizontal_loop_end', - 'horizontal_dimension', 'horizontal_loop_extent'] - -############################################################################### -# Supported vertical dimensions (should be defined in CCPP_STANDARD_VARS) -CCPP_VERTICAL_DIMENSIONS = ['ccpp_constant_one:vertical_layer_dimension', - 'ccpp_constant_one:vertical_interface_dimension', - 'vertical_layer_dimension', - 'vertical_interface_dimension', - 'vertical_layer_index', 'vertical_interface_index'] - -############################################################################### -# Substituions for run time dimension control -CCPP_LOOP_DIM_SUBSTS = {'ccpp_constant_one:horizontal_dimension' : - 'horizontal_loop_begin:horizontal_loop_end', - 'ccpp_constant_one:vertical_layer_dimension' : - 'vertical_layer_index', - 'ccpp_constant_one:vertical_interface_dimension' : - 'vertical_interface_index'} - -######################################################################## -def is_horizontal_dimension(dim_name): -######################################################################## - """Return True if it is a recognized horizontal - dimension or index, otherwise, return False - >>> is_horizontal_dimension('horizontal_loop_extent') - True - >>> is_horizontal_dimension('ccpp_constant_one:horizontal_loop_extent') - True - >>> is_horizontal_dimension('ccpp_constant_one:horizontal_dimension') - True - >>> is_horizontal_dimension('horizontal_loop_begin:horizontal_loop_end') - True - >>> is_horizontal_dimension('horizontal_loop_begin:horizontal_loop_extent') - False - >>> is_horizontal_dimension('ccpp_constant_one') - False - """ - return dim_name in CCPP_HORIZONTAL_DIMENSIONS - -######################################################################## -def is_vertical_dimension(dim_name): -######################################################################## - """Return True if it is a recognized vertical - dimension or index, otherwise, return False - >>> is_vertical_dimension('ccpp_constant_one:vertical_layer_dimension') - True - >>> is_vertical_dimension('ccpp_constant_one:vertical_interface_dimension') - True - >>> is_vertical_dimension('vertical_layer_index') - True - >>> is_vertical_dimension('vertical_interface_index') - True - >>> is_vertical_dimension('ccpp_constant_one:vertical_layer_index') - False - >>> is_vertical_dimension('ccpp_constant_one:vertical_interface_index') - False - >>> is_vertical_dimension('horizontal_loop_extent') - False - """ - return dim_name in CCPP_VERTICAL_DIMENSIONS - -######################################################################## -def find_horizontal_dimension(dims): -######################################################################## - """Return the horizontal dimension string and location in - or (None, -1). - Return form is (horizontal_dimension, index) where index is the - location of horizontal_dimension in """ - var_hdim = None - hindex = -1 - for index, dimname in enumerate(dims): - if is_horizontal_dimension(dimname): - var_hdim = dimname - hindex = index - break - # end if - # end for - return (var_hdim, hindex) - -######################################################################## -def find_vertical_dimension(dims): -######################################################################## - """Return the vertical dimension string and location in - or (None, -1). - Return form is (vertical_dimension, index) where index is the - location of vertical_dimension in """ - var_vdim = None - vindex = -1 - for index, dimname in enumerate(dims): - if is_vertical_dimension(dimname): - var_vdim = dimname - vindex = index - break - # end if - # end for - return (var_vdim, vindex) - -######################################################################## -def local_name_to_diag_name(prop_dict, context=None): -######################################################################## - """ - Translate a local_name to its default diagnostic name. - Currently, this is just equal to the local name. If no local name - exists in the property dictionary, a truncation of the standard - name is used. (256 characters = max length of NetCDF variable name) - >>> local_name_to_diag_name({'local_name':'foo', 'standard_name':'cloud_optical_depth'}) - 'foo' - >>> local_name_to_diag_name({'standard_name':'cloud_optical_depth_layers_from_0p55mu_to_0p99mu'}) - 'cloud_optical_depth_layers_from_0p55mu_to_0p99mu' - >>> local_name_to_diag_name({'units':'km'}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name or local name to convert to diagnostic name - >>> local_name_to_diag_name({'local_name':'', 'standard_name':'cloud_optical_depth'}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name or local name to convert to diagnostic name - >>> local_name_to_diag_name({'standard_name':''}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name or local name to convert to diagnostic name - """ - diag_name = None - if 'local_name' in prop_dict: - locname = prop_dict['local_name'] - if locname: - diag_name = prop_dict['local_name'] - # end if - elif 'standard_name' in prop_dict: - stdname = prop_dict['standard_name'] - if stdname: - maxlen = 256 - diag_name = stdname[:maxlen] - # end if - # end if - - if not diag_name: - emsg = 'No standard name or local name to convert to diagnostic name' - raise CCPPError(emsg) - # end if - - return diag_name - -######################################################################## -def standard_name_to_long_name(prop_dict, context=None): -######################################################################## - """Translate a standard_name to its default long_name - >>> standard_name_to_long_name({'standard_name':'cloud_optical_depth_layers_from_0p55mu_to_0p99mu'}) - 'Cloud optical depth layers from 0.55mu to 0.99mu' - >>> standard_name_to_long_name({'local_name':'foo'}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name to convert foo to long name - >>> standard_name_to_long_name({}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name to convert to long name - >>> standard_name_to_long_name({'local_name':'foo'}, context=ParseContext(linenum=3, filename='foo.F90')) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name to convert foo to long name, at foo.F90:4 - >>> standard_name_to_long_name({}, context=ParseContext(linenum=3, filename='foo.F90')) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No standard name to convert to long name, at foo.F90:4 - """ - # We assume that standard_name has been checked for validity - # Make the first char uppercase and replace each underscore with a space - if 'standard_name' in prop_dict: - standard_name = prop_dict['standard_name'] - if standard_name: - long_name = standard_name[0].upper() + re.sub("_", " ", - standard_name[1:]) - else: - long_name = '' - # end if - # Next, substitute a decimal point for the p in [:digit]p[:digit] - match = _REAL_SUBST_RE.match(long_name) - while match is not None: - long_name = match.group(1) + '.' + match.group(2) - match = _REAL_SUBST_RE.match(long_name) - # end while - else: - long_name = '' - if 'local_name' in prop_dict: - lname = ' {}'.format(prop_dict['local_name']) - else: - lname = '' - # end if - ctxt = context_string(context) - emsg = 'No standard name to convert{} to long name{}' - raise CCPPError(emsg.format(lname, ctxt)) - # end if - return long_name - -######################################################################## -def default_kind_val(prop_dict, context=None): -######################################################################## - """Choose a default kind based on a variable's type - >>> default_kind_val({'type':'REAL'}) - 'kind_phys' - >>> default_kind_val({'type':'complex'}) - 'kind_phys' - >>> default_kind_val({'type':'double precision'}) - 'kind_phys' - >>> default_kind_val({'type':'integer'}) - '' - >>> default_kind_val({'type':'character'}) - '' - >>> default_kind_val({'type':'logical'}) - '' - >>> default_kind_val({'local_name':'foo'}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No type to find default kind for foo - >>> default_kind_val({}) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No type to find default kind - >>> default_kind_val({'local_name':'foo'}, context=ParseContext(linenum=3, filename='foo.F90')) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No type to find default kind for foo, at foo.F90:4 - >>> default_kind_val({}, context=ParseContext(linenum=3, filename='foo.F90')) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: No type to find default kind, at foo.F90:4 - """ - if 'type' in prop_dict: - vtype = prop_dict['type'].lower() - if vtype == 'real': - kind = 'kind_phys' - elif vtype == 'complex': - kind = 'kind_phys' - elif FORTRAN_DP_RE.match(vtype) is not None: - kind = 'kind_phys' - else: - kind = '' - # end if - else: - kind = '' - if 'local_name' in prop_dict: - lname = ' {}'.format(prop_dict['local_name']) - errmsg = 'No type to find default kind for {ln}{ct}' - else: - lname = '' - errmsg = 'No type to find default kind{ct}' - # end if - ctxt = context_string(context) - raise CCPPError(errmsg.format(ln=lname, ct=ctxt)) - # end if - return kind - -######################################################################## - -class DimTransform: - """Class to represent a transformation between two variables with - compatible dimensions. - Compatible differences include permutations, sub-selection of the - horizontal dimension, and the ordering of the vertical dimension. - - The "forward" transformation transforms "var1" into "var2" - (i.e., var2 = forward_transform(var1)). - The "reverse" transformation transforms "var2" into "var1" - (i.e., var1 = reverse_transform(var2)). - """ - - def __init__(self, forward_permutation, reverse_permutation, - forward_hdim, forward_hdim_index, forward_vdim_index, - reverse_hdim, reverse_hdim_index, reverse_vdim_index): - """Initialize a dimension transform object. - : A tuple of integers with the location of the - "var1" index for each "var2" index. That is, the first index - for "var2" on the LHS of the forward transform is - [0]. - : A tuple of integers with the location of the - "var2" index for each "var1" index. That is, the first index - for "var1" on the LHS of the forward transform is - [0]. - : The name of the horizontal dimension for "var1". - This is used to determine if an offset needs to be added to - the forward and reverse transforms. - : This is the position of the horizontal dimension - for "var1". For instance, zero means that the horizontal axis is - the fastest varying. - : This is the position of the vertical dimension - for "var1". For instance, zero means that the vertical axis is - the fastest varying. - : The name of the horizontal dimension for "var2". - This is used to determine if an offset needs to be added to - the forward and reverse transforms. - : This is the position of the horizontal dimension - for "var2". For instance, zero means that the horizontal axis is - the fastest varying. - : This is the position of the vertical dimension - for "var2". For instance, zero means that the vertical axis is - the fastest varying. - - # Test that bad inputs are trapped: - >>> DimTransform((0, 1, 2), (2, 1), 'horizontal_dimension', 0, 1, \ - 'horizontal_dimension', \ - 1, 0) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseInternalError: Permutation mismatch, '(0, 1, 2)' and '(2, 1)' - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 3, 2, \ - 'horizontal_dimension', \ - 4, 3) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseInternalError: forward_hdim_index (3) out of range [0, 2] - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 0, 4, \ - 'horizontal_dimension', \ - 4, 3) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseInternalError: forward_vdim_index (4) out of range [0, 2] - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 0, 2, \ - 'horizontal_dimension', \ - 4, 3) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseInternalError: reverse_hdim_index (4) out of range [0, 2] - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 1, 2, \ - 'horizontal_dimension', \ - 0, 3) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseInternalError: reverse_vdim_index (3) out of range [0, 2] - """ - # Store inputs - if len(forward_permutation) != len(reverse_permutation): - emsg = "Permutation mismatch, '{}' and '{}'" - raise ParseInternalError(emsg.format(forward_permutation, - reverse_permutation)) - # end if - self.__forward_perm = forward_permutation - self.__reverse_perm = reverse_permutation - if ((forward_hdim_index < 0) or - (forward_hdim_index >= len(forward_permutation))): - emsg = "forward_hdim_index ({}) out of range [0, {}]" - raise ParseInternalError(emsg.format(forward_hdim_index, - len(forward_permutation)-1)) - # end if - self.__forward_hdim_index = forward_hdim_index - # We cannot test for negative forward_vdim_index because there may - # not be a vertical dimension - if forward_vdim_index >= len(forward_permutation): - emsg = "forward_vdim_index ({}) out of range [0, {}]" - raise ParseInternalError(emsg.format(forward_vdim_index, - len(forward_permutation)-1)) - # end if - self.__forward_vdim_index = forward_vdim_index - if ((reverse_hdim_index < 0) or - (reverse_hdim_index >= len(reverse_permutation))): - emsg = "reverse_hdim_index ({}) out of range [0, {}]" - raise ParseInternalError(emsg.format(reverse_hdim_index, - len(reverse_permutation)-1)) - # end if - self.__reverse_hdim_index = reverse_hdim_index - # We cannot test for negative reverse_vdim_index because there may - # not be a vertical dimension - if reverse_vdim_index >= len(reverse_permutation): - emsg = "reverse_vdim_index ({}) out of range [0, {}]" - raise ParseInternalError(emsg.format(reverse_vdim_index, - len(reverse_permutation)-1)) - # end if - self.__reverse_vdim_index = reverse_vdim_index - # Categorize horizontal dimensions - # v_hloop is True if "var" has extent "horizontal_loop_extent". - # The loop for these variables begins at one while variables with - # extent, "horizontal_dimension" begin at "horizontal_loop_begin" - # during the run phase. - self.__v1_hloop = self.__is_horizontal_loop_dimension(forward_hdim) - if ((not self.__v1_hloop) and - (not ("horizontal_dimension" in forward_hdim))): - emsg = "Uncategorized forward horizontal dimension, '{}'" - raise ParseInternalError(emsg.format(forward_hdim)) - # end if - self.__v2_hloop = self.__is_horizontal_loop_dimension(reverse_hdim) - if ((not self.__v2_hloop) and - (not ("horizontal_dimension" in reverse_hdim))): - emsg = "Uncategorized reverse horizontal dimension, '{}'" - raise ParseInternalError(emsg.format(reverse_hdim)) - # end if - - def forward_transform(self, var2_lname, indices, - adjust_hdim=None, flip_vdim=None): - """Compute and return the LHS of the forward transform from "var1" to - "var2". - is the local name of "var2". - is a tuple of the loop indices for "var1" (i.e., "var1" - will show up in the RHS of the transform as "var1(indices)". - If is not None, it should be a string containing the - local name of the "horizontal_loop_begin" variable. This is used to - compute the offset in the horizontal axis index between one and - "horizontal_loop_begin" (if any). This occurs when one of the - variables has extent "horizontal_loop_extent" and the other has - extent "horizontal_dimension". - If flip_vdim is not None, it should be a string containing the local - name of the vertical extent of the vertical axis for "var1" and - "var2" (i.e., "vertical_layer_dimension" or - "vertical_interface_dimension"). - - # Test forward transform with just horizontal adjustment - >>> DimTransform((0, 1), (0, 1), 'horizontal_dimension', 0, 1, \ - 'horizontal_loop_extent', \ - 0, 1).forward_transform("foo_lhs", ("hind", "vind"), \ - adjust_hdim="col_start") - 'foo_lhs(hind-col_start+1,vind)' - >>> DimTransform((0, 1), (0, 1), 'horizontal_loop_extent', 0, 1, \ - 'horizontal_dimension', \ - 0, 1).forward_transform("foo_lhs", ("hind", "vind"), \ - adjust_hdim="col_start") - 'foo_lhs(hind+col_start-1,vind)' - - # Test flipping vertical dimension - >>> DimTransform((0, 1), (0, 1), 'horizontal_dimension', 0, 1, \ - 'horizontal_dimension', \ - 0, 1).forward_transform("foo_lhs", ("hind", "vind"), \ - flip_vdim="pver") - 'foo_lhs(hind,pver-vind+1)' - - # Test simple permutations - >>> DimTransform((1, 0), (1, 0), 'horizontal_dimension', 0, 1, \ - 'horizontal_dimension', \ - 1, 0).forward_transform("foo_lhs", ("hind", "vind")) - 'foo_lhs(vind,hind)' - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 0, 2, \ - 'horizontal_dimension', \ - 0, 1).forward_transform("foo_lhs", \ - ("hind", "xdim", "vind")) - 'foo_lhs(vind,hind,xdim)' - """ - v2_indices = [indices[x] for x in self.__forward_perm] - if adjust_hdim is not None: - if self.__v1_hloop and (not self.__v2_hloop): - hdim = v2_indices[self.__forward_hdim_index] - adj_str = f"{hdim}+{adjust_hdim}-1" - v2_indices[self.__forward_hdim_index] = adj_str - elif self.__v2_hloop and (not self.__v1_hloop): - hdim = v2_indices[self.__forward_hdim_index] - adj_str = f"{hdim}-{adjust_hdim}+1" - v2_indices[self.__forward_hdim_index] = adj_str - # end if - # end if - if flip_vdim is not None: - vdim = v2_indices[self.__forward_vdim_index] - adj_str = f"{flip_vdim}-{vdim}+1" - v2_indices[self.__forward_vdim_index] = adj_str - # end if - return f"{var2_lname}({','.join(v2_indices)})" - - def reverse_transform(self, var1_lname, indices, - adjust_hdim=None, flip_vdim=None): - """Compute and return the LHS of the forward transform from "var2" to - "var1". - is the local name of "var1". - is a tuple of the loop indices for "var2" (i.e., "var2" - will show up in the RHS of the transform as "var2(indices)". - If is not None, it should be a string containing the - local name of the "horizontal_loop_begin" variable. This is used to - compute the offset in the horizontal axis index between one and - "horizontal_loop_begin" (if any). This occurs when one of the - variables has extent "horizontal_loop_extent" and the other has - extent "horizontal_dimension". - If flip_vdim is not None, it should be a string containing the local - name of the vertical extent of the vertical axis for "var2" and - "var1" (i.e., "vertical_layer_dimension" or - "vertical_interface_dimension"). - - # Test reverse transform with just horizontal adjustment - >>> DimTransform((0, 1), (0, 1), 'horizontal_dimension', 0, 1, \ - 'horizontal_loop_extent', \ - 0, 1).reverse_transform("bar_lhs", ("hind", "vind"), \ - adjust_hdim="col_start") - 'bar_lhs(hind+col_start-1,vind)' - >>> DimTransform((0, 1), (0, 1), 'horizontal_loop_extent', 0, 1, \ - 'horizontal_dimension', \ - 0, 1).reverse_transform("bar_lhs", ("hind", "vind"), \ - adjust_hdim="col_start") - 'bar_lhs(hind-col_start+1,vind)' - - # Test flipping vertical dimension - >>> DimTransform((0, 1), (0, 1), 'horizontal_dimension', 0, 1, \ - 'horizontal_dimension', \ - 0, 1).reverse_transform("bar_lhs", ("hind", "vind"), \ - flip_vdim="pver") - 'bar_lhs(hind,pver-vind+1)' - - # Test simple permutations - >>> DimTransform((1, 0), (1, 0), 'horizontal_dimension', 0, 1, \ - 'horizontal_dimension', \ - 1, 0).reverse_transform("bar_lhs", ("hind", "vind")) - 'bar_lhs(vind,hind)' - >>> DimTransform((2, 0, 1), (1, 2, 0), 'horizontal_dimension', 0, 2, \ - 'horizontal_dimension', \ - 0, 1).reverse_transform("bar_lhs", \ - ("vind", "hind", "xdim")) - 'bar_lhs(hind,xdim,vind)' - """ - v1_indices = [indices[x] for x in self.__reverse_perm] - if adjust_hdim is not None: - if self.__v1_hloop and (not self.__v2_hloop): - hdim = v1_indices[self.__reverse_hdim_index] - adj_str = f"{hdim}-{adjust_hdim}+1" - v1_indices[self.__reverse_hdim_index] = adj_str - elif self.__v2_hloop and (not self.__v1_hloop): - hdim = v1_indices[self.__reverse_hdim_index] - adj_str = f"{hdim}+{adjust_hdim}-1" - v1_indices[self.__reverse_hdim_index] = adj_str - # end if - # end if - if flip_vdim is not None: - vdim = v1_indices[self.__reverse_vdim_index] - adj_str = f"{flip_vdim}-{vdim}+1" - v1_indices[self.__reverse_vdim_index] = adj_str - # end if - return f"{var1_lname}({','.join(v1_indices)})" - - @staticmethod - def __is_horizontal_loop_dimension(hdim): - """Return True if is a run-phase horizontal dimension""" - return (is_horizontal_dimension(hdim) and - ("horizontal_dimension" not in hdim)) - -######################################################################## - -class VariableProperty: - """Class to represent a single property of a metadata header entry - >>> VariableProperty('local_name', str) #doctest: +ELLIPSIS - - >>> VariableProperty('standard_name', str) #doctest: +ELLIPSIS - - >>> VariableProperty('long_name', str) #doctest: +ELLIPSIS - - >>> VariableProperty('units', str) #doctest: +ELLIPSIS - - >>> VariableProperty('dimensions', list) #doctest: +ELLIPSIS - - >>> VariableProperty('type', str) #doctest: +ELLIPSIS - - >>> VariableProperty('kind', str) #doctest: +ELLIPSIS - - >>> VariableProperty('state_variable', str, valid_values_in=['True', 'False', '.true.', '.false.' ], optional_in=True, default_in=False) #doctest: +ELLIPSIS - - >>> VariableProperty('intent', str, valid_values_in=['in', 'out', 'inout']) #doctest: +ELLIPSIS - - >>> VariableProperty('optional', str, valid_values_in=['True', 'False', '.true.', '.false.' ], optional_in=True, default_in=False) #doctest: +ELLIPSIS - - >>> VariableProperty('local_name', str).name - 'local_name' - >>> VariableProperty('standard_name', str).ptype == str - True - >>> VariableProperty('units', str).is_match('units') - True - >>> VariableProperty('units', str).is_match('UNITS') - True - >>> VariableProperty('units', str).is_match('type') - False - >>> VariableProperty('value', int, valid_values_in=[1, 2 ]).valid_value('2') - 2 - >>> VariableProperty('value', int, valid_values_in=[1, 2 ]).valid_value('3') - - >>> VariableProperty('value', int, valid_values_in=[1, 2 ]).valid_value('3', error=True) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: Invalid value variable property, '3' - >>> VariableProperty('units', str, check_fn_in=check_units).valid_value('m s-1') - 'm s-1' - >>> VariableProperty('units', str, check_fn_in=check_units).valid_value(' ') - - >>> VariableProperty('units', str, check_fn_in=check_units).valid_value(' ', error=True) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: ' ' is not a valid unit - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('()') - [] - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('(x)') - ['x'] - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('x') - - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('(x:y)') - ['x:y'] - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('(w:x,y:z)') - ['w:x', 'y:z'] - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value(['size(foo)']) - ['size(foo)'] - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('(w:x,x:y:z:q)', error=True) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: 'x:y:z:q' is an invalid dimension range - >>> VariableProperty('dimensions', list, check_fn_in=check_dimensions).valid_value('(x:3y)', error=True) #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.CCPPError: '3y' is not a valid Fortran identifier - >>> VariableProperty('local_name', str, check_fn_in=check_local_name).valid_value('foo') - 'foo' - >>> VariableProperty('local_name', str, check_fn_in=check_local_name).valid_value('foo(bar)') - 'foo(bar)' - >>> VariableProperty('local_name', str, check_fn_in=check_local_name).valid_value('q(:,:,index_of_water_vapor_specific_humidity)') - 'q(:,:,index_of_water_vapor_specific_humidity)' - >>> VariableProperty('molar_mass', float, check_fn_in=check_molar_mass).valid_value('12.1') - 12.1 - """ - - __true_vals = ['t', 'true', '.true.'] - __false_vals = ['f', 'false', '.false.'] - - def __init__(self, name_in, type_in, valid_values_in=None, - optional_in=False, default_in=None, default_fn_in=None, - check_fn_in=None, mult_entry_ok=False): - """Conduct sanity checks and initialize this variable property.""" - self._name = name_in - self._type = type_in - if self._type not in [bool, int, list, str, float]: - emsg = "{} has invalid VariableProperty type, '{}'" - raise CCPPError(emsg.format(name_in, type_in)) - # end if - self._valid_values = valid_values_in - self._optional = optional_in - self._default = None - self._default_fn = None - if self.optional: - if (default_in is None) and (default_fn_in is None): - emsg = 'default_in or default_fn_in is a required property for {} because it is optional' - raise CCPPError(emsg.format(name_in)) - if (default_in is not None) and (default_fn_in is not None): - emsg = 'default_in and default_fn_in cannot both be provided' - raise CCPPError(emsg) - self._default = default_in - self._default_fn = default_fn_in - elif default_in is not None: - emsg = 'default_in is not a valid property for {} because it is not optional' - raise CCPPError(emsg.format(name_in)) - elif default_in is not None: - emsg = 'default_fn_in is not a valid property for {} because it is not optional' - raise CCPPError(emsg.format(name_in)) - self._check_fn = check_fn_in - self._add_multiple_ok = mult_entry_ok - - @property - def name(self): - """Return the name of the property""" - return self._name - - @property - def ptype(self): - """Return the type of the property""" - return self._type - - @property - def has_default_func(self): - """Return True iff this variable property has a default function""" - return self._default_fn is not None - - def get_default_val(self, prop_dict, context=None): - """Return this variable property's default value or raise an - exception if there is no default value or default value function.""" - if self.has_default_func: - return self._default_fn(prop_dict, context) - # end if - if self._default is not None: - return self._default - # end if - ctxt = context_string(context) - emsg = 'No default for variable property {}{}' - raise CCPPError(emsg.format(self.name, ctxt)) - - - @property - def optional(self): - """Return True iff this variable property is optional""" - return self._optional - - @property - def add_multiple(self): - """Return True iff multiple entries of this property should be - accumulated. If False, it should either be an error or new - instances should replace the old, however, this functionality - must be implemented by the calling routine (e.g., Var)""" - return self._add_multiple_ok - - def is_match(self, test_name): - """Return True iff is the name of this property""" - return self.name.lower() == test_name.lower() - - def valid_value(self, test_value, prop_dict=None, error=False): - """Return a valid version of if it is valid. - If is not valid, return None or raise an exception, - depending on the value of . - If is not None, it may be used in value validation. - """ - valid_val = None - if self.ptype is int: - try: - tval = int(test_value) - if self._valid_values is not None: - if tval in self._valid_values: - valid_val = tval - else: - valid_val = None # i.e. pass - else: - valid_val = tval - except CCPPError: - valid_val = None # Redundant but more expressive than pass - elif self.ptype is float: - try: - tval = float(test_value) - if self._valid_values is not None: - if tval in self._valid_values: - valid_val = tval - else: - valid_val = None # i.e. pass - # end if - else: - valid_val = tval - # end if - except CCPPError: - valid_val = None - # end try - elif self.ptype is list: - if isinstance(test_value, str): - tval = fortran_list_match(test_value) - if tval and (len(tval) == 1) and (not tval[0]): - # Scalar - tval = list() - # end if - else: - tval = test_value - # end if - if isinstance(tval, list): - valid_val = tval - elif isinstance(tval, tuple): - valid_val = list(tval) - else: - valid_val = None - # end if - if (valid_val is not None) and (self._valid_values is not None): - # Special case for lists, _valid_values applies to elements - for item in valid_val: - if item not in self._valid_values: - valid_val = None - break - # end if - # end for - else: - pass - elif self.ptype is bool: - if isinstance(test_value, str): - if test_value.lower() in VariableProperty.__true_vals + VariableProperty.__false_vals: - valid_val = test_value.lower() in VariableProperty.__true_vals - else: - valid_val = None # i.e., pass - # end if - else: - valid_val = not not test_value # pylint: disable=unneeded-not - elif self.ptype is str: - if isinstance(test_value, str): - if self._valid_values is not None: - if test_value in self._valid_values: - valid_val = test_value - else: - valid_val = None # i.e., pass - else: - valid_val = test_value - # end if - # end if - # end if - # Call a check function? - if valid_val and (self._check_fn is not None): - valid_val = self._check_fn(valid_val, prop_dict, error) - elif (valid_val is None) and error: - emsg = "Invalid {} variable property, '{}'" - raise CCPPError(emsg.format(self.name, test_value)) - # end if - return valid_val - -############################################################################## - -class VarCompatObj: - """Class to compare two Var objects and then answer questions about - the compatibility of the two variables. - There are three levels of compatibility. - * Compatible is when two variables match in all properties so that one - can be passed to another with no transformation. - * Comformable is when two variables have the same information but may - need some transformation between them. Examples are differences in - dimension ordering, units, or kind. - * Not Compatible is when information from one variable cannot be passed - to the other. - - Note that character(len=*) is considered equivalent to - character(len=) - - # Test that we can create a standard VarCompatObj object - >>> from parse_tools import init_log, set_log_to_null - >>> _DOCTEST_LOGGING = init_log('var_props') - >>> set_log_to_null(_DOCTEST_LOGGING) - >>> _DOCTEST_RUNENV = CCPPFrameworkEnv(_DOCTEST_LOGGING, \ - ndict={'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}, \ - kind_types=["kind_phys=REAL64", \ - "kind_dyn=REAL32", \ - "kind_host=REAL64"]) - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", [], "var1_lname", False,\ - "var_stdname", "real", "kind_phys", "m", [], "var2_lname", False,\ - _DOCTEST_RUNENV) #doctest: +ELLIPSIS - - - # Test that a 2-D var with no horizontal transform works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV) #doctest: +ELLIPSIS - - - # Test that a 2-D var with a horizontal transform works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "m", ['horizontal_loop_extent'], "var2_lname", False, \ - _DOCTEST_RUNENV) #doctest: +ELLIPSIS - - - # Test that a 1-D var with no vertical transform works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['vertical_layer_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "m", ['vertical_layer_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV) #doctest: +ELLIPSIS - - - # Test that a 1-D var with vertical flipping works and that it - # produces the correct reverse transformation - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['vertical_layer_dimension'], "var1_lname", False,\ - "var_stdname", "real", "kind_phys", "m", ['vertical_layer_dimension'], "var2_lname", True, \ - _DOCTEST_RUNENV).reverse_transform("var1_lname", "var2_lname", ('k',), ('nk-k+1',)) - 'var1_lname(nk-k+1) = var2_lname(k)' - - # Test that unit conversions with a scalar var works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "Pa", [], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "hPa", [], "var2_lname", False, \ - _DOCTEST_RUNENV).forward_transform("var1_lname", "var2_lname", [], []) #doctest: +ELLIPSIS - 'var1_lname = 1.0E-2_kind_phys*var2_lname' - - # Test that unit conversions with a scalar var works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "Pa", [], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "hPa", [], "var2_lname", False, \ - _DOCTEST_RUNENV).reverse_transform("var1_lname", "var2_lname", [], []) #doctest: +ELLIPSIS - 'var1_lname = 1.0E+2_kind_phys*var2_lname' - - # Test that a 2-D var with unit conversion m->km works - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "km", ['horizontal_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV) #doctest: +ELLIPSIS - - - # Test that a 2-D var with unit conversion m->km works and that it - # produces the correct forward transformation - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "km", ['horizontal_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV).forward_transform("var1_lname", "var2_lname", 'i', 'i') - 'var1_lname(i) = 1.0E-3_kind_phys*var2_lname(i)' - - # Test that a 3-D var with unit conversion m->km and vertical flipping - # works and that it produces the correct reverse transformation - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m", ['horizontal_dimension', 'vertical_layer_dimension'], "var1_lname", False,\ - "var_stdname", "real", "kind_phys", "km",['horizontal_dimension', 'vertical_layer_dimension'], "var2_lname", True, \ - _DOCTEST_RUNENV).reverse_transform("var1_lname", "var2_lname", ('i','k'), ('i','nk-k+1')) - 'var1_lname(i,nk-k+1) = 1.0E+3_kind_phys*var2_lname(i,k)' - - # Test that a 2-D var with equivalent units works and that it - # skips any unit transformations - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m2 s-2", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "J kg-1", ['horizontal_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV).forward_transform("var1_lname", "var2_lname", 'i', 'i') - 'var1_lname(i) = var2_lname(i)' - - # Test that a 2-D var with identical units works and that it - # skips any unit transformations - >>> VarCompatObj("var_stdname", "real", "kind_phys", "m2 s-2", ['horizontal_dimension'], "var1_lname", False, \ - "var_stdname", "real", "kind_phys", "m+2 s-2", ['horizontal_dimension'], "var2_lname", False, \ - _DOCTEST_RUNENV).forward_transform("var1_lname", "var2_lname", 'i', 'i') - 'var1_lname(i) = var2_lname(i)' - """ - - def __init__(self, var1_stdname, var1_type, var1_kind, var1_units, - var1_dims, var1_lname, var1_top, var2_stdname, var2_type, var2_kind, - var2_units, var2_dims, var2_lname, var2_top, run_env, v1_context=None, - v2_context=None, is_tend=False): - """Initialize this object with information on the equivalence and/or - conformability of two variables. - variable 1 is described by , , , - , , , , and . - variable 2 is described by , , , - , , , , and . - is the CCPPFrameworkEnv object used here to verify kind - equivalence or to produce kind transformations. - is a flag where, if true, we are validating a tendency variable (var1) - against it's equivalent state variable (var2) - """ - self.__equiv = True # No transformation required - self.__compat = True # Callable with transformation - self.__stdname = var1_stdname - self.__v1_context = v1_context - self.__v2_context = v2_context - self.__v1_kind = var1_kind - self.__v2_kind = var2_kind - self.v1_units = var1_units - self.v2_units = var2_units - self.v1_stdname = var1_stdname - self.v2_stdname = var2_stdname - # Default (null) transform information - self.__dim_transforms = None - self.__kind_transforms = None - self.__unit_transforms = None - self.has_vert_transforms = False - incompat_reason = list() - # First, check for fatal incompatibilities - # If it's a tendency variable, the standard name should be of the - # form "tendency_of_var2_stdname" - if is_tend and not var1_stdname.startswith('tendency_of'): - self.__equiv = False - self.__compat = False - incompat_reason.append('not a tendency variable') - if not is_tend and var1_stdname != var2_stdname: - self.__equiv = False - self.__compat = False - incompat_reason.append("standard names") - # end if - if var1_type != var2_type: - self.__equiv = False - self.__compat = False - incompat_reason.append("types") - # end if - # Check kind argument - if self.__compat: - if var1_type == 'character': - # First, make sure we have supported character 'kind' args: - v1_kind = self.char_kind_check(var1_kind) - if not v1_kind: - ctx = context_string(v1_context) - emsg = "Unsupported character kind/len argument, '{}', " - emsg += "in {}{}" - incompat_reason.append(emsg.format(var1_kind, - var1_lname, ctx)) - # end if - self.__v1_kind = None - v2_kind = self.char_kind_check(var2_kind) - if not v2_kind: - ctx = context_string(v2_context) - emsg = "Unsupported character kind/len argument, '{}', " - emsg += "in {}{}" - incompat_reason.append(emsg.format(var2_kind, - var2_lname, ctx)) - # end if - self.__v2_kind = None - # Character types have to 'match' or the variables are - # incompatible - kind_eq = ((v1_kind and v2_kind) and - ((v1_kind == v2_kind) or - (((v1_kind == 'len=*') and - (v2_kind.startswith('len='))) or - (v1_kind.startswith('len=') and - (v2_kind == 'len=*'))))) - if not kind_eq: - self.__equiv = False - self.__compat = False - incompat_reason.append("character len arguments") - # end if - else: - if var1_kind != var2_kind: - self.__kind_transforms = self._get_kind_convstrs(var1_kind, - var2_kind, - run_env) - self.__equiv = self.__kind_transforms is None - # end if - # end if - # end if - if self.__compat: - # Only "none" units are case-insensitive - if var1_units.lower() == 'none': - var1_units = 'none' - # end if - if var2_units.lower() == 'none': - var2_units = 'none' - # end if - # Check units argument - if is_tend: - # A tendency variable's units should be " s-1" - tendency_split_units = var1_units.split('s-1')[0].strip() - if tendency_split_units != var2_units: - # We don't currently support unit conversions for tendency variables, - # but we can check if the units are identical or equivalent - unit_transforms = self._get_unit_convstrs(tendency_split_units, - var2_units) - if not unit_transforms == (None, None): - emsg = f"\nMismatch tendency variable units '{var1_units}'" - emsg += f" for variable '{var1_stdname}'." - emsg += " No variable transforms supported for tendencies." - emsg += f" Tendency units should be '{var2_units} s-1' to match state variable." - self.__equiv = False - self.__compat = False - incompat_reason.append(emsg) - # end if - elif var1_units != var2_units: - # Try to find a set of unit conversions - unit_transforms = self._get_unit_convstrs(var1_units, - var2_units) - # Handle equivalent or identical units = (None, None) - if not unit_transforms == (None, None): - self.__equiv = False - self.__unit_transforms = unit_transforms - # end if - # end if - if self.__compat: - # Check for vertical array flipping (do later) - if var1_top != var2_top: - self.__compat = True - self.has_vert_transforms = True - # end if - # end if - if self.__compat: - # Check dimensions - if var1_dims or var2_dims: - _, vdim_ind = find_vertical_dimension(var1_dims) - if (var1_dims != var2_dims): - self.__dim_transforms = self._get_dim_transforms(var1_dims, - var2_dims) - self.__compat = self.__dim_transforms is not None - # end if - # end if - if not self.__compat: - incompat_reason.append('dimensions') - # end if - # end if - self.__incompat_reason = " and ".join([x for x in incompat_reason if x]) - - def forward_transform(self, lvar_lname, rvar_lname, rvar_indices, lvar_indices, - adjust_hdim=None, flip_vdim=None): - """Compute and return the the forward transform from "var1" to "var2". - is the local name of "var2". - is the local name of "var1". - is a tuple of the loop indices for "var1" (i.e., "var1" - will show up in the RHS of the transform as "var1(rvar_indices)". - is a tuple of the loop indices for "var2" (i.e., "var2" - will show up in the LHS of the transform as "var2(lvar_indices)". - If is not None, it should be a string containing the - local name of the "horizontal_loop_begin" variable. This is used to - compute the offset in the horizontal axis index between one and - "horizontal_loop_begin" (if any). This occurs when one of the - variables has extent "horizontal_loop_extent" and the other has - extent "horizontal_dimension". - If flip_vdim is not None, it should be a string containing the local - name of the vertical extent of the vertical axis for "var1" and - "var2" (i.e., "vertical_layer_dimension" or - "vertical_interface_dimension"). - """ - # Dimension transform (Indices handled externally) - if len(rvar_indices) == 0: - rhs_term = f"{rvar_lname}" - lhs_term = f"{lvar_lname}" - else: - rhs_term = f"{rvar_lname}({','.join(rvar_indices)})" - lhs_term = f"{lvar_lname}({','.join(lvar_indices)})" - # end if - - if self.has_kind_transforms: - kind = self.__kind_transforms[1] - rhs_term = f"real({rhs_term}, {kind})" - else: - kind = '' - # end if - if self.has_unit_transforms: - if kind: - kind = "_" + kind - elif self.__v2_kind: - kind = "_" + self.__v2_kind - # end if - rhs_term = self.__unit_transforms[0].format(var=rhs_term, kind=kind) - # end if - return f"{lhs_term} = {rhs_term}" - - def reverse_transform(self, lvar_lname, rvar_lname, rvar_indices, lvar_indices, - adjust_hdim=None, flip_vdim=None): - """Compute and return the the reverse transform from "var2" to "var1". - is the local name of "var1". - is the local name of "var2". - is a tuple of the loop indices for "var1" (i.e., "var1" - will show up in the RHS of the transform as "var1(rvar_indices)". - is a tuple of the loop indices for "var2" (i.e., "var2" - will show up in the LHS of the transform as "var2(lvar_indices)". - If is not None, it should be a string containing the - local name of the "horizontal_loop_begin" variable. This is used to - compute the offset in the horizontal axis index between one and - "horizontal_loop_begin" (if any). This occurs when one of the - variables has extent "horizontal_loop_extent" and the other has - extent "horizontal_dimension". - If flip_vdim is not None, it should be a string containing the local - name of the vertical extent of the vertical axis for "var1" and - "var2" (i.e., "vertical_layer_dimension" or - "vertical_interface_dimension"). - """ - # Dimension transforms (Indices handled externally) - if len(rvar_indices) == 0: - rhs_term = f"{rvar_lname}" - lhs_term = f"{lvar_lname}" - else: - lhs_term = f"{lvar_lname}({','.join(lvar_indices)})" - rhs_term = f"{rvar_lname}({','.join(rvar_indices)})" - # end if - - if self.has_kind_transforms: - kind = self.__kind_transforms[0] - rhs_term = f"real({rhs_term}, {kind})" - else: - kind = '' - # end if - if self.has_unit_transforms: - if kind: - kind = "_" + kind - elif self.__v1_kind: - kind = "_" + self.__v1_kind - # end if - rhs_term = self.__unit_transforms[1].format(var=rhs_term, kind=kind) - # end if - return f"{lhs_term} = {rhs_term}" - - def _get_kind_convstrs(self, var1_kind, var2_kind, run_env): - """Attempt to determine if no transformation is required (i.e., if - and will be the same at runtime. If so, - return None. - If a conversion is required, return a tuple with the two kinds, - i.e., (var1_kind, var2_kind). - - # Initial setup - >>> from parse_tools import init_log, set_log_to_null - >>> _DOCTEST_LOGGING = init_log('var_props') - >>> set_log_to_null(_DOCTEST_LOGGING) - >>> _DOCTEST_RUNENV = CCPPFrameworkEnv(_DOCTEST_LOGGING, \ - ndict={'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}, \ - kind_types=["kind_phys=REAL64", \ - "kind_dyn=REAL32", \ - "kind_host=REAL64"]) - >>> _DOCTEST_CONTEXT1 = ParseContext(linenum=3, filename='foo.F90') - >>> _DOCTEST_CONTEXT2 = ParseContext(linenum=5, filename='bar.F90') - >>> _DOCTEST_VCOMPAT = VarCompatObj("var_stdname", "real", "kind_phys", \ - "m", [], "var1_lname", False, "var_stdname", \ - "real", "kind_phys", "m", [], \ - "var2_lname", False, _DOCTEST_RUNENV, \ - v1_context=_DOCTEST_CONTEXT1, \ - v2_context=_DOCTEST_CONTEXT2) - - # Try some kind conversions - >>> _DOCTEST_VCOMPAT._get_kind_convstrs('kind_phys', 'kind_dyn', \ - _DOCTEST_RUNENV) - ('kind_phys', 'kind_dyn') - >>> _DOCTEST_VCOMPAT._get_kind_convstrs('kind_phys', 'REAL32', \ - _DOCTEST_RUNENV) - ('kind_phys', 'REAL32') - - # Try some non-conversions - >>> _DOCTEST_VCOMPAT._get_kind_convstrs('kind_phys', 'kind_host', \ - _DOCTEST_RUNENV) - - >>> _DOCTEST_VCOMPAT._get_kind_convstrs('REAL64', 'kind_host', \ - _DOCTEST_RUNENV) - - """ - kind1 = run_env.kind_spec(var1_kind) - if kind1 is None: - kind1 = var1_kind - # end if - kind2 = run_env.kind_spec(var2_kind) - if kind2 is None: - kind2 = var2_kind - # end if - if kind1 != kind2: - return (var1_kind, var2_kind) - # end if - return None - - def _get_unit_convstrs(self, var1_units, var2_units): - """Attempt to retrieve the forward and reverse unit transformations - for transforming a variable in to / from a variable in - . Return (None, None) if units are equivalent or identical - after parsing (this can happen when comparing m2 and m+2). - - # Initial setup - >>> from parse_tools import init_log, set_log_to_null - >>> _DOCTEST_LOGGING = init_log('var_props') - >>> set_log_to_null(_DOCTEST_LOGGING) - >>> _DOCTEST_RUNENV = CCPPFrameworkEnv(_DOCTEST_LOGGING, \ - ndict={'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}, \ - kind_types=["kind_phys=REAL64", \ - "kind_dyn=REAL32", \ - "kind_host=REAL64"]) - >>> _DOCTEST_CONTEXT1 = ParseContext(linenum=3, filename='foo.F90') - >>> _DOCTEST_CONTEXT2 = ParseContext(linenum=5, filename='bar.F90') - >>> _DOCTEST_VCOMPAT = VarCompatObj("var_stdname", "real", "kind_phys", \ - "m", [], "var1_lname", False, "var_stdname", \ - "real", "kind_phys", "m", [], \ - "var2_lname", False, _DOCTEST_RUNENV, \ - v1_context=_DOCTEST_CONTEXT1, \ - v2_context=_DOCTEST_CONTEXT2) - - # Try some working unit transforms - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('m', 'mm') - ('1.0E+3{kind}*{var}', '1.0E-3{kind}*{var}') - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('kg kg-1', 'g kg-1') - ('1.0E+3{kind}*{var}', '1.0E-3{kind}*{var}') - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('C', 'K') - ('{var}+273.15{kind}', '{var}-273.15{kind}') - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('V A', 'W') - (None, None) - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('N m-2', 'Pa') - (None, None) - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('m2 s-2', 'J kg-1') - (None, None) - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('m+2 s-2', 'J kg-1') - (None, None) - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('m+2 s-2', 'm2 s-2') - (None, None) - - # Try an invalid conversion - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('1', 'none') #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseSyntaxError: Unsupported unit conversion, '1' to 'none' for 'var_stdname' - - # Try an unsupported conversion - >>> _DOCTEST_VCOMPAT._get_unit_convstrs('C', 'm') #doctest: +ELLIPSIS - Traceback (most recent call last): - ... - parse_source.ParseSyntaxError: Unsupported unit conversion, 'C' to 'm' for 'var_stdname' - """ - u1_str = self.units_to_string(var1_units, self.__v1_context) - u2_str = self.units_to_string(var2_units, self.__v2_context) - # If u1_str and u2_str are identical, for example after parsing - # "m2 s-2" and "m+2 s-2", return (None, None) to signal that - # the units are in fact identical - if u1_str == u2_str: - return (None, None) - unit_conv_str = "{0}__to__{1}".format(u1_str, u2_str) - try: - forward_transform = getattr(unit_conversion, unit_conv_str)() - except AttributeError: - emsg = "Unsupported unit conversion, '{}' to '{}' for '{}'" - raise ParseSyntaxError(emsg.format(var1_units, var2_units, - self.__stdname, - context=self.__v2_context)) - # end if - unit_conv_str = "{0}__to__{1}".format(u2_str, u1_str) - try: - reverse_transform = getattr(unit_conversion, unit_conv_str)() - except AttributeError: - emsg = "Unsupported unit conversion, '{}' to '{}' for '{}'" - raise ParseSyntaxError(emsg.format(var2_units, var1_units, - self.__stdname, - context=self.__v1_context)) - # end if - # For equivalent units, return (None, None) - if forward_transform == '{var}' and reverse_transform == '{var}': - return (None, None) - else: - return (forward_transform, reverse_transform) - - def _get_dim_transforms(self, var1_dims, var2_dims): - """Attempt to find forward and reverse permutations for transforming a - variable with shape, , to / from a variable with shape, - . - Return the permutations, or None. - The forward dimension transformation is a permutation of the indices of - the first variable to the second. - The reverse dimension transformation is a permutation of the indices of - the second variable to the first. - - # Initial setup - >>> from parse_tools import init_log, set_log_to_null - >>> _DOCTEST_LOGGING = init_log('var_props') - >>> set_log_to_null(_DOCTEST_LOGGING) - >>> _DOCTEST_RUNENV = CCPPFrameworkEnv(_DOCTEST_LOGGING, \ - ndict={'host_files':'', \ - 'scheme_files':'', \ - 'suites':''}, \ - kind_types=["kind_phys=REAL64", \ - "kind_dyn=REAL32", \ - "kind_host=REAL64"]) - >>> _DOCTEST_CONTEXT1 = ParseContext(linenum=3, filename='foo.F90') - >>> _DOCTEST_CONTEXT2 = ParseContext(linenum=5, filename='bar.F90') - >>> _DOCTEST_VCOMPAT = VarCompatObj("var_stdname", "real", "kind_phys", \ - "m", [], "var1_lname", False, "var_stdname", \ - "real", "kind_phys", "m", [], \ - "var2_lname", False, _DOCTEST_RUNENV, \ - v1_context=_DOCTEST_CONTEXT1, \ - v2_context=_DOCTEST_CONTEXT2) - - # Test simple permutations - >>> _DOCTEST_VCOMPAT._get_dim_transforms(['horizontal_dimension', \ - 'vertical_layer_dimension'], \ - ['vertical_layer_dimension', \ - 'horizontal_dimension']) \ - #doctest: +ELLIPSIS - - >>> _DOCTEST_VCOMPAT._get_dim_transforms(['horizontal_dimension', \ - 'vertical_layer_dimension', \ - 'xdim'], \ - ['vertical_layer_dimension', \ - 'horizontal_dimension', \ - 'xdim']) #doctest: +ELLIPSIS - - >>> _DOCTEST_VCOMPAT._get_dim_transforms(['horizontal_dimension', \ - 'vertical_layer_dimension', \ - 'xdim'], \ - ['xdim', \ - 'horizontal_dimension', \ - 'vertical_layer_dimension']) \ - #doctest: +ELLIPSIS - - - # Test some mismatch sets - >>> _DOCTEST_VCOMPAT._get_dim_transforms(['horizontal_dimension', \ - 'vertical_layer_dimension', \ - 'xdim'], \ - ['horizontal_dimension', \ - 'vertical_layer_dimension']) \ - - >>> _DOCTEST_VCOMPAT._get_dim_transforms(['horizontal_dimension', \ - 'vertical_layer_dimension', \ - 'xdim'], \ - ['horizontal_dimension', \ - 'vertical_layer_dimension', \ - 'ydim']) - - """ - transforms = None - v1_dims = self.__regularize_dimensions(var1_dims) - v2_dims = self.__regularize_dimensions(var2_dims) - if v1_dims != v2_dims: - self.__equiv = False - # end if - # Is v2 a permutation of v1? - if len(v1_dims) == len(v2_dims): - v1_set = sorted(v1_dims) - v2_set = sorted(v2_dims) - if v1_set == v2_set: - forward_permutation = list() - reverse_permutation = [None] * len(v1_dims) - forward_hdim = '' - forward_hdim_index = -1 - forward_vdim_index = -1 - reverse_hdim = '' - reverse_hdim_index = -1 - reverse_vdim_index = -1 - for v2index, v2dim in enumerate(v2_dims): - for v1index, v1dim in enumerate(v1_dims): - if v1dim == v2dim: - # Add check for repeated indices - if v1index not in forward_permutation: - forward_permutation.append(v1index) - reverse_permutation[v1index] = v2index - if is_horizontal_dimension(var1_dims[v1index]): - forward_hdim = var1_dims[v1index] - forward_hdim_index = v1index - reverse_hdim = var2_dims[v2index] - reverse_hdim_index = v2index - elif is_vertical_dimension(var1_dims[v1index]): - forward_vdim_index = v1index - reverse_vdim_index = v2index - # end if - break - # end if - # end if (hope there is a repeated dimension) - # end for - # end for - if len(forward_permutation) != len(v1_dims): - emsg = "Bad dimension handling, '{}' and '{}'" - raise ParseInternalError(emsg.format(var1_dims, var2_dims)) - # end if - transforms = DimTransform(forward_permutation, - reverse_permutation, - forward_hdim, forward_hdim_index, - forward_vdim_index, - reverse_hdim, reverse_hdim_index, - reverse_vdim_index) - # end if - # end if - return transforms - - @staticmethod - def char_kind_check(kind_str): - """If is a supported character 'kind' argument, return its - standardized form, otherwise return False. - """ - kind_ok = False - if isinstance(kind_str, str): - # Character allows both len and kind but we only support len - kentries = [x.strip() for x in kind_str.split(',') if x.strip()] - if len(kentries) == 1: - if kentries[0][0:4].lower() == 'len=': - kind_ok = True - # end if (no else, kind_ok already False) - # end if (no else, kind_ok already False) - # end if (no else, kind_ok already False) - return kind_ok - - def units_to_string(self, units, context=None): - """Replace variable unit description with string that is a legal - Python identifier. - If the resulting string is a Python keyword, raise an exception.""" - # Start with breaking up the string by spaces - items = units.split() - # Identify units with positive exponents - # without a plus sign (m2 instead of m+2). - pattern = re.compile(r"([a-zA-Z]+)([0-9]+)") - for index, item in enumerate(items): - match = pattern.match(item) - if match: - items[index] = "+".join(match.groups()) - # Combine list into string using underscores - string = "_".join(items) - # Replace each minus sign with '_minus_' - string = string.replace("-","_minus_") - # Replace each plus sign with '_plus_' - string = string.replace("+","_plus_") - # "1" is a valid unit - if string == "1": - string = "one" - # end if - # Test that the resulting string is a valid Python identifier - if not string.isidentifier(): - emsg = "Unsupported units entry for {}, '{}'{}" - ctx = context_string(context) - raise ParseSyntaxError(emsg.format(self.__stdname, units ,ctx)) - # end if - # Test that the resulting string is NOT a Python keyword - if keyword.iskeyword(string): - emsg = "Invalid units entry, '{}', Python identifier" - raise ParseSyntaxError(emsg.format(units), - context=context) - # end if - return string - - @staticmethod - def __regularize_dimensions(dims): - """Regularize by substituting a standin for any horizontal - dimension description (e.g., 'ccpp_constant_one:horizontal_loop_extent', - 'horizontal_loop_begin:horizontal_loop_end'). Also, regularize all - other dimensions by adding 'ccpp_constant_one' to any singleton - dimension. - Return the regularized dimensions. - """ - new_dims = list() - for dim in dims: - if is_horizontal_dimension(dim): - new_dims.append(_HDIM_TEMPNAME) - elif ':' not in dim: - new_dims.append('ccpp_constant_one:' + dim) - else: - new_dims.append(dim) - # end if - # end for - return new_dims - - @property - def incompat_reason(self): - """Return the reason(s) the two variables are incompatible (or an - empty string)""" - return self.__incompat_reason - - @property - def equiv(self): - """Return True if this object describes two Var objects which are - equivalent (i.e., no transformation required to pass one to the other). - """ - return self.__equiv - - @property - def compat(self): - """Return True if this object describes two Var objects which are - compatible (i.e., the values from one can be transferred to the other - via the transformation(s) described in the object). - """ - return self.__compat - - @property - def has_dim_transforms(self): - """Return True if this object has dimension transformations. - The dimension transformations is a tuple for forward and reverse - transformation. - The forward dimension transformation is a permutation of the indices of - the first variable to the second. - The reverse dimension transformation is a permutation of the indices of - the second variable to the first. - """ - return self.__dim_transforms is not None - - @property - def has_kind_transforms(self): - """Return True if this object has the kind transformation. - The kind transformation is a tuple containing the forward and reverse - kind transformations. - The forward kind transformation is a string representation of the - kind of the second variable. - The reverse kind transformation is a string representation of the - kind of the first variable. - """ - return self.__kind_transforms is not None - - @property - def has_unit_transforms(self): - """Return True if this object has the unit transformations. - The unit transformations is a tuple with forward and reverse unit - transformations. - The forward unit transformation is a string representation of the - equation to transform the first variable into the units of the second - The reverse unit transformation is a string representation of the - equation to transform the second variable into the units of the first - Each unit transform is a string which can be formatted with - and arguments to produce code to transform one variable into - the correct units of the other. - """ - return self.__unit_transforms is not None and self.__unit_transforms[0] - - def __bool__(self): - """Return True if this object describes two Var objects which are - equivalent (i.e., no transformation required to pass one to the other). - """ - return self.equiv - -############################################################################### diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt deleted file mode 100644 index a7428a77..00000000 --- a/src/CMakeLists.txt +++ /dev/null @@ -1,45 +0,0 @@ -include(GNUInstallDirs) - -#------------------------------------------------------------------------------ -# Set the sources -set(SOURCES_F90 - ccpp_types.F90 -) - -set(CMAKE_Fortran_MODULE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}) - -#------------------------------------------------------------------------------ -# Define the executable and what to link -add_library(ccpp_framework STATIC ${SOURCES_F90}) -target_link_libraries(ccpp_framework PUBLIC MPI::MPI_Fortran) -if(OPENMP) - target_link_libraries(ccpp_framework PUBLIC OpenMP::OpenMP_Fortran) -endif() -set_target_properties(ccpp_framework PROPERTIES - VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) - -#------------------------------------------------------------------------------ -# Installation -# -target_include_directories(ccpp_framework PUBLIC - INTERFACE $ - $ -) - - -# Define where to install the library -install(TARGETS ccpp_framework - EXPORT ccpp_framework-targets - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION bin -) - -# Export our configuration -install(EXPORT ccpp_framework-targets - FILE ccpp_framework-config.cmake - DESTINATION lib/cmake -) - -install(DIRECTORY ${CMAKE_Fortran_MODULE_DIRECTORY}/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/src/ccpp_constituent_prop_mod.F90 b/src/ccpp_constituent_prop_mod.F90 deleted file mode 100644 index d881e308..00000000 --- a/src/ccpp_constituent_prop_mod.F90 +++ /dev/null @@ -1,2636 +0,0 @@ -module ccpp_constituent_prop_mod - - ! ccpp_contituent_prop_mod contains types and procedures for storing - ! and retrieving constituent properties - - use ccpp_hashable, only: ccpp_hashable_t, & - ccpp_hashable_char_t - use ccpp_hash_table, only: ccpp_hash_table_t, & - ccpp_hash_iterator_t - use ccpp_kinds, only: kind_phys - - implicit none - private - - !!XXgoldyXX: Implement "last_error" method so that functions do not - !! need to have output variables. - - ! Private module data - integer, parameter :: stdname_len = 256 - integer, parameter :: dimname_len = 32 - integer, parameter :: errmsg_len = 256 - integer, parameter :: dry_mixing_ratio = -2 - integer, parameter :: moist_mixing_ratio = -3 - integer, parameter :: wet_mixing_ratio = -4 - integer, parameter :: mass_mixing_ratio = -5 - integer, parameter :: volume_mixing_ratio = -6 - integer, parameter :: number_concentration = -7 - integer, public, parameter :: int_unassigned = -huge(1) - real(kind=kind_phys), parameter :: kphys_unassigned = huge(1.0_kind_phys) - - !! \section arg_table_ccpp_constituent_properties_t - !! \htmlinclude ccpp_constituent_properties_t.html - !! - type, public, extends(ccpp_hashable_char_t) :: ccpp_constituent_properties_t - ! A ccpp_constituent_properties_t object holds relevant metadata - ! for a constituent species and provides interfaces to access that data. - character(len=:), private, allocatable :: var_std_name - character(len=:), private, allocatable :: var_long_name - character(len=:), private, allocatable :: var_diag_name - character(len=:), private, allocatable :: var_units - character(len=:), private, allocatable :: vert_dim - integer, private :: const_ind = int_unassigned - logical, private :: advected = .false. - logical, private :: thermo_active = .false. - logical, private :: water_species = .false. - ! While the quantities below can be derived from the standard name, - ! this implementation avoids string searching in parameterizations - ! const_type distinguishes mass, volume, and number conc. mixing ratios - integer, private :: const_type = int_unassigned - ! const_water distinguishes dry, moist, and "wet" mixing ratios - integer, private :: const_water = int_unassigned - ! minimum_mr is the minimum allowed value (default zero) - real(kind=kind_phys), private :: min_val = 0.0_kind_phys - ! molar_mass_val is the molar mass of the constituent (kg mol-1) - real(kind=kind_phys), private :: molar_mass_val = kphys_unassigned - ! default_value is the default value that the constituent array will be - ! initialized to - real(kind=kind_phys), private :: const_default_value = kphys_unassigned - contains - ! Required hashable method - procedure :: key => ccp_properties_get_key - ! Informational methods - procedure :: is_instantiated => ccp_is_instantiated - procedure :: standard_name => ccp_get_standard_name - procedure :: long_name => ccp_get_long_name - procedure :: diagnostic_name => ccp_get_diagnostic_name - procedure :: units => ccp_get_units - procedure :: is_layer_var => ccp_is_layer_var - procedure :: is_interface_var => ccp_is_interface_var - procedure :: is_2d_var => ccp_is_2d_var - procedure :: vertical_dimension => ccp_get_vertical_dimension - procedure :: const_index => ccp_const_index - procedure :: is_advected => ccp_is_advected - procedure :: is_thermo_active => ccp_is_thermo_active - procedure :: is_water_species => ccp_is_water_species - procedure :: equivalent => ccp_is_equivalent - procedure :: is_mass_mixing_ratio => ccp_is_mass_mixing_ratio - procedure :: is_volume_mixing_ratio => ccp_is_volume_mixing_ratio - procedure :: is_number_concentration => ccp_is_number_concentration - procedure :: is_dry => ccp_is_dry - procedure :: is_moist => ccp_is_moist - procedure :: is_wet => ccp_is_wet - procedure :: minimum => ccp_min_val - procedure :: molar_mass => ccp_molar_mass - procedure :: default_value => ccp_default_value - procedure :: has_default => ccp_has_default - procedure :: is_match => ccp_is_match - ! Copy method (be sure to update this anytime fields are added) - procedure :: copyconstituent - generic :: assignment(=) => copyconstituent - ! Methods that change state (XXgoldyXX: make private?) - procedure :: instantiate => ccp_instantiate - procedure :: deallocate => ccp_deallocate - procedure :: set_const_index => ccp_set_const_index - procedure :: set_thermo_active => ccp_set_thermo_active - procedure :: set_water_species => ccp_set_water_species - procedure :: set_minimum => ccp_set_min_val - procedure :: set_molar_mass => ccp_set_molar_mass - end type ccpp_constituent_properties_t - - !! \section arg_table_ccpp_constituent_prop_ptr_t - !! \htmlinclude ccpp_constituent_prop_ptr_t.html - !! - type, public :: ccpp_constituent_prop_ptr_t - type(ccpp_constituent_properties_t), private, pointer :: prop => null() - contains - ! Informational methods - procedure :: standard_name => ccpt_get_standard_name - procedure :: long_name => ccpt_get_long_name - procedure :: diagnostic_name => ccpt_get_diagnostic_name - procedure :: units => ccpt_get_units - procedure :: is_layer_var => ccpt_is_layer_var - procedure :: is_interface_var => ccpt_is_interface_var - procedure :: is_2d_var => ccpt_is_2d_var - procedure :: vertical_dimension => ccpt_get_vertical_dimension - procedure :: const_index => ccpt_const_index - procedure :: is_advected => ccpt_is_advected - procedure :: is_thermo_active => ccpt_is_thermo_active - procedure :: is_water_species => ccpt_is_water_species - procedure :: is_mass_mixing_ratio => ccpt_is_mass_mixing_ratio - procedure :: is_volume_mixing_ratio => ccpt_is_volume_mixing_ratio - procedure :: is_number_concentration => ccpt_is_number_concentration - procedure :: is_dry => ccpt_is_dry - procedure :: is_moist => ccpt_is_moist - procedure :: is_wet => ccpt_is_wet - procedure :: minimum => ccpt_min_val - procedure :: molar_mass => ccpt_molar_mass - procedure :: default_value => ccpt_default_value - procedure :: has_default => ccpt_has_default - ! ccpt_set: Set the internal pointer - procedure :: set => ccpt_set - ! Methods that change state (XXgoldyXX: make private?) - procedure :: deallocate => ccpt_deallocate - procedure :: set_const_index => ccpt_set_const_index - procedure :: set_thermo_active => ccpt_set_thermo_active - procedure :: set_water_species => ccpt_set_water_species - procedure :: set_minimum => ccpt_set_min_val - procedure :: set_molar_mass => ccpt_set_molar_mass - end type ccpp_constituent_prop_ptr_t - - !! \section arg_table_ccpp_model_constituents_t - !! \htmlinclude ccpp_model_constituents_t.html - !! - type, public :: ccpp_model_constituents_t - ! A ccpp_model_constituents_t object holds all the metadata and field - ! data for a model run's constituents along with data and methods - ! to initialize and access the data. - !!XXgoldyXX: To do: allow accessor functions as CCPP local variable - !! names so that members can be private. - integer :: num_layer_vars = 0 - integer :: num_advected_vars = 0 - integer, private :: num_layers = 0 - type(ccpp_hash_table_t), private :: hash_table - logical, private :: table_locked = .false. - logical, private :: data_locked = .false. - ! These fields are public to allow for efficient (i.e., no copying) - ! usage even though it breaks object independence - real(kind=kind_phys), allocatable :: vars_layer(:, :, :) - real(kind=kind_phys), allocatable :: vars_layer_tend(:, :, :) - real(kind=kind_phys), allocatable :: vars_minvalue(:) - ! An array containing all the constituent metadata - ! Each element contains a pointer to a constituent from the hash table - type(ccpp_constituent_prop_ptr_t), allocatable :: const_metadata(:) - contains - ! Return .true. if a constituent matches pattern - procedure, private :: is_match => ccp_model_const_is_match - ! Return a constituent from the hash table - procedure, private :: find_const => ccp_model_const_find_const - ! Are both the properties table and data array locked (i.e., ready to be used)? - procedure :: locked => ccp_model_const_locked - ! Is the properties table locked (i.e., ready to be used)? - procedure :: const_props_locked => ccp_model_const_props_locked - ! Is the data array locked (i.e., ready to be used)? - procedure :: const_data_locked => ccp_model_const_data_locked - ! Is it okay to add new metadata fields? - procedure :: okay_to_add => ccp_model_const_okay_to_add - ! Add a constituent's metadata to the master hash table - procedure :: new_field => ccp_model_const_add_metadata - ! Initialize hash table - procedure :: initialize_table => ccp_model_const_initialize - ! Freeze hash table and set constituents properties - procedure :: lock_table => ccp_model_const_table_lock - ! Freeze and initialize constituent field arrays - procedure :: lock_data => ccp_model_const_data_lock - ! Empty (reset) the entire object - procedure :: reset => ccp_model_const_reset - ! Query number of constituents matching pattern - procedure :: num_constituents => ccp_model_const_num_match - ! Return index of constituent matching standard name - procedure :: const_index => ccp_model_const_index - ! Return metadata matching standard name - procedure :: field_metadata => ccp_model_const_metadata - ! Gather constituent fields matching pattern - procedure :: copy_in => ccp_model_const_copy_in_3d - ! Update constituent fields matching pattern - procedure :: copy_out => ccp_model_const_copy_out_3d - ! Return pointer to constituent array (for use by host model) - procedure :: field_data_ptr => ccp_field_data_ptr - ! Return pointer to advected constituent array (for use by host model) - procedure :: advected_constituents_ptr => ccp_advected_data_ptr - ! Return pointer to constituent properties array (for use by host model) - procedure :: constituent_props_ptr => ccp_constituent_props_ptr - end type ccpp_model_constituents_t - - ! Private interfaces - private to_str - private initialize_errvars - private append_errvars - private handle_allocate_error - private check_var_bounds - -contains - - !######################################################################## - ! - ! CCPP_CONSTITUENT_PROPERTIES_T (constituent metadata) methods - ! - !######################################################################## - - subroutine copyconstituent(outconst, inconst) - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(inout) :: outconst - type(ccpp_constituent_properties_t), intent(in) :: inconst - - outconst%var_std_name = inconst%var_std_name - outconst%var_long_name = inconst%var_long_name - outconst%var_diag_name = inconst%var_diag_name - outconst%vert_dim = inconst%vert_dim - outconst%const_ind = inconst%const_ind - outconst%advected = inconst%advected - outconst%const_type = inconst%const_type - outconst%const_water = inconst%const_water - outconst%min_val = inconst%min_val - outconst%const_default_value = inconst%const_default_value - outconst%molar_mass_val = inconst%molar_mass_val - outconst%thermo_active = inconst%thermo_active - outconst%water_species = inconst%water_species - outconst%var_units = inconst%var_units - outconst%const_water = inconst%const_water - end subroutine copyconstituent - - !####################################################################### - - character(len=10) function to_str(val) - ! return default integer as a left justified string - - ! Dummy argument - integer, intent(in) :: val - - write(to_str, '(i0)') val - - end function to_str - - !####################################################################### - - subroutine initialize_errvars(errcode, errmsg) - ! Initialize error variables, if present - - ! Dummy arguments - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - if (present(errcode)) then - errcode = 0 - end if - if (present(errmsg)) then - errmsg = '' - end if - end subroutine initialize_errvars - - !####################################################################### - - subroutine append_errvars(errcode_val, errmsg_val, subname, errcode, errmsg, caller) - ! Append to error variables, if present - - ! Dummy arguments - integer, intent(in) :: errcode_val - character(len=*), intent(in) :: errmsg_val - character(len=*), intent(in) :: subname - integer, optional, intent(inout) :: errcode - character(len=*), optional, intent(inout) :: errmsg - character(len=*), optional, intent(in) :: caller - ! Local variable - integer :: emsg_len - - if (present(errcode)) then - errcode = errcode + errcode_val - end if - if (present(errmsg)) then - emsg_len = len_trim(errmsg) - if (emsg_len > 0) then - errmsg(emsg_len + 1:) = '; ' - end if - emsg_len = len_trim(errmsg) - if (present(caller)) then - errmsg(emsg_len + 1:) = trim(caller) // " " // trim(errmsg_val) - else - errmsg(emsg_len + 1:) = trim(subname) // " " // trim(errmsg_val) - end if - end if - end subroutine append_errvars - - !####################################################################### - - subroutine handle_allocate_error(astat, fieldname, subname, errcode, errmsg) - ! Generate an error message if indicates an allocation failure - - ! Dummy arguments - integer, intent(in) :: astat - character(len=*), intent(in) :: fieldname - character(len=*), intent(in) :: subname - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - call initialize_errvars(errcode, errmsg) - if (astat /= 0) then - call append_errvars(astat, "Error allocating ccpp_constituent_properties_t object component " // & - trim(fieldname) // ", error code = " // to_str(astat), subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine handle_allocate_error - - !####################################################################### - - subroutine check_var_bounds(var, var_bound, varname, subname, errcode, errmsg) - ! Generate an error message if indicates an allocation failure - - ! Dummy arguments - integer, intent(in) :: var - integer, intent(in) :: var_bound - character(len=*), intent(in) :: varname - character(len=*), intent(in) :: subname - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - call initialize_errvars(errcode, errmsg) - if (var > var_bound) then - call append_errvars(1, trim(varname) // " exceeds its upper bound, " // & - to_str(var_bound), subname, errcode=errcode, errmsg=errmsg) - end if - end subroutine check_var_bounds - - !####################################################################### - - function ccp_properties_get_key(hashable) - ! Return the constituent properties class key (var_std_name) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: hashable - character(len=:), allocatable :: ccp_properties_get_key - - ccp_properties_get_key = hashable%var_std_name - - end function ccp_properties_get_key - - !####################################################################### - - logical function ccp_is_instantiated(this, errcode, errmsg) - ! Return .true. iff is instantiated - ! If is *not* instantiated and and/or is present, - ! fill these fields with an error status - ! If *is* instantiated and and/or is present, - ! clear these fields. - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - character(len=*), parameter :: subname = 'ccp_is_instantiated' - - ccp_is_instantiated = allocated(this%var_std_name) - call initialize_errvars(errcode, errmsg) - if (.not. ccp_is_instantiated) then - call append_errvars(1, "ccpp_constituent_properties_t object is not initialized", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end function ccp_is_instantiated - - !####################################################################### - - subroutine ccp_instantiate(this, std_name, long_name, diag_name, units, & - vertical_dim, advected, default_value, min_value, molar_mass, water_species, & - mixing_ratio_type, errcode, errmsg) - ! Initialize all fields in - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(inout) :: this - character(len=*), intent(in) :: std_name - character(len=*), intent(in) :: long_name - character(len=*), intent(in) :: diag_name - character(len=*), intent(in) :: units - character(len=*), intent(in) :: vertical_dim - logical, optional, intent(in) :: advected - real(kind=kind_phys), optional, intent(in) :: default_value - real(kind=kind_phys), optional, intent(in) :: min_value - real(kind=kind_phys), optional, intent(in) :: molar_mass - logical, optional, intent(in) :: water_species - character(len=*), optional, intent(in) :: mixing_ratio_type - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - - if (this%is_instantiated()) then - errcode = 1 - write(errmsg, *) 'ccpp_constituent_properties_t object, ', & - trim(std_name), ', is already initialized as ', this%var_std_name - else - errcode = 0 - errmsg = '' - this%var_std_name = trim(std_name) - end if - if (errcode == 0) then - this%var_long_name = trim(long_name) - this%var_diag_name = trim(diag_name) - this%var_units = trim(units) - this%vert_dim = trim(vertical_dim) - if (present(advected)) then - this%advected = advected - else - this%advected = .false. - end if - if (present(default_value)) then - this%const_default_value = default_value - end if - if (present(min_value)) then - this%min_val = min_value - end if - if (present(molar_mass)) then - this%molar_mass_val = molar_mass - end if - if (present(water_species)) then - this%water_species = water_species - end if - end if - if (errcode == 0) then - if (index(this%var_std_name, "volume_mixing_ratio") > 0) then - this%const_type = volume_mixing_ratio - else if (index(this%var_std_name, "number_concentration") > 0) then - this%const_type = number_concentration - else - this%const_type = mass_mixing_ratio - end if - end if - if (errcode == 0) then - ! Determine if this mixing ratio is dry, moist, or "wet". - ! If a type was provided, use that (if it's valid) - if (present(mixing_ratio_type)) then - if (trim(mixing_ratio_type) == 'wet') then - this%const_water = wet_mixing_ratio - else if (trim(mixing_ratio_type) == 'moist') then - this%const_water = moist_mixing_ratio - else if (trim(mixing_ratio_type) == 'dry') then - this%const_water = dry_mixing_ratio - else - errcode = 1 - write(errmsg, *) 'ccp_instantiate: invalid mixing ratio type. ', & - 'Must be one of: "wet", "moist", or "dry". Got: "', & - trim(mixing_ratio_type), '"' - end if - else - ! Otherwise, parse it from the standard name - if (index(this%var_std_name, "wrt_moist_air_and_condensed_water") > 0) then - this%const_water = wet_mixing_ratio - else if (index(this%var_std_name, "wrt_moist_air") > 0) then - this%const_water = moist_mixing_ratio - else - this%const_water = dry_mixing_ratio - end if - end if - end if - if (errcode /= 0) then - call this%deallocate() - end if - end subroutine ccp_instantiate - - !####################################################################### - - subroutine ccp_deallocate(this) - ! Deallocate memory associated with this constituent property object - - ! Dummy argument - class(ccpp_constituent_properties_t), intent(inout) :: this - - if (allocated(this%var_std_name)) then - deallocate(this%var_std_name) - end if - if (allocated(this%var_long_name)) then - deallocate(this%var_long_name) - end if - if (allocated(this%var_diag_name)) then - deallocate(this%var_diag_name) - end if - if (allocated(this%vert_dim)) then - deallocate(this%vert_dim) - end if - this%const_ind = int_unassigned - this%advected = .false. - this%const_type = int_unassigned - this%const_water = int_unassigned - this%const_default_value = kphys_unassigned - - end subroutine ccp_deallocate - - !####################################################################### - - subroutine ccp_get_standard_name(this, std_name, errcode, errmsg) - ! Return this constituent's standard name - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - character(len=*), intent(out) :: std_name - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - std_name = this%var_std_name - else - std_name = '' - end if - - end subroutine ccp_get_standard_name - - !####################################################################### - - subroutine ccp_get_long_name(this, long_name, errcode, errmsg) - ! Return this constituent's long name (description) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - character(len=*), intent(out) :: long_name - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - long_name = this%var_long_name - else - long_name = '' - end if - - end subroutine ccp_get_long_name - - !####################################################################### - - subroutine ccp_get_diagnostic_name(this, diag_name, errcode, errmsg) - ! Return this constituent's diagnostic name - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - character(len=*), intent(out) :: diag_name - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - diag_name = this%var_diag_name - else - diag_name = '' - end if - - end subroutine ccp_get_diagnostic_name - - !####################################################################### - - subroutine ccp_get_units(this, units, errcode, errmsg) - ! Return this constituent's units - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - character(len=*), intent(out) :: units - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - units = this%var_units - else - units = '' - end if - - end subroutine ccp_get_units - - !####################################################################### - - subroutine ccp_get_vertical_dimension(this, vert_dim, errcode, errmsg) - ! Return the standard name of this constituent's vertical dimension - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - character(len=*), intent(out) :: vert_dim - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - vert_dim = this%vert_dim - else - vert_dim = '' - end if - - end subroutine ccp_get_vertical_dimension - - !####################################################################### - - logical function ccp_is_layer_var(this) result(is_layer) - ! Return .true. iff this constituent has a layer vertical dimension - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - ! Local variable - character(len=dimname_len) :: dimname - - call this%vertical_dimension(dimname) - is_layer = trim(dimname) == 'vertical_layer_dimension' - - end function ccp_is_layer_var - - !####################################################################### - - logical function ccp_is_interface_var(this) result(is_interface) - ! Return .true. iff this constituent has a interface vertical dimension - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - ! Local variable - character(len=dimname_len) :: dimname - - call this%vertical_dimension(dimname) - is_interface = trim(dimname) == 'vertical_interface_dimension' - - end function ccp_is_interface_var - - !####################################################################### - - logical function ccp_is_2d_var(this) result(is_2d) - ! Return .true. iff this constituent has a 2d vertical dimension - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - ! Local variable - character(len=dimname_len) :: dimname - - call this%vertical_dimension(dimname) - is_2d = len_trim(dimname) == 0 - - end function ccp_is_2d_var - - !####################################################################### - - integer function ccp_const_index(this, errcode, errmsg) - ! Return this constituent's array index (or -1 of not assigned) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - ccp_const_index = this%const_ind - else - ccp_const_index = int_unassigned - end if - - end function ccp_const_index - - !####################################################################### - - subroutine ccp_set_const_index(this, index, errcode, errmsg) - ! Set this constituent's index in the master constituent array - ! It is an error to try to set an index if it is already set - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(inout) :: this - integer, intent(in) :: index - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - character(len=*), parameter :: subname = 'ccp_set_const_index' - - if (this%is_instantiated(errcode, errmsg)) then - if (this%const_ind == int_unassigned) then - this%const_ind = index - else - call append_errvars(1, "ccpp_constituent_properties_t const index " // & - "is already set", subname, errcode=errcode, errmsg=errmsg) - end if - end if - - end subroutine ccp_set_const_index - - !####################################################################### - - subroutine ccp_set_thermo_active(this, thermo_flag, errcode, errmsg) - ! Set whether this constituent is thermodynamically active, which - ! means that certain physics schemes will use this constitutent - ! when calculating thermodynamic quantities (e.g. enthalpy). - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(inout) :: this - logical, intent(in) :: thermo_flag - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - !Set thermodynamically active flag for this constituent: - if (this%is_instantiated(errcode, errmsg)) then - this%thermo_active = thermo_flag - end if - - end subroutine ccp_set_thermo_active - - !####################################################################### - - subroutine ccp_set_water_species(this, water_flag, errcode, errmsg) - ! Set whether this constituent is a water species, which means - ! that this constituent represents a particular phase or type - ! of water in the atmosphere. - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(inout) :: this - logical, intent(in) :: water_flag - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - !Set water species flag for this constituent: - if (this%is_instantiated(errcode, errmsg)) then - this%water_species = water_flag - end if - - end subroutine ccp_set_water_species - - !####################################################################### - - subroutine ccp_is_thermo_active(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - logical, intent(out) :: val_out - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - !If instantiated then check if constituent is - !thermodynamically active, otherwise return false: - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%thermo_active - else - val_out = .false. - end if - end subroutine ccp_is_thermo_active - - !####################################################################### - - subroutine ccp_is_water_species(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - logical, intent(out) :: val_out - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - !If instantiated then check if constituent is - !a water species, otherwise return false: - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%water_species - else - val_out = .false. - end if - end subroutine ccp_is_water_species - - !####################################################################### - - subroutine ccp_is_advected(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - logical, intent(out) :: val_out - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%advected - else - val_out = .false. - end if - end subroutine ccp_is_advected - - !####################################################################### - - subroutine ccp_is_equivalent(this, oconst, equiv, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - type(ccpp_constituent_properties_t), intent(in) :: oconst - logical, intent(out) :: equiv - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg) .and. & - oconst%is_instantiated(errcode, errmsg)) then - equiv = (trim(this%var_std_name) == trim(oconst%var_std_name)) .and. & - (trim(this%var_long_name) == trim(oconst%var_long_name)) .and. & - (trim(this%var_diag_name) == trim(oconst%var_diag_name)) .and. & - (trim(this%vert_dim) == trim(oconst%vert_dim)) .and. & - (trim(this%var_units) == trim(oconst%var_units)) .and. & - (this%advected .eqv. oconst%advected) .and. & - (this%const_default_value == oconst%const_default_value) .and. & - (this%min_val == oconst%min_val) .and. & - (this%molar_mass_val == oconst%molar_mass_val) .and. & - (this%thermo_active .eqv. oconst%thermo_active) .and. & - (this%const_water == oconst%const_water) .and. & - (this%water_species .eqv. oconst%water_species) - else - equiv = .false. - end if - - end subroutine ccp_is_equivalent - - !######################################################################## - - subroutine ccp_is_mass_mixing_ratio(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%const_type == mass_mixing_ratio - else - val_out = .false. - end if - end subroutine ccp_is_mass_mixing_ratio - - !######################################################################## - - subroutine ccp_is_volume_mixing_ratio(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%const_type == volume_mixing_ratio - else - val_out = .false. - end if - end subroutine ccp_is_volume_mixing_ratio - - !######################################################################## - - subroutine ccp_is_number_concentration(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%const_type == number_concentration - else - val_out = .false. - end if - end subroutine ccp_is_number_concentration - - !######################################################################## - - subroutine ccp_is_dry(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%const_water == dry_mixing_ratio - else - val_out = .false. - end if - - end subroutine ccp_is_dry - - !######################################################################## - - subroutine ccp_is_moist(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%const_water == moist_mixing_ratio - else - val_out = .false. - end if - - end subroutine ccp_is_moist - - !######################################################################## - - subroutine ccp_is_wet(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%const_water == wet_mixing_ratio - else - val_out = .false. - end if - - end subroutine ccp_is_wet - - !######################################################################## - - subroutine ccp_min_val(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - real(kind=kind_phys), intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%min_val - else - val_out = kphys_unassigned - end if - - end subroutine ccp_min_val - - !######################################################################## - - subroutine ccp_set_min_val(this, min_value, errcode, errmsg) - ! Set the minimum value of this particular constituent. - ! If this subroutine is never used then the minimum - ! value defaults to zero. - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(inout) :: this - real(kind=kind_phys), intent(in) :: min_value - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - !Set minimum allowed value for this constituent: - if (this%is_instantiated(errcode, errmsg)) then - this%min_val = min_value - end if - - end subroutine ccp_set_min_val - - !######################################################################## - - subroutine ccp_molar_mass(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - real(kind=kind_phys), intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%molar_mass_val - else - val_out = kphys_unassigned - end if - - end subroutine ccp_molar_mass - - !######################################################################## - - subroutine ccp_set_molar_mass(this, molar_mass, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(inout) :: this - real(kind=kind_phys), intent(in) :: molar_mass - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - this%molar_mass_val = molar_mass - end if - - end subroutine ccp_set_molar_mass - - !######################################################################## - - subroutine ccp_default_value(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - real(kind=kind_phys), intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%const_default_value - else - val_out = kphys_unassigned - end if - - end subroutine ccp_default_value - - !######################################################################## - - subroutine ccp_has_default(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccp_has_default' - - if (this%is_instantiated(errcode, errmsg)) then - val_out = this%const_default_value /= kphys_unassigned - else - val_out = .false. - end if - - end subroutine ccp_has_default - - !######################################################################## - - logical function ccp_is_match(this, comp_props) result(is_match) - ! Return .true. iff the constituent's properties match the checked - ! attributes of another constituent properties object - ! Since this is a private function, error checking for locked status - ! is *not* performed. - - ! Dummy arguments - class(ccpp_constituent_properties_t), intent(in) :: this - type(ccpp_constituent_properties_t), intent(in) :: comp_props - ! Local variable - logical :: val, comp_val - character(len=stdname_len) :: char_val, char_comp_val - - ! By default, every constituent is a match - is_match = .true. - ! Check: advected, thermo_active, water_species, units - call this%is_advected(val) - call comp_props%is_advected(comp_val) - if (val .neqv. comp_val) then - is_match = .false. - return - end if - - call this%is_thermo_active(val) - call comp_props%is_thermo_active(comp_val) - if (val .neqv. comp_val) then - is_match = .false. - return - end if - - call this%is_water_species(val) - call comp_props%is_water_species(comp_val) - if (val .neqv. comp_val) then - is_match = .false. - return - end if - - call this%units(char_val) - call comp_props%units(char_comp_val) - if (trim(char_val) /= trim(char_comp_val)) then - is_match = .false. - return - end if - - end function ccp_is_match - - !######################################################################## - ! - ! CCPP_MODEL_CONSTITUENTS_T (constituent field data) methods - ! - !######################################################################## - - logical function ccp_model_const_locked(this, errcode, errmsg, warn_func) - ! Return .true. iff is locked (i.e., ready to use) - ! Optionally fill out and if object not initialized - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(in) :: this - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - character(len=*), optional, intent(in) :: warn_func - ! Local variable - character(len=*), parameter :: subname = 'ccp_model_const_locked' - - call initialize_errvars(errcode, errmsg) - ccp_model_const_locked = .false. - ! Use an initialized hash table as double check - if (this%hash_table%is_initialized()) then - ccp_model_const_locked = this%table_locked .and. this%data_locked - if ((.not. (this%table_locked .and. this%data_locked)) .and. & - present(errmsg) .and. present(warn_func)) then - ! Write a warning as a courtesy to calling function but do not set - ! errcode (let caller decide). - write(errmsg, *) trim(warn_func), & - ' WARNING: Model constituents not ready to use' - end if - else - call append_errvars(1, "WARNING: Model constituents not initialized", & - subname, errcode=errcode, errmsg=errmsg, caller=warn_func) - end if - - end function ccp_model_const_locked - - !######################################################################## - - logical function ccp_model_const_props_locked(this, errcode, errmsg, warn_func) - ! Return .true. iff 's constituent properties are ready to use - ! Optionally fill out and if object not initialized - ! Dummy arguments - class(ccpp_model_constituents_t), intent(in) :: this - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - character(len=*), optional, intent(in) :: warn_func - ! Local variable - character(len=*), parameter :: subname = 'ccp_model_const_table_locked' - - call initialize_errvars(errcode, errmsg) - ccp_model_const_props_locked = .false. - ! Use an initialized hash table as double check - if (this%hash_table%is_initialized()) then - ccp_model_const_props_locked = this%table_locked - if (.not. this%table_locked .and. & - present(errmsg) .and. present(warn_func)) then - ! Write a warning as a courtesy to calling function but do not set - ! errcode (let caller decide). - write(errmsg, *) trim(warn_func), & - ' WARNING: Model constituent properties not ready to use' - end if - else - call append_errvars(1, & - "WARNING: Model constituent properties not initialized", & - subname, errcode=errcode, errmsg=errmsg, caller=warn_func) - end if - - end function ccp_model_const_props_locked - - !######################################################################## - - logical function ccp_model_const_data_locked(this, errcode, errmsg, warn_func) - ! Return .true. iff 's data are ready to use - ! Optionally fill out and if object not initialized - ! Dummy arguments - class(ccpp_model_constituents_t), intent(in) :: this - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - character(len=*), optional, intent(in) :: warn_func - ! Local variable - character(len=*), parameter :: subname = 'ccp_model_const_data_locked' - - call initialize_errvars(errcode, errmsg) - ccp_model_const_data_locked = .false. - ! Use an initialized hash table as double check - if (this%hash_table%is_initialized()) then - ccp_model_const_data_locked = this%data_locked - if (.not. this%data_locked .and. & - present(errmsg) .and. present(warn_func)) then - ! Write a warning as a courtesy to calling function but do not set - ! errcode (let caller decide). - write(errmsg, *) trim(warn_func), & - ' WARNING: Model constituent data not ready to use' - end if - else - call append_errvars(1, & - "WARNING: Model constituent data not initialized", & - subname, errcode=errcode, errmsg=errmsg, caller=warn_func) - end if - - end function ccp_model_const_data_locked - - !######################################################################## - - logical function ccp_model_const_okay_to_add(this, errcode, errmsg, & - warn_func) - ! Return .true. iff is initialized and not locked - ! Optionally fill out and if the conditions - ! are not met. - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(inout) :: this - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - character(len=*), optional, intent(in) :: warn_func - ! Local variable - character(len=*), parameter :: subname = 'ccp_model_const_okay_to_add' - - ccp_model_const_okay_to_add = this%hash_table%is_initialized() - if (ccp_model_const_okay_to_add) then - ccp_model_const_okay_to_add = .not. (this%const_props_locked(errcode=errcode, & - errmsg=errmsg, warn_func=subname) .or. this%const_data_locked(errcode=errcode, & - errmsg=errmsg, warn_func=subname)) - if (.not. ccp_model_const_okay_to_add) then - call append_errvars(1, & - "WARNING: Model constituents are locked", & - subname, errcode=errcode, errmsg=errmsg, caller=warn_func) - end if - else - call append_errvars(1, & - "WARNING: Model constituents not initialized", & - subname, errcode=errcode, errmsg=errmsg, caller=warn_func) - end if - - end function ccp_model_const_okay_to_add - - !######################################################################## - - subroutine ccp_model_const_add_metadata(this, field_data, errcode, errmsg) - ! Add a constituent's metadata to the master hash table - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(inout) :: this - type(ccpp_constituent_properties_t), target, intent(in) :: field_data - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variables - character(len=errmsg_len) :: error - character(len=*), parameter :: subname = 'ccp_model_const_add_metadata' - type(ccpp_constituent_properties_t), pointer :: cprop => null() - character(len=stdname_len) :: standard_name - logical :: match - - if (this%okay_to_add(errcode=errcode, errmsg=errmsg, & - warn_func=subname)) then - error = '' - ! Check to see if standard name is already in the table - call field_data%standard_name(standard_name, errcode, errmsg) - cprop => this%find_const(standard_name) - if (associated(cprop)) then - ! Standard name already in table, let's see if the existing constituent is the same - match = cprop%is_match(field_data) - if (match) then - ! Existing constituent is a match - no need to throw an error, just don't add - return - else - ! Existing constituent is not a match - this is an error - call append_errvars(1, "ERROR: Trying to add constituent " // & - trim(standard_name) // " but an incompatible" // & - " constituent with this name already exists", subname, & - errcode=errcode, errmsg=errmsg) - return - end if - end if - call this%hash_table%add_hash_key(field_data, error) - if (len_trim(error) > 0) then - call append_errvars(1, trim(error), subname, errcode=errcode, errmsg=errmsg) - else - ! If we get here we are successful, add to variable count - if (field_data%is_layer_var()) then - this%num_layer_vars = this%num_layer_vars + 1 - else - if (present(errmsg)) then - call field_data%vertical_dimension(error, & - errcode=errcode, errmsg=errmsg) - if (errcode /= 0) then - call append_errvars(1, & - "ERROR: Unknown vertical dimension, '" // & - trim(error) // "'", subname, & - errcode=errcode, errmsg=errmsg) - end if - end if - end if - end if - else - call append_errvars(1, "WARNING: Model constituents are locked", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccp_model_const_add_metadata - - !######################################################################## - - subroutine ccp_model_const_initialize(this, num_elements) - ! Initialize hash table, is total number of elements - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(inout) :: this - integer, intent(in) :: num_elements - ! Local variable - integer :: tbl_size - - ! Clear any data - call this%reset() - ! Figure a log base 2 for initializing hash table - tbl_size = num_elements * 10 ! Hash padding - tbl_size = int((log(real(tbl_size, kind_phys)) / log(2.0_kind_phys)) + & - 1.0_kind_phys) - ! Initialize hash table - call this%hash_table%initialize(tbl_size) - this%table_locked = .false. - - end subroutine ccp_model_const_initialize - - !######################################################################## - - function ccp_model_const_find_const(this, standard_name, errcode, errmsg) & - result(cprop) - ! Return a constituent with key, , from the hash table - ! must be locked to execute this function - ! Since this is a private function, error checking for locked status - ! is *not* performed. - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(in) :: this - character(len=*), intent(in) :: standard_name - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - type(ccpp_constituent_properties_t), pointer :: cprop - ! Local variables - class(ccpp_hashable_t), pointer :: hval - character(len=errmsg_len) :: error - character(len=*), parameter :: subname = 'ccp_model_const_find_const' - - nullify(cprop) - - hval => this%hash_table%table_value(standard_name, errmsg=error) - if (len_trim(error) > 0) then - call append_errvars(1, trim(error), subname, & - errcode=errcode, errmsg=errmsg) - else - select type (hval) - type is (ccpp_constituent_properties_t) - cprop => hval - class default - call append_errvars(1, "ERROR: Bad hash table value " // & - trim(standard_name), subname, errcode=errcode, errmsg=errmsg) - end select - end if - - end function ccp_model_const_find_const - - !######################################################################## - - subroutine ccp_model_const_table_lock(this, errcode, errmsg) - ! Freeze hash table and initialize constituent properties - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(inout) :: this - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variables - integer :: index_const - integer :: index_advect - integer :: num_vars - integer :: astat - integer :: errcode_local - logical :: check - type(ccpp_hash_iterator_t) :: hiter - class(ccpp_hashable_t), pointer :: hval - type(ccpp_constituent_properties_t), pointer :: cprop - character(len=dimname_len) :: dimname - character(len=*), parameter :: subname = 'ccp_model_const_table_lock' - - astat = 0 - errcode_local = 0 - if (this%const_props_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then - call append_errvars(1, & - "WARNING: Model constituents properties already locked, ignoring", & - subname, errcode=errcode, errmsg=errmsg) - errcode_local = 1 - else - ! Make sure everything is really initialized - call this%reset(clear_hash_table=.false.) - this%num_advected_vars = 0 - ! Allocate the constituent array - num_vars = this%hash_table%num_values() - allocate(this%const_metadata(num_vars), stat=astat) - call handle_allocate_error(astat, 'const_metadata', & - subname, errcode=errcode, errmsg=errmsg) - ! We want to pack the advected constituents at the beginning of - ! the field array so we need to know how many there are - if (astat == 0) then - call hiter%initialize(this%hash_table) - do - if (hiter%valid()) then - hval => hiter%value() - select type (hval) - type is (ccpp_constituent_properties_t) - cprop => hval - call cprop%is_advected(check) - if (check) then - this%num_advected_vars = this%num_advected_vars + 1 - end if - end select - call hiter%next() - else - exit - end if - end do - ! Sanity check on num_advect - if (this%num_advected_vars > num_vars) then - call append_errvars(1, "ERROR: num_advected_vars index " // & - to_str(this%num_advected_vars) // & - " out of bounds " // to_str(num_vars), & - subname, errcode=errcode, errmsg=errmsg) - errcode_local = 1 - end if - end if - index_advect = 0 - index_const = this%num_advected_vars - ! Iterate through the hash table to find entries - if (errcode_local == 0) then - call hiter%initialize(this%hash_table) - do - if (hiter%valid()) then - hval => hiter%value() - select type (hval) - type is (ccpp_constituent_properties_t) - cprop => hval - call cprop%is_advected(check) - if (check) then - index_advect = index_advect + 1 - if (index_advect > this%num_advected_vars) then - call append_errvars(1, "ERROR: const a index " // & - to_str(index_advect) // " out of bounds " // & - to_str(this%num_advected_vars), & - subname, errcode=errcode, errmsg=errmsg) - errcode_local = errcode_local + 1 - exit - end if - call cprop%set_const_index(index_advect, & - errcode=errcode, errmsg=errmsg) - call this%const_metadata(index_advect)%set(cprop) - else - index_const = index_const + 1 - if (index_const > num_vars) then - call append_errvars(1, "ERROR: const v index " // & - to_str(index_const) // " out of bounds " // & - to_str(num_vars), subname, errcode=errcode, & - errmsg=errmsg) - errcode_local = errcode_local + 1 - exit - end if - call cprop%set_const_index(index_const, & - errcode=errcode, errmsg=errmsg) - call this%const_metadata(index_const)%set(cprop) - end if - ! Make sure this is a layer variable - if (.not. cprop%is_layer_var()) then - call cprop%vertical_dimension(dimname, & - errcode=errcode, errmsg=errmsg) - call append_errvars(1, "ERROR: Bad vertical dimension, '" // & - trim(dimname), subname, errcode=errcode, errmsg=errmsg) - errcode_local = errcode_local + 1 - exit - end if - class default - call append_errvars(1, "ERROR: Bad hash table value", & - subname, errcode=errcode, errmsg=errmsg) - errcode_local = errcode_local + 1 - exit - end select - call hiter%next() - else - exit - end if - end do - ! Some size sanity checks - if (index_const /= this%hash_table%num_values()) then - call append_errvars(1, "ERROR: Too few constituents " // & - to_str(index_const) // " found in hash table " // & - to_str(this%hash_table%num_values()), subname, & - errcode=errcode, errmsg=errmsg) - errcode_local = errcode_local + 1 - end if - if (index_advect /= this%num_advected_vars) then - call append_errvars(1, "ERROR: Too few advected constituents " // & - to_str(index_const) // " found in hash table " // & - to_str(this%hash_table%num_values()), subname, & - errcode=errcode, errmsg=errmsg) - errcode_local = errcode_local + 1 - end if - if (present(errcode)) then - if (errcode /= 0) then - errcode_local = 1 - end if - end if - if (errcode_local == 0) then - this%table_locked = .true. - end if - end if - end if - - end subroutine ccp_model_const_table_lock - - !######################################################################## - - subroutine ccp_model_const_data_lock(this, ncols, num_layers, errcode, errmsg) - ! Freeze hash table and initialize constituent arrays - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(inout) :: this - integer, intent(in) :: ncols - integer, intent(in) :: num_layers - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variables - integer :: astat, index, errcode_local - real(kind=kind_phys) :: default_value - real(kind=kind_phys) :: minvalue - character(len=*), parameter :: subname = 'ccp_model_const_data_lock' - - errcode_local = 0 - if (this%const_data_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then - call append_errvars(1, & - "WARNING: Model constituent data already locked, ignoring", & - subname, errcode=errcode, errmsg=errmsg) - errcode_local = errcode_local + 1 - else if (.not. this%const_props_locked(errcode=errcode, errmsg=errmsg, & - warn_func=subname)) then - call append_errvars(1, & - "WARNING: Model constituent properties not yet locked, ignoring", & - subname, errcode=errcode, errmsg=errmsg) - errcode_local = errcode_local + 1 - else - allocate(this%vars_layer(ncols, num_layers, this%hash_table%num_values()), & - stat=astat) - call handle_allocate_error(astat, 'vars_layer', & - subname, errcode=errcode, errmsg=errmsg) - errcode_local = astat - if (astat == 0) then - allocate(this%vars_layer_tend(ncols, num_layers, this%hash_table%num_values()), & - stat=astat) - call handle_allocate_error(astat, 'vars_layer_tend', & - subname, errcode=errcode, errmsg=errmsg) - errcode_local = astat - end if - if (astat == 0) then - allocate(this%vars_minvalue(this%hash_table%num_values()), stat=astat) - call handle_allocate_error(astat, 'vars_minvalue', & - subname, errcode=errcode, errmsg=errmsg) - errcode_local = astat - end if - ! Initialize tendencies to 0 - this%vars_layer_tend(:, :, :) = 0._kind_phys - if (errcode_local == 0) then - this%num_layers = num_layers - do index = 1, this%hash_table%num_values() - !Set all constituents to their default values: - call this%const_metadata(index)%default_value(default_value, & - errcode, errmsg) - this%vars_layer(:, :, index) = default_value - - ! Also set the minimum allowed value array - call this%const_metadata(index)%minimum(minvalue, errcode, & - errmsg) - this%vars_minvalue(index) = minvalue - end do - end if - if (present(errcode)) then - if (errcode /= 0) then - errcode_local = 1 - end if - end if - if (errcode_local == 0) then - this%data_locked = .true. - end if - end if - - end subroutine ccp_model_const_data_lock - - !######################################################################## - - subroutine ccp_model_const_reset(this, clear_hash_table) - ! Empty (reset) the entire object - ! Optionally do not clear the hash table (and its data) - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(inout) :: this - logical, optional, intent(in) :: clear_hash_table - ! Local variables - logical :: clear_table - integer :: index - - if (present(clear_hash_table)) then - clear_table = clear_hash_table - else - clear_table = .true. - end if - if (allocated(this%vars_layer)) then - deallocate(this%vars_layer) - end if - if (allocated(this%vars_minvalue)) then - deallocate(this%vars_minvalue) - end if - if (allocated(this%vars_layer_tend)) then - deallocate(this%vars_layer_tend) - end if - if (allocated(this%const_metadata)) then - if (clear_table) then - do index = 1, size(this%const_metadata, 1) - call this%const_metadata(index)%deallocate() - end do - end if - deallocate(this%const_metadata) - end if - if (clear_table) then - this%num_layer_vars = 0 - this%num_advected_vars = 0 - this%num_layers = 0 - call this%hash_table%clear() - end if - - end subroutine ccp_model_const_reset - - !######################################################################## - - logical function ccp_model_const_is_match(this, index, advected, & - thermo_active, water_species) result(is_match) - ! Return .true. iff the constituent at matches a pattern - ! Each (optional) property which is present represents something - ! which is required as part of a match. - ! Since this is a private function, error checking for locked status - ! is *not* performed. - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(in) :: this - integer, intent(in) :: index - logical, optional, intent(in) :: advected - logical, optional, intent(in) :: thermo_active - logical, optional, intent(in) :: water_species - ! Local variable - logical :: check - - ! By default, every constituent is a match - is_match = .true. - if (present(advected)) then - call this%const_metadata(index)%is_advected(check) - if (advected .neqv. check) then - is_match = .false. - end if - end if - - if (present(thermo_active)) then - call this%const_metadata(index)%is_thermo_active(check) - if (thermo_active .neqv. check) then - is_match = .false. - end if - end if - - if (present(water_species)) then - call this%const_metadata(index)%is_water_species(check) - if (water_species .neqv. check) then - is_match = .false. - end if - end if - - end function ccp_model_const_is_match - - !######################################################################## - - subroutine ccp_model_const_num_match(this, nmatch, advected, thermo_active, & - water_species, errcode, errmsg) - ! Query number of constituents matching pattern - ! Each (optional) property which is present represents something - ! which is required as part of a match. - ! must be locked to execute this function - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(in) :: this - integer, intent(out) :: nmatch - logical, optional, intent(in) :: advected - logical, optional, intent(in) :: thermo_active - logical, optional, intent(in) :: water_species - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variables - integer :: index - character(len=*), parameter :: subname = "ccp_model_const_num_match" - - nmatch = 0 - if (this%const_props_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then - do index = 1, size(this%const_metadata) - if (this%is_match(index, advected=advected, thermo_active=thermo_active, & - water_species=water_species)) then - nmatch = nmatch + 1 - end if - end do - end if - - end subroutine ccp_model_const_num_match - - !######################################################################## - - subroutine ccp_model_const_index(this, index, standard_name, errcode, errmsg) - ! Return index of metadata matching . - ! must be locked to execute this function - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(in) :: this - character(len=*), intent(in) :: standard_name - integer, intent(out) :: index - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variables - type(ccpp_constituent_properties_t), pointer :: cprop => null() - character(len=*), parameter :: subname = "ccp_model_const_index" - - if (this%const_props_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then - cprop => this%find_const(standard_name) - if (associated(cprop)) then - index = cprop%const_index() - else - index = int_unassigned - end if - else - index = int_unassigned - end if - - end subroutine ccp_model_const_index - - !######################################################################## - - subroutine ccp_model_const_metadata(this, standard_name, const_data, & - errcode, errmsg) - ! Return metadata matching standard name - ! must be locked to execute this function - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(in) :: this - character(len=*), intent(in) :: standard_name - type(ccpp_constituent_properties_t), intent(out) :: const_data - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variables - type(ccpp_constituent_properties_t), pointer :: cprop => null() - character(len=*), parameter :: subname = "ccp_model_const_metadata" - - if (this%const_props_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then - cprop => this%find_const(standard_name, errcode=errcode, errmsg=errmsg) - if (associated(cprop)) then - const_data = cprop - end if - end if - - end subroutine ccp_model_const_metadata - - !######################################################################## - - subroutine ccp_model_const_copy_in_3d(this, const_array, advected, & - thermo_active, water_species, errcode, errmsg) - ! Gather constituent fields matching pattern - ! Each (optional) property which is present represents something - ! which is required as part of a match. - ! must be locked to execute this function - - ! Dummy arguments - class(ccpp_model_constituents_t), intent(in) :: this - real(kind=kind_phys), intent(out) :: const_array(:, :, :) - logical, optional, intent(in) :: advected - logical, optional, intent(in) :: thermo_active - logical, optional, intent(in) :: water_species - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variables - integer :: index ! const_metadata index - integer :: cindex ! const_array index - integer :: fld_ind ! const field index - integer :: max_cind ! Size of const_array - integer :: num_levels ! Levels of const_array - character(len=stdname_len) :: std_name - character(len=*), parameter :: subname = "ccp_model_const_copy_in_3d" - - if (this%locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then - cindex = 0 - max_cind = size(const_array, 3) - num_levels = size(const_array, 2) - do index = 1, size(this%const_metadata) - if (this%is_match(index, advected=advected, & - thermo_active=thermo_active, & - water_species=water_species)) then - ! See if we have room for another constituent - cindex = cindex + 1 - if (cindex > max_cind) then - call append_errvars(1, & - ": Too many constituents for ", & - subname, errcode=errcode, errmsg=errmsg) - exit - end if - ! Copy this constituent's field data to - call this%const_metadata(index)%const_index(fld_ind) - if (fld_ind /= index) then - call this%const_metadata(index)%standard_name(std_name) - call append_errvars(1, ": ERROR: " // & - "bad field index, " // to_str(fld_ind) // & - " for '" // trim(std_name) // "', should have been " // & - to_str(index), subname, errcode=errcode, errmsg=errmsg) - exit - else if (this%const_metadata(index)%is_layer_var()) then - if (this%num_layers == num_levels) then - const_array(:, :, cindex) = this%vars_layer(:, :, fld_ind) - else - call this%const_metadata(index)%standard_name(std_name) - call append_errvars(1, ": ERROR: " // & - "Wrong number of vertical levels for '" // & - trim(std_name) // "', " // to_str(num_levels) // & - ", expected " // to_str(this%num_layers), & - subname, errcode=errcode, errmsg=errmsg) - exit - end if - else - call this%const_metadata(index)%standard_name(std_name) - call append_errvars(1, ": Unsupported var type," // & - " wrong number of vertical levels for '" // & - trim(std_name) // "', " // to_str(num_levels) // & - ", expected" // to_str(this%num_layers), & - subname, errcode=errcode, errmsg=errmsg) - exit - end if - end if - end do - end if - - end subroutine ccp_model_const_copy_in_3d - - !######################################################################## - - subroutine ccp_model_const_copy_out_3d(this, const_array, advected, & - thermo_active, water_species, errcode, errmsg) - ! Update constituent fields matching pattern - ! Each (optional) property which is present represents something - ! which is required as part of a match. - ! must be locked to execute this function - - ! Dummy argument - class(ccpp_model_constituents_t), intent(inout) :: this - real(kind=kind_phys), intent(in) :: const_array(:, :, :) - logical, optional, intent(in) :: advected - logical, optional, intent(in) :: thermo_active - logical, optional, intent(in) :: water_species - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variables - integer :: index ! const_metadata index - integer :: cindex ! const_array index - integer :: fld_ind ! const field index - integer :: max_cind ! Size of const_array - integer :: num_levels ! Levels of const_array - character(len=stdname_len) :: std_name - character(len=*), parameter :: subname = "ccp_model_const_copy_out_3d" - - if (this%locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then - cindex = 0 - max_cind = size(const_array, 3) - num_levels = size(const_array, 2) - do index = 1, size(this%const_metadata) - if (this%is_match(index, advected=advected, & - thermo_active=thermo_active, & - water_species=water_species)) then - ! See if we have room for another constituent - cindex = cindex + 1 - if (cindex > max_cind) then - call append_errvars(1, & - ": Too many constituents for ", & - subname, errcode=errcode, errmsg=errmsg) - exit - end if - ! Copy this field of to to constituent's field data - call this%const_metadata(index)%const_index(fld_ind) - if (fld_ind /= index) then - call this%const_metadata(index)%standard_name(std_name) - call append_errvars(1, ": ERROR: " // & - "bad field index, " // to_str(fld_ind) // & - " for '" // trim(std_name) // "', should have been" // & - to_str(index), subname, errcode=errcode, errmsg=errmsg) - exit - else if (this%const_metadata(index)%is_layer_var()) then - if (this%num_layers == num_levels) then - this%vars_layer(:, :, fld_ind) = const_array(:, :, cindex) - else - call this%const_metadata(index)%standard_name(std_name) - call append_errvars(1, & - ": Wrong number of vertical levels for '" // & - trim(std_name) // "', " // to_str(num_levels) // & - ", expected" // to_str(this%num_layers), & - subname, errcode=errcode, errmsg=errmsg) - exit - end if - else - call this%const_metadata(index)%standard_name(std_name) - call append_errvars(1, ": Unsupported var type," // & - " wrong number of vertical levels for'" // & - trim(std_name) // "', " // to_str(num_levels) // & - ", expected " // to_str(this%num_layers), & - subname, errcode=errcode, errmsg=errmsg) - exit - end if - end if - end do - end if - - end subroutine ccp_model_const_copy_out_3d - - !######################################################################## - - function ccp_field_data_ptr(this) result(const_ptr) - ! Return pointer to constituent array (for use by host model) - - ! Dummy arguments - class(ccpp_model_constituents_t), target, intent(inout) :: this - real(kind=kind_phys), pointer :: const_ptr(:, :, :) - ! Local variables - integer :: errcode - character(len=errmsg_len) :: errmsg - character(len=*), parameter :: subname = 'ccp_field_data_ptr' - - if (this%locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then - const_ptr => this%vars_layer - else - ! We don't want output variables in a function so just nullify - ! See note above about creating a 'last_error' method - nullify(const_ptr) - end if - - end function ccp_field_data_ptr - - !######################################################################## - - function ccp_advected_data_ptr(this) result(const_ptr) - ! Return pointer to advected constituent array (for use by host model) - - ! Dummy arguments - class(ccpp_model_constituents_t), target, intent(inout) :: this - real(kind=kind_phys), pointer :: const_ptr(:, :, :) - ! Local variables - integer :: errcode - character(len=errmsg_len) :: errmsg - character(len=*), parameter :: subname = 'ccp_advected_data_ptr' - - if (this%locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then - const_ptr => this%vars_layer(:, :, 1:this%num_advected_vars) - else - ! We don't want output variables in a function so just nullify - ! See note above about creating a 'last_error' method - nullify(const_ptr) - end if - - end function ccp_advected_data_ptr - - function ccp_constituent_props_ptr(this) result(const_ptr) - ! Return pointer to constituent properties array (for use by host model) - - ! Dummy arguments - class(ccpp_model_constituents_t), target, intent(inout) :: this - type(ccpp_constituent_prop_ptr_t), pointer :: const_ptr(:) - ! Local variables - integer :: errcode - character(len=errmsg_len) :: errmsg - character(len=*), parameter :: subname = 'ccp_constituent_props_ptr' - - if (this%const_props_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then - const_ptr => this%const_metadata - else - ! We don't want output variables in a function so just nullify - ! See note above about creating a 'last_error' method - nullify(const_ptr) - end if - - end function ccp_constituent_props_ptr - - !######################################################################## - - !##################################### - ! ccpp_constituent_prop_ptr_t methods - !##################################### - - !####################################################################### - - subroutine ccpt_get_standard_name(this, std_name, errcode, errmsg) - ! Return this constituent's standard name - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - character(len=*), intent(out) :: std_name - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_get_standard_name' - - if (associated(this%prop)) then - call this%prop%standard_name(std_name, errcode, errmsg) - else - std_name = '' - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_get_standard_name - - !####################################################################### - - subroutine ccpt_get_long_name(this, long_name, errcode, errmsg) - ! Return this constituent's long name (description) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - character(len=*), intent(out) :: long_name - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_get_long_name' - - if (associated(this%prop)) then - call this%prop%long_name(long_name, errcode, errmsg) - else - long_name = '' - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_get_long_name - - !####################################################################### - - subroutine ccpt_get_diagnostic_name(this, diag_name, errcode, errmsg) - ! Return this constituent's diagnostic name - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - character(len=*), intent(out) :: diag_name - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_get_diagnostic_name' - - if (associated(this%prop)) then - call this%prop%diagnostic_name(diag_name, errcode, errmsg) - else - diag_name = '' - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_get_diagnostic_name - - !####################################################################### - - subroutine ccpt_get_units(this, units, errcode, errmsg) - ! Return this constituent's units - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - character(len=*), intent(out) :: units - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_get_units' - - if (associated(this%prop)) then - call this%prop%units(units, errcode, errmsg) - else - units = '' - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_get_units - - !####################################################################### - - subroutine ccpt_get_vertical_dimension(this, vert_dim, errcode, errmsg) - ! Return the standard name of this constituent's vertical dimension - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - character(len=*), intent(out) :: vert_dim - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_get_vertical_dimension' - - if (associated(this%prop)) then - if (this%prop%is_instantiated(errcode, errmsg)) then - call this%prop%vertical_dimension(vert_dim, errcode, errmsg) - end if - else - vert_dim = '' - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_get_vertical_dimension - - !####################################################################### - - logical function ccpt_is_layer_var(this) result(is_layer) - ! Return .true. iff this constituent has a layer vertical dimension - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - ! Local variables - character(len=dimname_len) :: dimname - character(len=*), parameter :: subname = 'ccpt_is_layer_var' - - if (associated(this%prop)) then - call this%prop%vertical_dimension(dimname) - is_layer = trim(dimname) == 'vertical_layer_dimension' - else - is_layer = .false. - end if - - end function ccpt_is_layer_var - - !####################################################################### - - logical function ccpt_is_interface_var(this) result(is_interface) - ! Return .true. iff this constituent has a interface vertical dimension - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - ! Local variables - character(len=dimname_len) :: dimname - character(len=*), parameter :: subname = 'ccpt_is_interface_var' - - if (associated(this%prop)) then - call this%prop%vertical_dimension(dimname) - is_interface = trim(dimname) == 'vertical_interface_dimension' - else - is_interface = .false. - end if - - end function ccpt_is_interface_var - - !####################################################################### - - logical function ccpt_is_2d_var(this) result(is_2d) - ! Return .true. iff this constituent has a 2d vertical dimension - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - ! Local variables - character(len=dimname_len) :: dimname - character(len=*), parameter :: subname = 'ccpt_is_2d_var' - - if (associated(this%prop)) then - call this%prop%vertical_dimension(dimname) - is_2d = len_trim(dimname) == 0 - else - is_2d = .false. - end if - - end function ccpt_is_2d_var - - !####################################################################### - - subroutine ccpt_const_index(this, index, errcode, errmsg) - ! Return this constituent's master index (or -1 of not assigned) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - integer, intent(out) :: index - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_const_index' - - if (associated(this%prop)) then - index = this%prop%const_index(errcode, errmsg) - else - index = int_unassigned - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_const_index - - !####################################################################### - - subroutine ccpt_is_thermo_active(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - logical, intent(out) :: val_out - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_is_thermo_active' - - if (associated(this%prop)) then - call this%prop%is_thermo_active(val_out, errcode, errmsg) - else - val_out = .false. - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_is_thermo_active - - !####################################################################### - - subroutine ccpt_is_water_species(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - logical, intent(out) :: val_out - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_is_water_species' - - if (associated(this%prop)) then - call this%prop%is_water_species(val_out, errcode, errmsg) - else - val_out = .false. - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_is_water_species - - !####################################################################### - - subroutine ccpt_is_advected(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - logical, intent(out) :: val_out - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_is_advected' - - if (associated(this%prop)) then - call this%prop%is_advected(val_out, errcode, errmsg) - else - val_out = .false. - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_is_advected - - !######################################################################## - - subroutine ccpt_is_mass_mixing_ratio(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_is_mass_mixing_ratio' - - if (associated(this%prop)) then - call this%prop%is_mass_mixing_ratio(val_out, errcode, errmsg) - else - val_out = .false. - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_is_mass_mixing_ratio - - !######################################################################## - - subroutine ccpt_is_volume_mixing_ratio(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_is_volume_mixing_ratio' - - if (associated(this%prop)) then - call this%prop%is_volume_mixing_ratio(val_out, errcode, errmsg) - else - val_out = .false. - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_is_volume_mixing_ratio - - !######################################################################## - - subroutine ccpt_is_number_concentration(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_is_number_concentration' - - if (associated(this%prop)) then - call this%prop%is_number_concentration(val_out, errcode, errmsg) - else - val_out = .false. - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_is_number_concentration - - !######################################################################## - - subroutine ccpt_is_dry(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_is_dry' - - if (associated(this%prop)) then - call this%prop%is_dry(val_out, errcode, errmsg) - else - val_out = .false. - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_is_dry - - !######################################################################## - - subroutine ccpt_is_moist(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_is_moist' - - if (associated(this%prop)) then - call this%prop%is_moist(val_out, errcode, errmsg) - else - val_out = .false. - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_is_moist - - !######################################################################## - - subroutine ccpt_is_wet(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_is_wet' - - if (associated(this%prop)) then - call this%prop%is_wet(val_out, errcode, errmsg) - else - val_out = .false. - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_is_wet - - !######################################################################## - - subroutine ccpt_min_val(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - real(kind=kind_phys), intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_min_val' - - if (associated(this%prop)) then - call this%prop%minimum(val_out, errcode, errmsg) - else - val_out = kphys_unassigned - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_min_val - - !######################################################################## - - subroutine ccpt_set_min_val(this, min_value, errcode, errmsg) - ! Set the minimum value of this particular constituent. - ! If this subroutine is never used then the minimum - ! value defaults to zero. - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(inout) :: this - real(kind=kind_phys), intent(in) :: min_value - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_set_min_val' - - !Set minimum value for this constituent: - if (associated(this%prop)) then - call this%prop%set_minimum(min_value, errcode, errmsg) - else - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_set_min_val - - !######################################################################## - - subroutine ccpt_molar_mass(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - real(kind=kind_phys), intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_molar_mass' - - if (associated(this%prop)) then - call this%prop%molar_mass(val_out, errcode, errmsg) - else - val_out = kphys_unassigned - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_molar_mass - - !######################################################################## - - subroutine ccpt_set_molar_mass(this, molar_mass, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(inout) :: this - real(kind=kind_phys), intent(in) :: molar_mass - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_set_molar_mass' - - if (associated(this%prop)) then - call this%prop%set_molar_mass(molar_mass, errcode, errmsg) - else - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_set_molar_mass - - !######################################################################## - - subroutine ccpt_default_value(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - real(kind=kind_phys), intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_default_value' - - if (associated(this%prop)) then - call this%prop%default_value(val_out, errcode, errmsg) - else - val_out = kphys_unassigned - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_default_value - - !######################################################################## - - subroutine ccpt_has_default(this, val_out, errcode, errmsg) - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(in) :: this - logical, intent(out) :: val_out - integer, intent(out) :: errcode - character(len=*), intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_has_default' - - if (associated(this%prop)) then - call this%prop%has_default(val_out, errcode, errmsg) - else - val_out = .false. - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_has_default - - !######################################################################## - - subroutine ccpt_set(this, const_ptr, errcode, errmsg) - ! Set the pointer to , however, an error is recorded if - ! the pointer is already set. - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(inout) :: this - type(ccpp_constituent_properties_t), pointer :: const_ptr - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variables - character(len=stdname_len) :: stdname - character(len=errmsg_len) :: errmsg2 - character(len=*), parameter :: subname = 'ccpt_set' - - call initialize_errvars(errcode, errmsg) - if (associated(this%prop)) then - call this%standard_name(stdname, errcode=errcode, errmsg=errmsg2) - if (errcode == 0) then - write(errmsg2, *) "Pointer already allocated as '", & - trim(stdname), "'" - end if - errcode = errcode + 1 - call append_errvars(1, trim(errmsg2), subname, errcode=errcode, & - errmsg=errmsg) - else - this%prop => const_ptr - end if - - end subroutine ccpt_set - - !######################################################################## - - subroutine ccpt_deallocate(this) - ! Deallocate the constituent object pointer if it is allocated. - - ! Dummy argument - class(ccpp_constituent_prop_ptr_t), intent(inout) :: this - - if (associated(this%prop)) then - call this%prop%deallocate() - deallocate(this%prop) - end if - nullify(this%prop) - - end subroutine ccpt_deallocate - - !####################################################################### - - subroutine ccpt_set_const_index(this, index, errcode, errmsg) - ! Set this constituent's index in the master constituent array - ! It is an error to try to set an index if it is already set - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(inout) :: this - integer, intent(in) :: index - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_set_const_index' - - if (associated(this%prop)) then - if (this%prop%is_instantiated(errcode, errmsg)) then - if (this%prop%const_ind == int_unassigned) then - this%prop%const_ind = index - else - call append_errvars(1, "ccpp_constituent_prop_ptr_t " // & - "const index is already set", & - subname, errcode=errcode, errmsg=errmsg) - end if - end if - else - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_set_const_index - - !####################################################################### - - subroutine ccpt_set_thermo_active(this, thermo_flag, errcode, errmsg) - ! Set whether this constituent is thermodynamically active, which - ! means that certain physics schemes will use this constitutent - ! when calculating thermodynamic quantities (e.g. enthalpy). - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(inout) :: this - logical, intent(in) :: thermo_flag - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_set_thermo_active' - - if (associated(this%prop)) then - if (this%prop%is_instantiated(errcode, errmsg)) then - this%prop%thermo_active = thermo_flag - end if - else - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_set_thermo_active - - !####################################################################### - - subroutine ccpt_set_water_species(this, water_flag, errcode, errmsg) - ! Set whether this constituent is a water species, which means - ! that this constituent represents a particular phase or type - ! of water in the atmosphere. - - ! Dummy arguments - class(ccpp_constituent_prop_ptr_t), intent(inout) :: this - logical, intent(in) :: water_flag - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - ! Local variable - character(len=*), parameter :: subname = 'ccpt_set_water_species' - - if (associated(this%prop)) then - if (this%prop%is_instantiated(errcode, errmsg)) then - this%prop%water_species = water_flag - end if - else - call append_errvars(1, ": invalid constituent pointer", & - subname, errcode=errcode, errmsg=errmsg) - end if - - end subroutine ccpt_set_water_species - -end module ccpp_constituent_prop_mod diff --git a/src/ccpp_constituent_prop_mod.meta b/src/ccpp_constituent_prop_mod.meta deleted file mode 100644 index 77f446e6..00000000 --- a/src/ccpp_constituent_prop_mod.meta +++ /dev/null @@ -1,64 +0,0 @@ -######################################################################## - -[ccpp-table-properties] - name = ccpp_constituent_properties_t - type = ddt - -[ccpp-arg-table] - name = ccpp_constituent_properties_t - type = ddt - -######################################################################## - -[ccpp-table-properties] - name = ccpp_constituent_prop_ptr_t - type = ddt - -[ccpp-arg-table] - name = ccpp_constituent_prop_ptr_t - type = ddt - -######################################################################## -[ccpp-table-properties] - name = ccpp_model_constituents_t - type = ddt - -[ccpp-arg-table] - name = ccpp_model_constituents_t - type = ddt -[ num_layer_vars ] - standard_name = number_of_ccpp_constituents - long_name = Number of constituents managed by CCPP Framework - units = count - dimensions = () - type = integer -[ num_advected_vars ] - standard_name = number_of_ccpp_advected_constituents - long_name = Number of advected constituents managed by CCPP Framework - units = count - dimensions = () - type = integer -[ vars_layer ] - standard_name = ccpp_constituents - long_name = Array of constituents managed by CCPP Framework - units = none - state_variable = true - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys -[ vars_layer_tend ] - standard_name = ccpp_constituent_tendencies - long_name = Array of constituent tendencies managed by CCPP Framework - units = none - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys -[ const_metadata ] - standard_name = ccpp_constituent_properties - units = None - type = ccpp_constituent_prop_ptr_t - dimensions = (number_of_ccpp_constituents) -[ vars_minvalue ] - standard_name = ccpp_constituent_minimum_values - units = kg kg-1 - type = real | kind = kind_phys - dimensions = (number_of_ccpp_constituents) - protected = True diff --git a/src/ccpp_hash_table.F90 b/src/ccpp_hash_table.F90 deleted file mode 100644 index 9f175a3a..00000000 --- a/src/ccpp_hash_table.F90 +++ /dev/null @@ -1,520 +0,0 @@ -!!XXgoldyXX: To do, statistics output -module ccpp_hash_table - - use ccpp_hashable, only: ccpp_hashable_t - - implicit none - private - - ! - ! Constants used in hashing function gen_hash_key. - ! - - integer, parameter :: gen_hash_key_offset = 21467 ! z'000053db' - - integer, parameter :: tbl_max_idx = 15 - integer, parameter, dimension(0:tbl_max_idx) :: tbl_gen_hash_key = & - (/ 61, 59, 53, 47, 43, 41, 37, 31, 29, 23, 17, 13, 11, 7, 3, 1 /) - - integer, parameter :: table_factor_size = 8 ! Table size / # entries - integer, parameter :: table_overflow_factor = 4 ! # entries / Overflow size - - type :: table_entry_t - ! Any table entry contains a key and a value - class(ccpp_hashable_t), pointer :: entry_value => null() - type(table_entry_t), pointer :: next => null() - contains - final :: finalize_table_entry - end type table_entry_t - - type, public :: ccpp_hash_table_t - ! ccpp_hash_table_t contains all information to build and use a hash table - ! It also keeps track of statistics such as collision frequency and size - integer, private :: table_size = -1 - integer, private :: key_offset = gen_hash_key_offset - type(table_entry_t), private, allocatable :: table(:) - ! Statistics - integer, private :: num_keys = 0 - integer, private :: num_key_collisions = 0 - integer, private :: max_collision = 0 - contains - procedure :: is_initialized => hash_table_is_initialized - procedure :: initialize => hash_table_initialize_table - procedure :: key_hash => hash_table_key_hash - procedure :: add_hash_key => hash_table_add_hash_key - procedure :: table_value => hash_table_table_value - procedure :: num_values => hash_table_num_values - procedure :: clear => hash_table_clear_table - end type ccpp_hash_table_t - - type, public :: ccpp_hash_iterator_t - ! ccpp_hash_iterator contains information allowing iteration through all - ! entries in a hash table - integer, private :: index = 0 - type(table_entry_t), private, pointer :: table_entry => null() - type(ccpp_hash_table_t), private, pointer :: hash_table => null() - contains - procedure :: initialize => hash_iterator_initialize - procedure :: key => hash_iterator_key - procedure :: next => hash_iterator_next_entry - procedure :: valid => hash_iterator_is_valid - procedure :: value => hash_iterator_value - end type ccpp_hash_iterator_t - - !! Private interfaces - private :: have_error ! Has a called routine detected an error? - private :: clear_optstring ! Clear a string, if present - -contains - - !####################################################################### - ! - ! Hash table methods - ! - !####################################################################### - - logical function have_error(errmsg) - ! Return .true. iff is present and contains text - - ! Dummy argument - character(len=*), optional, intent(in) :: errmsg - - have_error = present(errmsg) - if (have_error) then - have_error = len_trim(errmsg) > 0 - end if - end function have_error - - !####################################################################### - - subroutine clear_optstring(str) - ! clear if it is present - - ! Dummy argument - character(len=*), optional, intent(inout) :: str - - if (present(str)) then - str = '' - end if - end subroutine clear_optstring - - !####################################################################### - - elemental subroutine finalize_table_entry(te) - - ! Dummy argument - type(table_entry_t), intent(inout) :: te - ! Local variable - type(table_entry_t), pointer :: temp - - if (associated(te%entry_value)) then - nullify(te%entry_value) ! We may not own the memory - temp => te%next - nullify(te%next) - if (associated(temp)) then - deallocate(temp) - nullify(temp) - end if - end if - - end subroutine finalize_table_entry - - !####################################################################### - - logical function hash_table_is_initialized(this) - ! Return .true. iff is an initialized hash table - - ! Dummy argument - class(ccpp_hash_table_t) :: this - - hash_table_is_initialized = allocated(this%table) - - end function hash_table_is_initialized - - !####################################################################### - - subroutine hash_table_initialize_table(this, tbl_size, key_off) - ! Initialize this table. - - ! Dummy arguments - class(ccpp_hash_table_t) :: this - integer, intent(in) :: tbl_size ! new table size - integer, optional, intent(in) :: key_off ! key offset - - ! Clear this table so it can be initialized - if (allocated(this%table)) then - deallocate(this%table) - end if - this%num_keys = 0 - this%num_key_collisions = 0 - this%max_collision = 0 - ! Avoid too-large tables - this%table_size = ishft(1, min(tbl_size, bit_size(1) - 2)) - allocate(this%table(this%table_size)) - if (present(key_off)) then - this%key_offset = key_off - end if - end subroutine hash_table_initialize_table - - !####################################################################### - - integer function hash_table_key_hash(this, string, errmsg) result(hash_key) - ! - !----------------------------------------------------------------------- - ! - ! Purpose: Generate a hash key on the interval [0 .. tbl_hash_pri_sz-1] - ! given a character string. - ! - ! Algorithm is a variant of perl's internal hashing function. - ! - !----------------------------------------------------------------------- - ! - ! - ! Dummy Arguments: - ! - class(ccpp_hash_table_t) :: this - character(len=*), intent(in) :: string - character(len=*), optional, intent(out) :: errmsg - character(len=*), parameter :: subname = 'HASH_TABLE_KEY_HASH' - ! - ! Local. - ! - integer :: hash - integer :: index - integer :: ind_fact - integer :: hash_fact - - hash = this%key_offset - ind_fact = 0 - do index = 1, len_trim(string) - ind_fact = ind_fact + 1 - if (ind_fact > tbl_max_idx) then - ind_fact = 1 - end if - hash_fact = tbl_gen_hash_key(ind_fact) - hash = ieor(hash, (ichar(string(index:index)) * hash_fact)) - end do - - hash_key = iand(hash, this%table_size - 1) + 1 - if ((hash_key < 1) .or. (hash_key > this%table_size)) then - if (present(errmsg)) then - write(errmsg, '(2a,2(i0,a))') subname, ' ERROR: Key Hash, ', & - hash_key, ' out of bounds, [1, ', this%table_size, ']' - else - write(6, '(2a,2(i0,a))') subname, ' ERROR: Key Hash, ', & - hash_key, ' out of bounds, [1, ', this%table_size, ']' - stop 1 - end if - end if - - end function hash_table_key_hash - - !####################################################################### - - function hash_table_table_value(this, key, errmsg) result(tbl_val) - ! - !----------------------------------------------------------------------- - ! - ! Purpose: Return the the key value of - ! - ! If the object is not found, return NULL - ! - !----------------------------------------------------------------------- - ! - ! Dummy Arguments: - ! - class(ccpp_hash_table_t) :: this - character(len=*), intent(in) :: key - character(len=*), optional, intent(out) :: errmsg - class(ccpp_hashable_t), pointer :: tbl_val - ! - ! Local. - ! - integer :: hash_key - type(table_entry_t), pointer :: next_ptr - character(len=*), parameter :: subname = 'HASH_TABLE_TABLE_INDEX' - - call clear_optstring(errmsg) - nullify(tbl_val) - hash_key = this%key_hash(key, errmsg=errmsg) - if (have_error(errmsg)) then - errmsg = trim(errmsg) // ', called from ' // subname - else if (associated(this%table(hash_key)%entry_value)) then - if (this%table(hash_key)%entry_value%key() == trim(key)) then - tbl_val => this%table(hash_key)%entry_value - else - next_ptr => this%table(hash_key)%next - do - if (associated(next_ptr)) then - if (associated(next_ptr%entry_value)) then - if (next_ptr%entry_value%key() == trim(key)) then - tbl_val => next_ptr%entry_value - exit - end if - end if - next_ptr => next_ptr%next - else - exit - end if - end do - end if - end if - - if ((.not. associated(tbl_val)) .and. present(errmsg)) then - if (.not. have_error(errmsg)) then ! Still need to test for empty - write(errmsg, *) subname, ": No entry for '", trim(key), "'" - end if - end if - - end function hash_table_table_value - - !####################################################################### - - subroutine hash_table_add_hash_key(this, newval, errmsg) - ! - !----------------------------------------------------------------------- - ! - ! Purpose: Add to this hash table using its key - ! Its key must not be an empty string - ! It is an error to try to add a key more than once - ! - ! - !----------------------------------------------------------------------- - - ! Dummy arguments: - class(ccpp_hash_table_t) :: this - class(ccpp_hashable_t), target :: newval - character(len=*), optional, intent(out) :: errmsg - ! Local variables - integer :: hash_ind - integer :: ovflw_len - character(len=:), allocatable :: newkey - type(table_entry_t), pointer :: next_ptr - type(table_entry_t), pointer :: new_entry - character(len=*), parameter :: subname = 'HASH_TABLE_ADD_HASH_KEY' - - call clear_optstring(errmsg) - nullify(new_entry) - newkey = newval%key() - hash_ind = this%key_hash(newkey, errmsg=errmsg) - ! Check for this entry - if (have_error(errmsg)) then - errmsg = trim(errmsg) // ', called from ' // subname - else if (associated(this%table_value(newkey))) then - if (present(errmsg)) then - write(errmsg, *) subname, " ERROR: key, '", newkey, & - "' already in table" - end if - else - if (associated(this%table(hash_ind)%entry_value)) then - ! We have a collision, make a new entry - allocate(new_entry) - new_entry%entry_value => newval - ! Now, find a spot - if (associated(this%table(hash_ind)%next)) then - ovflw_len = 1 - next_ptr => this%table(hash_ind)%next - do - if (associated(next_ptr%next)) then - ovflw_len = ovflw_len + 1 - next_ptr => next_ptr%next - else - exit - end if - end do - ovflw_len = ovflw_len + 1 - next_ptr%next => new_entry - else - this%num_key_collisions = this%num_key_collisions + 1 - this%table(hash_ind)%next => new_entry - ovflw_len = 1 - end if - nullify(new_entry) - this%max_collision = max(this%max_collision, ovflw_len) - else - this%table(hash_ind)%entry_value => newval - end if - this%num_keys = this%num_keys + 1 - end if - - end subroutine hash_table_add_hash_key - - !####################################################################### - - integer function hash_table_num_values(this) result(numval) - ! - !----------------------------------------------------------------------- - ! - ! Purpose: Return the number of populated table values - ! - !----------------------------------------------------------------------- - - ! Dummy argument: - class(ccpp_hash_table_t) :: this - - numval = this%num_keys - - end function hash_table_num_values - - !####################################################################### - - subroutine hash_table_clear_table(this) - ! - !----------------------------------------------------------------------- - ! - ! Purpose: Deallocate the hash table and all of its entries - ! - !----------------------------------------------------------------------- - - ! Dummy argument: - class(ccpp_hash_table_t) :: this - - ! Clear all the table entries - if (this%is_initialized()) then - if (allocated(this%table)) then - ! This should deallocate the entire chain of entries - deallocate(this%table) - end if - end if - this%table_size = -1 - this%num_keys = 0 - this%num_key_collisions = 0 - this%max_collision = 0 - - end subroutine hash_table_clear_table - - !####################################################################### - ! - ! Hash iterator methods - ! - !####################################################################### - - subroutine hash_iterator_initialize(this, hash_table) - ! Initialize a hash_table iterator to the first value in the hash table - ! Note that the table_entry pointer is only used for the "next" field - ! in the hash table (entry itself is not a pointer). - - ! Dummy arguments - class(ccpp_hash_iterator_t) :: this - class(ccpp_hash_table_t), target :: hash_table - - this%hash_table => hash_table - this%index = 0 - nullify(this%table_entry) - do - this%index = this%index + 1 - if (associated(hash_table%table(this%index)%entry_value)) then - exit - else if (this%index > hash_table%table_size) then - this%index = 0 - end if - end do - end subroutine hash_iterator_initialize - - !####################################################################### - - function hash_iterator_key(this) result(key) - ! Return the key for this hash iterator entry - - ! Dummy arguments - class(ccpp_hash_iterator_t) :: this - character(len=:), allocatable :: key - - if (this%valid()) then - if (associated(this%table_entry)) then - key = this%table_entry%entry_value%key() - else - key = this%hash_table%table(this%index)%entry_value%key() - end if - else - key = '' - end if - - end function hash_iterator_key - - !####################################################################### - - subroutine hash_iterator_next_entry(this) - ! Set the iterator to the next valid hash table value - - ! Dummy argument - class(ccpp_hash_iterator_t) :: this - ! Local variable - logical :: has_table_entry - logical :: has_table_next - - if (this%index > 0) then - ! We have initialized this table, so look for next entry - has_table_entry = associated(this%table_entry) - if (has_table_entry) then - has_table_next = associated(this%table_entry%next) - else - has_table_next = .false. - end if - if (has_table_next) then - this%table_entry => this%table_entry%next - else if ((.not. has_table_entry) .and. & - associated(this%hash_table%table(this%index)%next)) then - this%table_entry => this%hash_table%table(this%index)%next - else - do - if (this%index >= this%hash_table%table_size) then - this%index = 0 - nullify(this%table_entry) - exit - else - this%index = this%index + 1 - nullify(this%table_entry) - associate(t_entry => this%hash_table%table(this%index)) - if (associated(t_entry%entry_value)) then - exit - end if - end associate - end if - end do - end if - else - ! This is an invalid iterator state - nullify(this%table_entry) - end if - - end subroutine hash_iterator_next_entry - - !####################################################################### - - logical function hash_iterator_is_valid(this) result(valid) - ! Return .true. iff this iterator is in a valid (active entry) state - - ! Dummy arguments - class(ccpp_hash_iterator_t) :: this - - valid = .false. - if ((this%index > 0) .and. & - (this%index <= this%hash_table%table_size)) then - valid = .true. - end if - - end function hash_iterator_is_valid - - !####################################################################### - - function hash_iterator_value(this) result(val) - ! Return the value or this hash iterator entry - - ! Dummy arguments - class(ccpp_hash_iterator_t) :: this - class(ccpp_hashable_t), pointer :: val - - if (this%valid()) then - if (associated(this%table_entry)) then - val => this%table_entry%entry_value - else - val => this%hash_table%table(this%index)%entry_value - end if - else - nullify(val) - end if - - end function hash_iterator_value - -end module ccpp_hash_table diff --git a/src/ccpp_hashable.F90 b/src/ccpp_hashable.F90 deleted file mode 100644 index 21a70902..00000000 --- a/src/ccpp_hashable.F90 +++ /dev/null @@ -1,98 +0,0 @@ -module ccpp_hashable - - implicit none - private - - ! Public interfaces - public :: new_hashable_char - public :: new_hashable_int - - type, abstract, public :: ccpp_hashable_t - ! The hashable type is a base type that contains a hash key. - contains - procedure(ccpp_hashable_get_key), deferred :: key - end type ccpp_hashable_t - - type, public, extends(ccpp_hashable_t) :: ccpp_hashable_char_t - character(len=:), private, allocatable :: name - contains - procedure :: key => ccpp_hashable_char_get_key - end type ccpp_hashable_char_t - - type, public, extends(ccpp_hashable_t) :: ccpp_hashable_int_t - integer, private :: value - contains - procedure :: key => ccpp_hashable_int_get_key - procedure :: val => ccpp_hashable_int_get_val - end type ccpp_hashable_int_t - - ! Abstract interface for key procedure of ccpp_hashable_t class - abstract interface - function ccpp_hashable_get_key(hashable) - import :: ccpp_hashable_t - class(ccpp_hashable_t), intent(in) :: hashable - character(len=:), allocatable :: ccpp_hashable_get_key - end function ccpp_hashable_get_key - end interface - -contains - - !####################################################################### - - subroutine new_hashable_char(name_in, new_obj) - character(len=*), intent(in) :: name_in - type(ccpp_hashable_char_t), pointer :: new_obj - - if (associated(new_obj)) then - deallocate(new_obj) - end if - allocate(new_obj) - new_obj%name = name_in - end subroutine new_hashable_char - - !####################################################################### - - function ccpp_hashable_char_get_key(hashable) - ! Return the hashable char class key (name) - class(ccpp_hashable_char_t), intent(in) :: hashable - character(len=:), allocatable :: ccpp_hashable_char_get_key - - ccpp_hashable_char_get_key = hashable%name - end function ccpp_hashable_char_get_key - - !####################################################################### - - subroutine new_hashable_int(val_in, new_obj) - integer, intent(in) :: val_in - type(ccpp_hashable_int_t), pointer :: new_obj - - if (associated(new_obj)) then - deallocate(new_obj) - end if - allocate(new_obj) - new_obj%value = val_in - end subroutine new_hashable_int - - !####################################################################### - - function ccpp_hashable_int_get_key(hashable) - ! Return the hashable int class key (value ==> string) - class(ccpp_hashable_int_t), intent(in) :: hashable - character(len=:), allocatable :: ccpp_hashable_int_get_key - - character(len=32) :: key_str - - write(key_str, '(i0)') hashable%val() - ccpp_hashable_int_get_key = trim(key_str) - end function ccpp_hashable_int_get_key - - !####################################################################### - - integer function ccpp_hashable_int_get_val(hashable) - ! Return the hashable int class value - class(ccpp_hashable_int_t), intent(in) :: hashable - - ccpp_hashable_int_get_val = hashable%value - end function ccpp_hashable_int_get_val - -end module ccpp_hashable diff --git a/src/ccpp_scheme_utils.F90 b/src/ccpp_scheme_utils.F90 deleted file mode 100644 index d4de6499..00000000 --- a/src/ccpp_scheme_utils.F90 +++ /dev/null @@ -1,122 +0,0 @@ -module ccpp_scheme_utils - - ! Module of utilities available to CCPP schemes - - use ccpp_constituent_prop_mod, only: ccpp_model_constituents_t, & - int_unassigned - - implicit none - private - - !! Public interfaces - public :: ccpp_initialize_constituent_ptr ! Used by framework to initialize - public :: ccpp_constituent_index ! Lookup index constituent by name - public :: ccpp_constituent_indices ! Lookup indices of consitutents by name - - !! Private module variables & interfaces - - ! initialized set to .true. once hash table pointer is initialized - logical :: initialized = .false. - type(ccpp_model_constituents_t), pointer :: constituent_obj => null() - - private :: check_initialization - private :: status_ok - -contains - - subroutine check_initialization(caller, errcode, errmsg) - ! Dummy arguments - character(len=*), intent(in) :: caller - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - if (initialized) then - if (present(errcode)) then - errcode = 0 - end if - if (present(errmsg)) then - errmsg = '' - end if - else - if (present(errcode)) then - errcode = 1 - end if - if (present(errmsg)) then - errmsg = trim(caller) // ' FAILED, module not initialized' - end if - end if - end subroutine check_initialization - - logical function status_ok(errcode) - ! Dummy argument - integer, optional, intent(in) :: errcode - - if (present(errcode)) then - status_ok = (errcode == 0) .and. initialized - else - status_ok = initialized - end if - - end function status_ok - - subroutine ccpp_initialize_constituent_ptr(const_obj) - ! Dummy arguments - type(ccpp_model_constituents_t), pointer, intent(in) :: const_obj - - if (.not. initialized) then - constituent_obj => const_obj - initialized = .true. - end if - end subroutine ccpp_initialize_constituent_ptr - - subroutine ccpp_constituent_index(standard_name, const_index, errcode, errmsg) - ! Dummy arguments - character(len=*), intent(in) :: standard_name - integer, intent(out) :: const_index - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - ! Local variable - character(len=*), parameter :: subname = 'ccpp_constituent_index' - - call check_initialization(caller=subname, errcode=errcode, errmsg=errmsg) - if (status_ok(errcode)) then - call constituent_obj%const_index(const_index, standard_name, & - errcode, errmsg) - else - const_index = int_unassigned - end if - end subroutine ccpp_constituent_index - - subroutine ccpp_constituent_indices(standard_names, const_inds, errcode, errmsg) - ! Dummy arguments - character(len=*), intent(in) :: standard_names(:) - integer, intent(out) :: const_inds(:) - integer, optional, intent(out) :: errcode - character(len=*), optional, intent(out) :: errmsg - - ! Local variables - integer :: indx - character(len=*), parameter :: subname = 'ccpp_constituent_indices' - - const_inds = int_unassigned - call check_initialization(caller=subname, errcode=errcode, errmsg=errmsg) - if (status_ok(errcode)) then - if (size(const_inds) < size(standard_names)) then - errcode = 1 - write(errmsg, '(3a)') subname, ": const_inds array too small. ", & - "Must be greater than or equal to the size of standard_names" - else - do indx = 1, size(standard_names) - ! For each std name in , find the const. index - call constituent_obj%const_index(const_inds(indx), & - standard_names(indx), errcode, errmsg) - if (errcode /= 0) then - exit - end if - end do - end if - end if - end subroutine ccpp_constituent_indices - -end module ccpp_scheme_utils diff --git a/src/ccpp_types.F90 b/src/ccpp_types.F90 deleted file mode 100644 index cdc1d655..00000000 --- a/src/ccpp_types.F90 +++ /dev/null @@ -1,92 +0,0 @@ -! -! This work (Common Community Physics Package), identified by NOAA, NCAR, -! CU/CIRES, is free of known copyright restrictions and is placed in the -! public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -! - -!> -!! @brief Type definitions module. -!! -!! @details The types module provides definitions for -!! atmospheric driver to call the CCPP. -! -module ccpp_types - - use mpi_f08, only: mpi_comm - - !! \section arg_table_ccpp_types - !! \htmlinclude ccpp_types.html - !! - - implicit none - - private - public :: ccpp_t, one - public :: mpi_comm - - !> @var Definition of constant one - integer, parameter :: one = 1 - - !> @var The default loop counter indicating outside of a subcycle loop - integer, parameter :: ccpp_default_loop_cnt = -999 - integer, parameter :: ccpp_default_loop_max = -999 - - !> @var The default values for block, chunk and thread numbers indicating invalid data - integer, parameter :: ccpp_default_block_number = -999 - integer, parameter :: ccpp_default_chunk_number = -999 - integer, parameter :: ccpp_default_thread_number = -999 - - !> @var The default maximum number of threads for CCPP - integer, parameter :: ccpp_default_thread_count = -999 - - !! \section arg_table_ccpp_t - !! \htmlinclude ccpp_t.html - !! - !> - !! @brief CCPP physics type. - !! - !! Generic type that contains all components to run the CCPP. - !! - !! - Array of fields to all the data needing to go - !! the physics drivers. - !! - The suite definitions in a ccpp_suite_t type. - ! - type :: ccpp_t - ! CCPP-internal variables for physics schemes - integer :: errflg = 0 - character(len=512) :: errmsg = '' - integer :: loop_cnt = ccpp_default_loop_cnt - integer :: loop_max = ccpp_default_loop_max - integer :: blk_no = ccpp_default_block_number - integer :: chunk_no = ccpp_default_chunk_number - integer :: thrd_no = ccpp_default_thread_number - integer :: thrd_cnt = ccpp_default_thread_count - integer :: ccpp_instance = 1 - - contains - - procedure :: initialized => ccpp_t_initialized - - end type ccpp_t - -contains - - function ccpp_t_initialized(ccpp_d) result(initialized) - implicit none - ! - class(ccpp_t) :: ccpp_d - logical :: initialized - ! - initialized = ccpp_d%thrd_no /= ccpp_default_thread_number .or. & - ccpp_d%blk_no /= ccpp_default_block_number .or. & - ccpp_d%chunk_no /= ccpp_default_chunk_number - end function ccpp_t_initialized - -end module ccpp_types diff --git a/src/ccpp_types.meta b/src/ccpp_types.meta deleted file mode 100644 index cdec1dc2..00000000 --- a/src/ccpp_types.meta +++ /dev/null @@ -1,103 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ccpp_t - type = ddt - dependencies = - -[ccpp-arg-table] - name = ccpp_t - type = ddt -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 -[loop_cnt] - standard_name = ccpp_loop_counter - long_name = loop counter for subcycling loops in CCPP - units = index - dimensions = () - type = integer -[loop_max] - standard_name = ccpp_loop_extent - long_name = loop extent for subcycling loops in CCPP - units = count - dimensions = () - type = integer -[blk_no] - standard_name = ccpp_block_number - long_name = number of block for explicit data blocking in CCPP - units = index - dimensions = () - type = integer -[chunk_no] - standard_name = ccpp_chunk_number - long_name = number of chunk for using array chunks in CCPP - units = index - dimensions = () - type = integer -[thrd_no] - standard_name = ccpp_thread_number - long_name = number of thread for threading in CCPP - units = index - dimensions = () - type = integer -[thrd_cnt] - standard_name = ccpp_thread_count - long_name = total number of threads for threading in CCPP - units = index - dimensions = () - type = integer -[ccpp_instance] - standard_name = ccpp_instance - long_name = ccpp_instance - units = index - dimensions = () - type = integer - -######################################################################## -[ccpp-table-properties] - name = MPI_Comm - type = ddt - dependencies = - -[ccpp-arg-table] - name = MPI_Comm - type = ddt - -######################################################################## - -[ccpp-table-properties] - name = ccpp_types - type = module - dependencies = - -[ccpp-arg-table] - name = ccpp_types - type = module -[ccpp_t] - standard_name = ccpp_t - long_name = definition of type ccpp_t - units = DDT - dimensions = () - type = ccpp_t -[one] - standard_name = ccpp_constant_one - long_name = definition of constant one - units = 1 - dimensions = () - type = integer -[MPI_Comm] - standard_name = MPI_Comm - long_name = definition of type MPI_Comm - units = DDT - dimensions = () - type = MPI_Comm diff --git a/test/.pylintrc b/test/.pylintrc deleted file mode 100644 index b380843f..00000000 --- a/test/.pylintrc +++ /dev/null @@ -1,466 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=C0330,too-many-lines,too-many-public-methods,too-many-locals,too-many-arguments,too-many-instance-attributes,unnecessary-pass - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=15 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=optparse.Values,sys.exit - - -[BASIC] - -# Naming style matching correct argument names -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style -#argument-rgx= - -# Naming style matching correct attribute names -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo, - bar, - baz, - qux, - toto, - tutu, - tata - -# Naming style matching correct class attribute names -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style -#class-attribute-rgx= - -# Naming style matching correct class names -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming-style -#class-rgx= - -# Naming style matching correct constant names -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma -good-names=_ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming style matching correct inline iteration names -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style -#inlinevar-rgx= - -# Naming style matching correct method names -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style -#method-rgx= - -# Naming style matching correct module names -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style -#variable-rgx= - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=(^\s*(# )??$)|(^\s*>>> .*$)||(^\s*CCPPError:) - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=0 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module -max-module-lines=2000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,io,builtins - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=10 - -# Maximum number of attributes for a class (see R0902). -max-attributes=10 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=35 - -# Maximum number of locals for function / method body -max-locals=25 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=150 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub, - TERMIOS, - Bastion, - rexec - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt deleted file mode 100644 index 342ddad8..00000000 --- a/test/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -add_subdirectory(utils) - -if(CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_RUN_ADVECTION_TEST) - add_subdirectory(advection_test) -endif() -if(CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_RUN_CAPGEN_TEST) - add_subdirectory(capgen_test) -endif() -if(CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_RUN_DDT_HOST_TEST) - add_subdirectory(ddthost_test) -endif() -if(CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_RUN_VAR_COMPATIBILITY_TEST) - add_subdirectory(var_compatibility_test) -endif() -if(CCPP_FRAMEWORK_ENABLE_TESTS OR CCPP_RUN_NESTED_SUITE_TEST) - add_subdirectory(nested_suite_test) -endif() diff --git a/test/README.md b/test/README.md deleted file mode 100644 index 03363532..00000000 --- a/test/README.md +++ /dev/null @@ -1,75 +0,0 @@ -# Testing - -## Unit tests -To run the Python based unit tests, see the associated documentation in the `unit_tests` directory. - -## Doc tests -The Python source code has a wide range of doctests that can be used to verify implementation details quickly. To run the Python based doc tests, run: -```bash -$ export PYTHONPATH=/scripts:/scripts/parse_tools -$ pytest -v /scripts/ --doctest-modules -``` - -## Regression tests -The run the regression tests with mock host models, build the main project with your test option: - -```bash -$ cmake --fresh -S -B ... -$ cd -$ make -$ ctest -``` - -For example, to run all of the regression tests from the root of the project, you can use: -```bash -cmake --fresh -B./build -S./ -DCCPP_FRAMEWORK_ENABLE_TESTS=ON -``` - -Currently (as of July 2025), if everything works as expected, you should see something like: -``` -Test project - Start 1: ctest_advection_host_integration -1/4 Test #1: ctest_advection_host_integration ........... Passed 0.01 sec - Start 2: ctest_capgen_host_integration -2/4 Test #2: ctest_capgen_host_integration .............. Passed 0.01 sec - Start 3: ctest_ddt_host_integration -3/4 Test #3: ctest_ddt_host_integration ................. Passed 0.01 sec - Start 4: ctest_var_compatibility_host_integration -4/4 Test #4: ctest_var_compatibility_host_integration ... Passed 0.02 sec - -100% tests passed, 0 tests failed out of 4 - -Total Test time (real) = 0.06 sec -``` - -There are several `...` to enable tests: - -1) `-DCCPP_FRAMEWORK_ENABLE_TESTS=ON` Turns on all regression tests. -2) `-DCCPP_RUN_ADVECTION_TEST=ON` Turns on only the advection test -3) `-DCCPP_RUN_CAPGEN_TEST=ON` Turns on only the capgen test -4) `-DCCPP_RUN_DDT_HOST_TEST=ON` Turns on only the ddt host test -5) `-DCCPP_RUN_VAR_COMPATIBILITY_TEST=ON` Turns on only the variable compatibility test - -By default, the tests will build in debug mode. To enable release mode, you will need to set the build type: `-DCMAKE_BUILD_TYPE=Release` (or if you want release with debug symbols: `-DCMAKE_BUILD_TYPE=RelWithDebInfo`). - -To enable more verbose output for `ccpp_capgen.py`, add `-DCCPP_VERBOSITY=` to the `cmake` command line arguments where `n={1,2,3}` (`n=0` or no verbosity by default). - -If needed, the generated caps will be in `/test//ccpp`. - -### Python regression test interface - -There is a matching Python based API for each regression test. To run the corresponding python tests, build the framework using the build process from above and then you can run: - -```bash -BUILD_DIR= \ -PYTHONPATH=/test/:/scripts/ \ - pytest \ - /test/capgen_test/capgen_test_reports.py \ - /test/advection_test/advection_test_reports.py \ - /test/ddthost_test/ddthost_test_reports.py \ - /test/var_compatibility_test/var_compatibility_test_reports.py -``` - -You may run tests individually instead of all tests as your use case needs. - -Please see each test directory for more information on that specific test. diff --git a/test/advection_test/CMakeLists.txt b/test/advection_test/CMakeLists.txt deleted file mode 100644 index 4c20835b..00000000 --- a/test/advection_test/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES -set(SCHEME_FILES "cld_liq" "cld_ice" "apply_constituent_tendencies" "const_indices") -set(SCHEME_FILES_ERROR "cld_liq" "cld_ice" "dlc_liq") -set(HOST_FILES "test_host_data" "test_host_mod") -set(SUITE_FILES "cld_suite.xml") -set(SUITE_FILES_ERROR "cld_suite_error.xml") -# HOST is the name of the executable we will build. -set(HOST "test_host") - -# By default, generated caps go in this test specific ccpp subdir -set(CCPP_CAP_FILES "${CMAKE_CURRENT_BINARY_DIR}/ccpp") - -# Create lists for Fortran and meta data files from file names -list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) -list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_META_FILES) -list(TRANSFORM SCHEME_FILES_ERROR APPEND ".F90" OUTPUT_VARIABLE SCHEME_ERROR_FORTRAN_FILES) -list(TRANSFORM SCHEME_FILES_ERROR APPEND ".meta" OUTPUT_VARIABLE SCHEME_ERROR_META_FILES) -list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE ADVECTION_HOST_FORTRAN_FILES) -list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE ADVECTION_HOST_METADATA_FILES) - -list(APPEND ADVECTION_HOST_METADATA_FILES "${HOST}.meta") - -# Run ccpp_capgen that we expect to fail -ccpp_capgen(CAPGEN_EXPECT_THROW_ERROR ON - VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${ADVECTION_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_ERROR_META_FILES} - SUITES ${SUITE_FILES_ERROR} - HOST_NAME ${HOST} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Run ccpp_capgen -ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${ADVECTION_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_META_FILES} - SUITES ${SUITE_FILES} - HOST_NAME ${HOST} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Retrieve the list of Fortran files required for test host from datatable.xml and set to CCPP_CAPS_LIST -ccpp_datafile(DATATABLE "${CCPP_CAP_FILES}/datatable.xml" - REPORT_NAME "--ccpp-files") - -# Create test host library -add_library(ADVECTION_TESTLIB OBJECT ${SCHEME_FORTRAN_FILES} - ${ADVECTION_HOST_FORTRAN_FILES} - ${CCPP_CAPS_LIST}) - -# Setup test executable with needed dependencies -add_executable(advection_host_integration test_advection_host_integration.F90 ${HOST}.F90) -target_link_libraries(advection_host_integration PRIVATE ADVECTION_TESTLIB test_utils) -target_include_directories(advection_host_integration PRIVATE "$") - -# Add executable to be called with ctest -add_test(NAME ctest_advection_host_integration COMMAND advection_host_integration) diff --git a/test/advection_test/README.md b/test/advection_test/README.md deleted file mode 100644 index 6dd53e9f..00000000 --- a/test/advection_test/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Advection Test - -Contains tests to exercise the capabilities of the constituents object, including: -- Adding a build-time constituent via metadata property -- Adding a run-time constituent via a register phase - - Also tests that trying to add a constituent outside of the register phase errors as expected -- Passing around and modifying the constituent array -- Accessing and modifying a constituent tendency variable -- Passing around the constituent tendency array -- Dimensions are case-insensitive - -## Building/Running - -To explicitly build/run the advection test host, run: - -```bash -$ cmake --fresh -S -B -DCCPP_RUN_ADVECTION_TEST=ON -$ cd -$ make -$ ctest -``` diff --git a/test/advection_test/advection_test_reports.py b/test/advection_test/advection_test_reports.py deleted file mode 100644 index 4fbe8e68..00000000 --- a/test/advection_test/advection_test_reports.py +++ /dev/null @@ -1,127 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Test advection database report python interface - - Assumptions: - - Command line arguments: build_dir database_filepath - - Usage: python test_reports ------------------------------------------------------------------------ -""" -import os -import unittest - -from test_stub import BaseTests - -_BUILD_DIR = os.path.join(os.path.abspath(os.environ['BUILD_DIR']), "test", "advection_test") -_DATABASE = os.path.abspath(os.path.join(_BUILD_DIR, "ccpp", "datatable.xml")) - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_FRAMEWORK_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, os.pardir)) -_SCRIPTS_DIR = os.path.abspath(os.path.join(_FRAMEWORK_DIR, "scripts")) - -# Check data -_HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] -_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_cld_suite_cap.F90")] -_UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), - os.path.join(_FRAMEWORK_DIR, "src", - "ccpp_constituent_prop_mod.F90"), - os.path.join(_FRAMEWORK_DIR, "src", - "ccpp_scheme_utils.F90"), - os.path.join(_FRAMEWORK_DIR, "src", "ccpp_hashable.F90"), - os.path.join(_FRAMEWORK_DIR, "src", "ccpp_hash_table.F90")] -_CCPP_FILES = _UTILITY_FILES + _HOST_FILES + _SUITE_FILES -_DEPENDENCIES = [""] -_PROCESS_LIST = [""] -_MODULE_LIST = ["cld_ice", "cld_liq", "const_indices", "apply_constituent_tendencies"] -_SUITE_LIST = ["cld_suite"] -_REQUIRED_VARS_CLD = ["ccpp_error_code", "ccpp_error_message", - "horizontal_loop_begin", "horizontal_loop_end", - "surface_air_pressure", "temperature", - "tendency_of_cloud_liquid_dry_mixing_ratio", - "time_step_for_physics", "water_temperature_at_freezing", - "water_vapor_specific_humidity", - "cloud_ice_dry_mixing_ratio", - "cloud_liquid_dry_mixing_ratio", - "ccpp_constituents", - "ccpp_constituent_tendencies", - "number_of_ccpp_constituents", - "dynamic_constituents_for_cld_ice", - "dynamic_constituents_for_cld_liq", - "test_banana_constituent_indices", "test_banana_name", - "banana_array_dim", - "test_banana_name_array", - "test_banana_constituent_index", - # Added by --debug option - "horizontal_dimension", - "vertical_layer_dimension"] -_INPUT_VARS_CLD = ["surface_air_pressure", "temperature", - "horizontal_loop_begin", "horizontal_loop_end", - "time_step_for_physics", "water_temperature_at_freezing", - "water_vapor_specific_humidity", - "cloud_ice_dry_mixing_ratio", - "cloud_liquid_dry_mixing_ratio", - "tendency_of_cloud_liquid_dry_mixing_ratio", - "ccpp_constituents", - "ccpp_constituent_tendencies", - "number_of_ccpp_constituents", - "banana_array_dim", - "test_banana_name_array", "test_banana_name", - # Added by --debug option - "horizontal_dimension", - "vertical_layer_dimension"] -_OUTPUT_VARS_CLD = ["ccpp_error_code", "ccpp_error_message", - "water_vapor_specific_humidity", "temperature", - "tendency_of_cloud_liquid_dry_mixing_ratio", - "cloud_ice_dry_mixing_ratio", - "ccpp_constituents", - "ccpp_constituent_tendencies", - "cloud_liquid_dry_mixing_ratio", - "dynamic_constituents_for_cld_ice", - "dynamic_constituents_for_cld_liq", - "dynamic_constituents_for_cld_liq", - "test_banana_constituent_indices", - "test_banana_constituent_index"] - - -class TestAdvectionHostDataTables(unittest.TestCase, BaseTests.TestHostDataTables): - database = _DATABASE - host_files = _HOST_FILES - suite_files = _SUITE_FILES - utility_files = _UTILITY_FILES - ccpp_files = _CCPP_FILES - process_list = _PROCESS_LIST - module_list = _MODULE_LIST - dependencies = _DEPENDENCIES - suite_list = _SUITE_LIST - -class CommandLineAdvectionHostDatafileRequiredFiles(unittest.TestCase, BaseTests.TestHostCommandLineDataFiles): - database = _DATABASE - host_files = _HOST_FILES - suite_files = _SUITE_FILES - utility_files = _UTILITY_FILES - ccpp_files = _CCPP_FILES - process_list = _PROCESS_LIST - module_list = _MODULE_LIST - dependencies = _DEPENDENCIES - suite_list = _SUITE_LIST - datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" - - -class TestCapgenCldSuite(unittest.TestCase, BaseTests.TestSuite): - database = _DATABASE - required_vars = _REQUIRED_VARS_CLD - input_vars = _INPUT_VARS_CLD - output_vars = _OUTPUT_VARS_CLD - suite_name = "cld_suite" - - -class CommandLineCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuiteCommandLine): - database = _DATABASE - required_vars = _REQUIRED_VARS_CLD - input_vars = _INPUT_VARS_CLD - output_vars = _OUTPUT_VARS_CLD - suite_name = "cld_suite" - datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" diff --git a/test/advection_test/apply_constituent_tendencies.F90 b/test/advection_test/apply_constituent_tendencies.F90 deleted file mode 100644 index 63a1881c..00000000 --- a/test/advection_test/apply_constituent_tendencies.F90 +++ /dev/null @@ -1,39 +0,0 @@ -module apply_constituent_tendencies - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: apply_constituent_tendencies_run - -contains - - !> \section arg_table_apply_constituent_tendencies_run Argument Table - !!! \htmlinclude apply_constituent_tendencies_run.html - subroutine apply_constituent_tendencies_run(const_tend, const, errcode, errmsg) - ! Dummy arguments - real(kind=kind_phys), intent(inout) :: const_tend(:, :, :) ! constituent tendency array - real(kind=kind_phys), intent(inout) :: const(:, :, :) ! constituent state array - integer, intent(out) :: errcode - character(len=512), intent(out) :: errmsg - - ! Local variables - integer :: klev, jcnst, icol - - errcode = 0 - errmsg = '' - - do icol = 1, size(const_tend, 1) - do klev = 1, size(const_tend, 2) - do jcnst = 1, size(const_tend, 3) - const(icol, klev, jcnst) = const(icol, klev, jcnst) + const_tend(icol, klev, jcnst) - end do - end do - end do - - const_tend = 0._kind_phys - - end subroutine apply_constituent_tendencies_run - -end module apply_constituent_tendencies diff --git a/test/advection_test/apply_constituent_tendencies.meta b/test/advection_test/apply_constituent_tendencies.meta deleted file mode 100644 index b7645a1b..00000000 --- a/test/advection_test/apply_constituent_tendencies.meta +++ /dev/null @@ -1,36 +0,0 @@ -##################################################################### -[ccpp-table-properties] - name = apply_constituent_tendencies - type = scheme -[ccpp-arg-table] - name = apply_constituent_tendencies_run - type = scheme -[ const_tend ] - standard_name = ccpp_constituent_tendencies - long_name = ccpp constituent tendencies - units = none - type = real | kind = kind_phys - dimensions = (horizontal_loop_extent, vertical_layer_dimension, number_of_ccpp_constituents) - intent = inout -[ const ] - standard_name = ccpp_constituents - long_name = ccpp constituents - units = none - type = real | kind = kind_phys - dimensions = (horizontal_loop_extent, vertical_layer_dimension, number_of_ccpp_constituents) - intent = inout -[ errcode ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - type = integer - dimensions = () - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - type = character | kind = len=512 - dimensions = () - intent = out -######################################################### diff --git a/test/advection_test/cld_ice.F90 b/test/advection_test/cld_ice.F90 deleted file mode 100644 index 3ace2f91..00000000 --- a/test/advection_test/cld_ice.F90 +++ /dev/null @@ -1,127 +0,0 @@ -! Test parameterization with advected species -! - -module cld_ice - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: cld_ice_register - public :: cld_ice_init - public :: cld_ice_run - public :: cld_ice_final - - real(kind=kind_phys), private :: tcld = huge(1.0_kind_phys) - -contains - - !> \section arg_table_cld_ice_register Argument Table - !! \htmlinclude arg_table_cld_ice_register.html - !! - subroutine cld_ice_register(dyn_const_ice, errmsg, errcode) - use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t - type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const_ice(:) - integer, intent(out) :: errcode - character(len=512), intent(out) :: errmsg - - errmsg = '' - errcode = 0 - allocate(dyn_const_ice(2), stat=errcode) - if (errcode /= 0) then - errmsg = 'Error allocating dyn_const in cld_ice_dynamic_constituents' - return - end if - call dyn_const_ice(1)%instantiate(std_name='dyn_const1', long_name='dyn const1', & - diag_name='DYNCONST1', units='kg kg-1', default_value=0._kind_phys, & - vertical_dim='vertical_layer_dimension', advected=.true., & - min_value=1000._kind_phys, water_species=.true., mixing_ratio_type='wet', & - errcode=errcode, errmsg=errmsg) - call dyn_const_ice(2)%instantiate(std_name='dyn_const2_wrt_moist_air', long_name='dyn const2', & - diag_name='DYNCONST2', units='kg kg-1', default_value=0._kind_phys, & - vertical_dim='vertical_layer_dimension', advected=.true., & - water_species=.false., errcode=errcode, errmsg=errmsg) - - end subroutine cld_ice_register - - !> \section arg_table_cld_ice_run Argument Table - !! \htmlinclude arg_table_cld_ice_run.html - !! - subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & - errmsg, errflg) - - integer, intent(in) :: ncol - real(kind=kind_phys), intent(in) :: timestep - real(kind=kind_phys), intent(inout) :: temp(:, :) - real(kind=kind_phys), intent(inout) :: qv(:, :) - real(kind=kind_phys), intent(in) :: ps(:) - real(kind=kind_phys), intent(inout) :: cld_ice_array(:, :) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: icol - integer :: ilev - real(kind=kind_phys) :: frz - - errmsg = '' - errflg = 0 - - ! Apply state-of-the-art thermodynamics :) - do icol = 1, ncol - do ilev = 1, size(temp, 2) - if (temp(icol, ilev) < tcld) then - frz = max(qv(icol, ilev) - 0.5_kind_phys, 0.0_kind_phys) - cld_ice_array(icol, ilev) = cld_ice_array(icol, ilev) + frz - qv(icol, ilev) = qv(icol, ilev) - frz - if (frz > 0.0_kind_phys) then - temp(icol, ilev) = temp(icol, ilev) + 1.0_kind_phys - end if - end if - end do - end do - - end subroutine cld_ice_run - - !> \section arg_table_cld_ice_init Argument Table - !! \htmlinclude arg_table_cld_ice_init.html - !! - subroutine cld_ice_init(tfreeze, cld_ice_array, errmsg, errflg) - - real(kind=kind_phys), intent(in) :: tfreeze - real(kind=kind_phys), intent(inout) :: cld_ice_array(:, :) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - cld_ice_array = 0.0_kind_phys - tcld = tfreeze - 20.0_kind_phys - - end subroutine cld_ice_init - - !> \section arg_table_cld_ice_final Argument Table - !! \htmlinclude arg_table_cld_ice_final.html - !! - - !> @{ - !! This routine does nothing, but it tests if blank - !! lines and doxygen comments between metadata hooks - !! and the subroutine are parsed correctly. - !! @{ - - subroutine cld_ice_final(errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - end subroutine cld_ice_final - - !! @} - !! @} - -end module cld_ice diff --git a/test/advection_test/cld_ice.meta b/test/advection_test/cld_ice.meta deleted file mode 100644 index e57d0b08..00000000 --- a/test/advection_test/cld_ice.meta +++ /dev/null @@ -1,143 +0,0 @@ -# cld_ice is a scheme that produces a cloud ice amount -[ccpp-table-properties] - name = cld_ice - type = scheme -[ccpp-arg-table] - name = cld_ice_register - type = scheme -[ dyn_const_ice ] - standard_name = dynamic_constituents_for_cld_ice - units = none - dimensions = (:) - allocatable = True - type = ccpp_constituent_properties_t - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errcode ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = cld_ice_run - type = scheme -[ ncol ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp ] - standard_name = temperature - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = in -[ cld_ice_array ] - standard_name = cloud_ice_dry_mixing_ratio - advected = .true. - default_value = 0.0_kind_phys - units = kg kg-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real | kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = cld_ice_init - type = scheme -[ tfreeze ] - standard_name = water_temperature_at_freezing - long_name = Freezing temperature of water at sea level - units = K - dimensions = () - type = real | kind = kind_phys - intent = in -[ cld_ice_array ] - standard_name = cloud_ice_dry_mixing_ratio - advected = .true. - default_value = 0.0_kind_phys - units = kg kg-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys - # Advected species that needs to be supplied by framework - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = cld_ice_final - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/advection_test/cld_liq.F90 b/test/advection_test/cld_liq.F90 deleted file mode 100644 index cb02cf11..00000000 --- a/test/advection_test/cld_liq.F90 +++ /dev/null @@ -1,102 +0,0 @@ -! Test parameterization with advected species -! - -module cld_liq - - use ccpp_kinds, only: kind_phys - use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t - - implicit none - private - - public :: cld_liq_register - public :: cld_liq_init - public :: cld_liq_run - -contains - - !> \section arg_table_cld_liq_register Argument Table - !! \htmlinclude arg_table_cld_liq_register.html - !! - subroutine cld_liq_register(dyn_const, errmsg, errflg) - type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - allocate(dyn_const(1), stat=errflg) - if (errflg /= 0) then - errmsg = 'Error allocating dyn_const in cld_liq_register' - return - end if - call dyn_const(1)%instantiate(std_name="dyn_const3_wrt_moist_air_and_condensed_water", long_name='dyn const3', & - diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & - vertical_dim='vertical_layer_dimension', advected=.true., & - water_species=.true., mixing_ratio_type='dry', & - errcode=errflg, errmsg=errmsg) - - end subroutine cld_liq_register - - !> \section arg_table_cld_liq_run Argument Table - !! \htmlinclude arg_table_cld_liq_run.html - !! - subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & - cld_liq_tend, errmsg, errflg) - - integer, intent(in) :: ncol - real(kind=kind_phys), intent(in) :: timestep - real(kind=kind_phys), intent(in) :: tcld - real(kind=kind_phys), intent(inout) :: temp(:, :) - real(kind=kind_phys), intent(inout) :: qv(:, :) - real(kind=kind_phys), intent(in) :: ps(:) - real(kind=kind_phys), intent(inout) :: cld_liq_tend(:, :) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: icol - integer :: ilev - real(kind=kind_phys) :: cond - - errmsg = '' - errflg = 0 - - ! Apply state-of-the-art thermodynamics :) - do icol = 1, ncol - do ilev = 1, size(temp, 2) - if ((qv(icol, ilev) > 0.0_kind_phys) .and. & - (temp(icol, ilev) <= tcld)) then - cond = min(qv(icol, ilev), 0.1_kind_phys) - cld_liq_tend(icol, ilev) = cond - qv(icol, ilev) = qv(icol, ilev) - cond - if (cond > 0.0_kind_phys) then - temp(icol, ilev) = temp(icol, ilev) + (cond * 5.0_kind_phys) - end if - end if - end do - end do - - end subroutine cld_liq_run - - !> \section arg_table_cld_liq_init Argument Table - !! \htmlinclude arg_table_cld_liq_init.html - !! - subroutine cld_liq_init(tfreeze, cld_liq_array, tcld, errmsg, errflg) - - real(kind=kind_phys), intent(in) :: tfreeze - real(kind=kind_phys), intent(out) :: cld_liq_array(:, :) - real(kind=kind_phys), intent(out) :: tcld - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - cld_liq_array = 0.0_kind_phys - tcld = tfreeze - 20.0_kind_phys - - end subroutine cld_liq_init - -end module cld_liq diff --git a/test/advection_test/cld_liq.meta b/test/advection_test/cld_liq.meta deleted file mode 100644 index b3ef3a0d..00000000 --- a/test/advection_test/cld_liq.meta +++ /dev/null @@ -1,135 +0,0 @@ -# cld_liq is a scheme that produces a cloud liquid amount -[ccpp-table-properties] - name = cld_liq - type = scheme -[ccpp-arg-table] - name = cld_liq_register - type = scheme -[ dyn_const ] - standard_name = dynamic_constituents_for_cld_liq - dimensions = (:) - type = ccpp_constituent_properties_t - intent = out - allocatable = true -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = cld_liq_run - type = scheme -[ ncol ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ tcld] - standard_name = minimum_temperature_for_cloud_liquid - units = K - dimensions = () - type = real | kind = kind_phys - intent = in -[ temp ] - standard_name = temperature - units = K - dimensions = (horizontal_loop_extent, vertical_LAYER_dimension) - type = real - kind = kind_phys - intent = inout -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = hPa - dimensions = (horizontal_loop_extent) - intent = in -[ cld_liq_tend ] - standard_name = tendency_of_cloud_liquid_dry_mixing_ratio - units = kg kg-1 s-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real | kind = kind_phys - intent = inout - constituent = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = cld_liq_init - type = scheme -[ tfreeze] - standard_name = water_temperature_at_freezing - long_name = Freezing temperature of water at sea level - units = K - dimensions = () - type = real | kind = kind_phys - intent = in -[ cld_liq_array ] - standard_name = cloud_liquid_dry_mixing_ratio - diagnostic_name = CLDLIQ - advected = .true. - units = kg kg-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys - # Advected species that needs to be promoted from suite. - intent = out -[ tcld] - standard_name = minimum_temperature_for_cloud_liquid - units = K - dimensions = () - type = real | kind = kind_phys - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/advection_test/cld_suite.xml b/test/advection_test/cld_suite.xml deleted file mode 100644 index fac613e8..00000000 --- a/test/advection_test/cld_suite.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - const_indices - cld_liq - apply_constituent_tendencies - cld_ice - apply_constituent_tendencies - - diff --git a/test/advection_test/cld_suite_error.xml b/test/advection_test/cld_suite_error.xml deleted file mode 100644 index 80acac91..00000000 --- a/test/advection_test/cld_suite_error.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - dlc_liq - cld_liq - cld_ice - - diff --git a/test/advection_test/const_indices.F90 b/test/advection_test/const_indices.F90 deleted file mode 100644 index bc3b46a7..00000000 --- a/test/advection_test/const_indices.F90 +++ /dev/null @@ -1,95 +0,0 @@ -! Test collection of constituent indices -! - -module const_indices - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: const_indices_init - public :: const_indices_run - -contains - - !> \section arg_table_const_indices_run Argument Table - !! \htmlinclude arg_table_const_indices_run.html - !! - subroutine const_indices_run(const_std_name, num_consts, test_stdname_array, & - const_index, const_inds, errmsg, errflg) - use ccpp_constituent_prop_mod, only: int_unassigned - use ccpp_scheme_utils, only: ccpp_constituent_index - use ccpp_scheme_utils, only: ccpp_constituent_indices - - character(len=*), intent(in) :: const_std_name - integer, intent(in) :: num_consts - character(len=*), intent(in) :: test_stdname_array(:) - integer, intent(out) :: const_index - integer, intent(out) :: const_inds(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: indx - integer :: test_indx - - errmsg = '' - errflg = 0 - - ! Find the constituent index for - call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) - if (errflg == 0) then - call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) - end if - ! Check that a non-registered constituent is detectable but - ! does not cause an error - if (errflg == 0) then - call ccpp_constituent_index('unobtainium', test_indx, errflg, errmsg) - if (test_indx /= int_unassigned) then - if (errflg == 0) then - ! Do not add an error if one is already reported - errflg = 2 - write(errmsg, '(2a,i0,a,i0)') "ccpp_constituent_index called for ", & - "'unobtainium' returned an index of ", test_indx, ", not ", & - int_unassigned - end if - end if - end if - - end subroutine const_indices_run - - !> \section arg_table_const_indices_init Argument Table - !! \htmlinclude arg_table_const_indices_init.html - !! - subroutine const_indices_init(const_std_name, num_consts, test_stdname_array, & - const_index, const_inds, errmsg, errflg) - use ccpp_scheme_utils, only: ccpp_constituent_index, & - ccpp_constituent_indices - - character(len=*), intent(in) :: const_std_name - integer, intent(in) :: num_consts - character(len=*), intent(in) :: test_stdname_array(:) - integer, intent(out) :: const_index - integer, intent(out) :: const_inds(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: indx - - errmsg = '' - errflg = 0 - - ! Find the constituent index for - call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) - if (errflg == 0) then - call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) - end if - - end subroutine const_indices_init - - !! @} - !! @} - -end module const_indices diff --git a/test/advection_test/const_indices.meta b/test/advection_test/const_indices.meta deleted file mode 100644 index a4cc98e2..00000000 --- a/test/advection_test/const_indices.meta +++ /dev/null @@ -1,108 +0,0 @@ -# const_indices just returns some constituent indices as a test -[ccpp-table-properties] - name = const_indices - type = scheme -[ccpp-arg-table] - name = const_indices_run - type = scheme -[ const_std_name ] - standard_name = test_banana_name - type = character | kind = len=* - units = 1 - dimensions = () - protected = true - intent = in -[ num_consts ] - standard_name = banana_array_dim - long_name = Size of test_banana_name_array - units = 1 - dimensions = () - type = integer - intent = in -[ test_stdname_array ] - standard_name = test_banana_name_array - type = character | kind = len=* - units = count - dimensions = (banana_array_dim) - intent = in -[ const_index ] - standard_name = test_banana_constituent_index - long_name = Constituent index - units = 1 - dimensions = () - type = integer - intent = out -[ const_inds ] - standard_name = test_banana_constituent_indices - long_name = Array of constituent indices - units = 1 - dimensions = (banana_array_dim) - type = integer - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = const_indices_init - type = scheme -[ const_std_name ] - standard_name = test_banana_name - type = character | kind = len=* - units = 1 - dimensions = () - protected = true - intent = in -[ num_consts ] - standard_name = banana_array_dim - long_name = Size of test_banana_name_array - units = 1 - dimensions = () - type = integer - intent = in -[ test_stdname_array ] - standard_name = test_banana_name_array - type = character | kind = len=* - units = count - dimensions = (banana_array_dim) - intent = in -[ const_index ] - standard_name = test_banana_constituent_index - long_name = Constituent index - units = 1 - dimensions = () - type = integer - intent = out -[ const_inds ] - standard_name = test_banana_constituent_indices - long_name = Array of constituent indices - units = 1 - dimensions = (banana_array_dim) - type = integer - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/advection_test/dlc_liq.F90 b/test/advection_test/dlc_liq.F90 deleted file mode 100644 index 20ff4b7b..00000000 --- a/test/advection_test/dlc_liq.F90 +++ /dev/null @@ -1,41 +0,0 @@ -! Test parameterization with a runtime constituents -! properties object outside of the register phase - -module dlc_liq - - use ccpp_kinds, only: kind_phys - use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t - - implicit none - private - - public :: dlc_liq_init - -contains - - !> \section arg_table_dlc_liq_init Argument Table - !! \htmlinclude arg_table_dlc_liq_init.html - !! - subroutine dlc_liq_init(dyn_const, errmsg, errflg) - type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - character(len=256) :: stdname - - errmsg = '' - errflg = 0 - allocate(dyn_const(1), stat=errflg) - if (errflg /= 0) then - errmsg = 'Error allocating dyn_const in dlc_liq_init' - return - end if - call dyn_const(1)%instantiate(std_name="dyn_const3", long_name='dyn const3', & - diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & - vertical_dim='vertical_layer_dimension', advected=.true., & - errcode=errflg, errmsg=errmsg) - call dyn_const(1)%standard_name(stdname, errcode=errflg, errmsg=errmsg) - - end subroutine dlc_liq_init - -end module dlc_liq diff --git a/test/advection_test/dlc_liq.meta b/test/advection_test/dlc_liq.meta deleted file mode 100644 index fedb6243..00000000 --- a/test/advection_test/dlc_liq.meta +++ /dev/null @@ -1,29 +0,0 @@ -# dlc_liq is a scheme that has a ccpp_constituent_properties_t variable -# outside of the register phase -[ccpp-table-properties] - name = dlc_liq - type = scheme -[ccpp-arg-table] - name = dlc_liq_init - type = scheme -[ dyn_const ] - standard_name = dynamic_constituents_for_dlc_liq - dimensions = (:) - type = ccpp_constituent_properties_t - intent = out - allocatable = true -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/advection_test/test_advection_host_integration.F90 b/test/advection_test/test_advection_host_integration.F90 deleted file mode 100644 index 0ee54da7..00000000 --- a/test/advection_test/test_advection_host_integration.F90 +++ /dev/null @@ -1,80 +0,0 @@ -program test - use test_prog, only: test_host, & - suite_info, & - cm, & - cs - - implicit none - - character(len=cs), target :: test_parts1(1) - character(len=cm), target :: test_invars1(12) - character(len=cm), target :: test_outvars1(13) - character(len=cm), target :: test_reqvars1(18) - - type(suite_info) :: test_suites(1) - logical :: run_okay - - test_parts1 = (/ 'physics '/) - test_invars1 = (/ & - 'banana_array_dim ', & - 'cloud_ice_dry_mixing_ratio ', & - 'cloud_liquid_dry_mixing_ratio ', & - 'tendency_of_cloud_liquid_dry_mixing_ratio', & - 'surface_air_pressure ', & - 'temperature ', & - 'time_step_for_physics ', & - 'water_temperature_at_freezing ', & - 'ccpp_constituent_tendencies ', & - 'ccpp_constituents ', & - 'number_of_ccpp_constituents ', & - 'water_vapor_specific_humidity ' /) - test_outvars1 = (/ & - 'ccpp_error_message ', & - 'ccpp_error_code ', & - 'temperature ', & - 'water_vapor_specific_humidity ', & - 'cloud_liquid_dry_mixing_ratio ', & - 'ccpp_constituent_tendencies ', & - 'ccpp_constituents ', & - 'dynamic_constituents_for_cld_liq ', & - 'dynamic_constituents_for_cld_ice ', & - 'tendency_of_cloud_liquid_dry_mixing_ratio', & - 'test_banana_constituent_index ', & - 'test_banana_constituent_indices ', & - 'cloud_ice_dry_mixing_ratio ' /) - test_reqvars1 = (/ & - 'banana_array_dim ', & - 'surface_air_pressure ', & - 'temperature ', & - 'time_step_for_physics ', & - 'cloud_liquid_dry_mixing_ratio ', & - 'tendency_of_cloud_liquid_dry_mixing_ratio', & - 'cloud_ice_dry_mixing_ratio ', & - 'dynamic_constituents_for_cld_liq ', & - 'dynamic_constituents_for_cld_ice ', & - 'water_temperature_at_freezing ', & - 'ccpp_constituent_tendencies ', & - 'ccpp_constituents ', & - 'number_of_ccpp_constituents ', & - 'test_banana_constituent_index ', & - 'test_banana_constituent_indices ', & - 'water_vapor_specific_humidity ', & - 'ccpp_error_message ', & - 'ccpp_error_code ' /) - - ! Setup expected test suite info - test_suites(1)%suite_name = 'cld_suite' - test_suites(1)%suite_parts => test_parts1 - test_suites(1)%suite_input_vars => test_invars1 - test_suites(1)%suite_output_vars => test_outvars1 - test_suites(1)%suite_required_vars => test_reqvars1 - - call test_host(run_okay, test_suites) - - if (run_okay) then - stop 0 - else - stop -1 - end if - -end program test diff --git a/test/advection_test/test_host.F90 b/test/advection_test/test_host.F90 deleted file mode 100644 index cc8bbf89..00000000 --- a/test/advection_test/test_host.F90 +++ /dev/null @@ -1,1114 +0,0 @@ -module test_prog - - use ccpp_kinds, only: kind_phys - use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t - - implicit none - private - - public test_host - - ! Public data and interfaces - integer, public, parameter :: cs = 16 - integer, public, parameter :: cm = 41 - - !> \section arg_table_suite_info Argument Table - !! \htmlinclude arg_table_suite_info.html - !! - type, public :: suite_info - character(len=cs) :: suite_name = '' - character(len=cs), pointer :: suite_parts(:) => null() - character(len=cm), pointer :: suite_input_vars(:) => null() - character(len=cm), pointer :: suite_output_vars(:) => null() - character(len=cm), pointer :: suite_required_vars(:) => null() - end type suite_info - - type(ccpp_constituent_properties_t), private, target, allocatable :: host_constituents(:) - - private :: check_suite - private :: advect_constituents ! Move data around - private :: check_errflg - -contains - - subroutine check_errflg(subname, errflg, errmsg, errflg_final) - ! If errflg is not zero, print an error message - character(len=*), intent(in) :: subname - integer, intent(in) :: errflg - character(len=*), intent(in) :: errmsg - - integer, intent(out) :: errflg_final - - if (errflg /= 0) then - write(6, '(a,i0,4a)') "Error ", errflg, " from ", trim(subname), & - ':', trim(errmsg) - !Notify test script that a failure occurred: - errflg_final = -1 !Notify test script that a failure occured - end if - - end subroutine check_errflg - - logical function check_suite(test_suite) - use test_host_ccpp_cap, only: ccpp_physics_suite_part_list - use test_host_ccpp_cap, only: ccpp_physics_suite_variables - use test_utils, only: check_list - - ! Dummy argument - type(suite_info), intent(in) :: test_suite - ! Local variables - logical :: check - integer :: errflg - character(len=512) :: errmsg - character(len=128), allocatable :: test_list(:) - - check_suite = .true. - ! First, check the suite parts - call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_parts, 'part names', & - suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check the input variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.true., output_vars=.false.) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_input_vars, & - 'input variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check the output variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.false., output_vars=.true.) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_output_vars, & - 'output variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check all required variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_required_vars, & - 'required variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - end function check_suite - - subroutine advect_constituents() - use test_host_mod, only: phys_state, & - ncnst - use test_host_mod, only: twist_array - - ! Local variables - integer :: q_ind ! Constituent index - - do q_ind = 1, ncnst ! Skip checks, they were done in constituents_in - call twist_array(phys_state%q(:, :, q_ind)) - end do - end subroutine advect_constituents - - !> \section arg_table_test_host Argument Table - !! \htmlinclude arg_table_test_host.html - !! - subroutine test_host(retval, test_suites) - - use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t - use test_host_mod, only: num_time_steps - use test_host_mod, only: init_data, & - compare_data - use test_host_mod, only: ncols, & - pver - use test_host_data, only: num_consts, & - std_name_array, & - const_std_name - use test_host_data, only: check_constituent_indices - use test_host_ccpp_cap, only: test_host_ccpp_deallocate_dynamic_constituents - use test_host_ccpp_cap, only: test_host_ccpp_register_constituents - use test_host_ccpp_cap, only: test_host_ccpp_is_scheme_constituent - use test_host_ccpp_cap, only: test_host_ccpp_initialize_constituents - use test_host_ccpp_cap, only: test_host_ccpp_number_constituents - use test_host_ccpp_cap, only: test_host_constituents_array - use test_host_ccpp_cap, only: test_host_ccpp_physics_register - use test_host_ccpp_cap, only: test_host_ccpp_physics_initialize - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_initial - use test_host_ccpp_cap, only: test_host_ccpp_physics_run - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_final - use test_host_ccpp_cap, only: test_host_ccpp_physics_finalize - use test_host_ccpp_cap, only: ccpp_physics_suite_list - use test_host_ccpp_cap, only: test_host_const_get_index - use test_host_ccpp_cap, only: test_host_model_const_properties - use test_utils, only: check_list - - type(suite_info), intent(in) :: test_suites(:) - logical, intent(out) :: retval - - logical :: check - integer :: col_start, col_end - integer :: index, sind - integer :: index_liq, index_ice - integer :: index_dyn1, index_dyn2, index_dyn3 - integer :: time_step - integer :: num_suites - integer :: num_advected ! Num advected species - logical :: const_log - logical :: is_constituent - logical :: has_default - integer :: test_scalar_const_index - integer :: test_const_indices(num_consts) - character(len=128), allocatable :: suite_names(:) - character(len=256) :: const_str - character(len=512) :: errmsg - character(len=512) :: expected_error - integer :: errflg - integer :: errflg_final ! Used to notify testing script of test failure - real(kind=kind_phys), pointer :: const_ptr(:, :, :) - real(kind=kind_phys) :: default_value - real(kind=kind_phys) :: check_value - type(ccpp_constituent_prop_ptr_t), pointer :: const_props(:) - character(len=*), parameter :: subname = 'test_host' - - ! Initialized "final" error flag used to report a failure to the larged - ! testing script: - errflg_final = 0 - - ! Gather and test the inspection routines - num_suites = size(test_suites) - call ccpp_physics_suite_list(suite_names) - retval = check_list(suite_names, test_suites(:)%suite_name, & - 'suite names') - write(6, *) 'Available suites are:' - do index = 1, size(suite_names) - do sind = 1, num_suites - if (trim(test_suites(sind)%suite_name) == & - trim(suite_names(index))) then - exit - end if - end do - write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & - ' = test_suites(', sind, ')' - end do - if (retval) then - do sind = 1, num_suites - check = check_suite(test_suites(sind)) - retval = retval .and. check - end do - end if - !!! Return here if any check failed - if (.not. retval) then - return - end if - - errflg = 0 - errmsg = '' - - ! Check that is_scheme_constituent works as expected - call test_host_ccpp_is_scheme_constituent('specific_humidity', & - is_constituent, errflg, errmsg) - call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & - errmsg, errflg_final) - ! specific_humidity should not be an existing constituent - if (is_constituent) then - write(6, *) "ERROR: specific humidity is already a constituent" - errflg_final = -1 ! Notify test script that a failure occurred - end if - call test_host_ccpp_is_scheme_constituent('cloud_ice_dry_mixing_ratio', & - is_constituent, errflg, errmsg) - call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & - errmsg, errflg_final) - ! cloud_ice_dry_mixing_ratio should be an existing constituent - if (.not. is_constituent) then - write(6, *) "ERROR: cloud_ice_dry_mixing ratio not found in ", & - "host cap constituent list" - errflg_final = -1 ! Notify test script that a failure occurred - end if - - ! Use the suite information to call the register phase - do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_register( & - test_suites(sind)%suite_name, errmsg, errflg) - if (errflg /= 0) then - write(6, '(4a)') 'ERROR in register of ', & - trim(test_suites(sind)%suite_name), ': ', trim(errmsg) - exit - end if - end if - end do - - ! Register the constituents to find out what needs advecting - ! DO A COUPLE OF TESTS FIRST - - ! First confirm the correct error occurs if you try to add an - ! incompatible constituent with the same standard name - expected_error = 'ccp_model_const_add_metadata ERROR: Trying to add ' //& - 'constituent specific_humidity but an incompatible ' // & - 'constituent with this name already exists' - allocate(host_constituents(2)) - call host_constituents(1)%instantiate(std_name="specific_humidity", & - long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & - vertical_dim="vertical_layer_dimension", advected=.true., & - min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) - call host_constituents(2)%instantiate(std_name="specific_humidity", & - long_name="Specific humidity", diag_name='H2O', units="kg kg", & - vertical_dim="vertical_layer_dimension", advected=.true., & - min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) - call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) - if (errflg == 0) then - call test_host_ccpp_register_constituents(host_constituents, & - errmsg=errmsg, errflg=errflg) - end if - ! Check the error - if (errflg == 0) then - write(6, '(2a)') 'ERROR register_constituents: expected this error: ', & - trim(expected_error) - else - if (trim(errmsg) /= trim(expected_error)) then - write(6, '(4a)') 'ERROR register_constituents: expected this error: ', & - trim(expected_error), ' Got: ', trim(errmsg) - end if - end if - ! Now try again but with a compatible constituent - should be ignored when - ! the constituents object is created - ! Use the suite information to call the register phase - errflg = 0 - call test_host_ccpp_deallocate_dynamic_constituents() - deallocate(host_constituents) - do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_register( & - test_suites(sind)%suite_name, errmsg, errflg) - if (errflg /= 0) then - write(6, '(4a)') 'ERROR in register of ', & - trim(test_suites(sind)%suite_name), ': ', trim(errmsg) - exit - end if - end if - end do - allocate(host_constituents(2)) - call host_constituents(1)%instantiate(std_name="specific_humidity", & - long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & - vertical_dim="vertical_layer_dimension", advected=.true., & - min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) - call host_constituents(2)%instantiate(std_name="specific_humidity", & - long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & - vertical_dim="vertical_layer_dimension", advected=.true., & - min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) - call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) - if (errflg == 0) then - call test_host_ccpp_register_constituents(host_constituents, & - errmsg=errmsg, errflg=errflg) - end if - if (errflg /= 0) then - write(6, '(2a)') 'ERROR register_constituents: ', trim(errmsg) - retval = .false. - return - end if - ! Check number of advected constituents - if (errflg == 0) then - call test_host_ccpp_number_constituents(num_advected, errmsg=errmsg, & - errflg=errflg) - call check_errflg(subname // ".num_advected", errflg, errmsg, errflg_final) - end if - if (num_advected /= 6) then - write(6, '(a,i0)') "ERROR: num advected constituents = ", num_advected - retval = .false. - return - end if - ! Initialize constituent data - call test_host_ccpp_initialize_constituents(ncols, pver, errflg, errmsg) - - ! Stop tests here if initialization failed (as all other tests will likely - ! fail as well: - if (errflg /= 0) then - retval = .false. - return - end if - - ! Initialize our 'data' - const_ptr => test_host_constituents_array() - - ! Check if the specific humidity index can be found: - call test_host_const_get_index('specific_humidity', index, & - errflg, errmsg) - call check_errflg(subname // ".index_specific_humidity", errflg, errmsg, & - errflg_final) - - ! Check if the cloud liquid index can be found: - call test_host_const_get_index('cloud_liquid_dry_mixing_ratio', & - index_liq, errflg, errmsg) - call check_errflg(subname // ".index_cld_liq", errflg, errmsg, & - errflg_final) - - ! Check if the cloud ice index can be found: - call test_host_const_get_index('cloud_ice_dry_mixing_ratio', & - index_ice, errflg, errmsg) - call check_errflg(subname // ".index_cld_ice", errflg, errmsg, & - errflg_final) - - ! Check if the dynamic constituents indices can be found - call test_host_const_get_index('dyn_const1', index_dyn1, errflg, errmsg) - call check_errflg(subname // ".index_dyn_const1", errflg, errmsg, & - errflg_final) - call test_host_const_get_index('dyn_const2_wrt_moist_air', index_dyn2, errflg, errmsg) - call check_errflg(subname // ".index_dyn_const2", errflg, errmsg, & - errflg_final) - call test_host_const_get_index('dyn_const3_wrt_moist_air_and_condensed_water', index_dyn3, errflg, errmsg) - call check_errflg(subname // ".index_dyn_const3", errflg, errmsg, & - errflg_final) - - ! Load up the test array indices - call test_host_const_get_index(const_std_name, test_scalar_const_index, errflg, errmsg) - call check_errflg(subname // "." // const_std_name, errflg, errmsg, & - errflg_final) - do sind = 1, num_consts - call test_host_const_get_index(std_name_array(sind), & - test_const_indices(sind), errflg, errmsg) - call check_errflg(subname // "." // std_name_array(sind), errflg, errmsg, & - errflg_final) - end do - - ! Stop tests here if the index checks failed, as all other tests will - ! likely fail as well: - if (errflg_final /= 0) then - retval = .false. - return - end if - - call init_data(const_ptr, index, index_liq, index_ice, index_dyn3) - - ! Check some constituent properties - ! ++++++++++++++++++++++++++++++++++ - - const_props => test_host_model_const_properties() - - ! Standard name: - call const_props(index)%standard_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get standard_name for specific_humidity, index = ", & - index, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured - end if - if (errflg == 0) then - if (trim(const_str) /= 'specific_humidity') then - write(6, *) "ERROR: standard name, '", trim(const_str), & - "' should be 'specific_humidity'" - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - ! Check standard name for a dynamic constituent - call const_props(index_dyn2)%standard_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get standard_name for dyn_const2, index = ", & - index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured - end if - if (errflg == 0) then - if (trim(const_str) /= 'dyn_const2_wrt_moist_air') then - write(6, *) "ERROR: standard name, '", trim(const_str), & - "' should be 'dyn_const2_wrt_moist_air'" - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! Long name: - call const_props(index_liq)%long_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get long_name for cld_liq index = ", & - index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured - end if - if (errflg == 0) then - if (trim(const_str) /= 'Cloud liquid dry mixing ratio') then - write(6, *) "ERROR: long name, '", trim(const_str), & - "' should be 'Cloud liquid dry mixing ratio'" - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - ! Check long name for a dynamic constituent - call const_props(index_dyn1)%long_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get long_name for dyn_const1 index = ", & - index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured - end if - if (errflg == 0) then - if (trim(const_str) /= 'dyn const1') then - write(6, *) "ERROR: long name, '", trim(const_str), & - "' should be 'dyn const1'" - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! Diagnostic name: - call const_props(index_liq)%diagnostic_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get diagnostic name for cld_liq index = ", & - index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured - end if - if (errflg == 0) then - if (trim(const_str) /= 'CLDLIQ') then - write(6, *) "ERROR: diagnostic name, '", trim(const_str), & - "' should be 'CLDLIQ'" - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - ! Check default diagnostic name is set correctly - call const_props(index_ice)%diagnostic_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get diagnostic name for cld_ice index = ", & - index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured - end if - if (errflg == 0) then - if (trim(const_str) /= 'cld_ice_array') then - write(6, *) "ERROR: diagnostic name, '", trim(const_str), & - "' should be 'cld_ice_array'" - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - ! Check diagnostic name of a dynamic constituent - call const_props(index_dyn2)%diagnostic_name(const_str, errflg, & - errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get diagnostic name for dyn_const2 index = ", & - index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured - end if - if (errflg == 0) then - if (trim(const_str) /= 'DYNCONST2') then - write(6, *) "ERROR: diagnostic name, '", trim(const_str), & - "' should be 'DYNCONST2'" - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! Mass mixing ratio: - call const_props(index_ice)%is_mass_mixing_ratio(const_log, errflg, & - errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get mass mixing ratio prop for cld_ice index = ", & - index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured - end if - if (errflg == 0) then - if (.not. const_log) then - write(6, *) "ERROR: cloud ice is not a mass mixing_ratio" - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - ! Check mass mixing ratio for a dynamic constituent - call const_props(index_dyn2)%is_mass_mixing_ratio(const_log, errflg, & - errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get mass mixing ratio prop for dyn_const2 index = ", & - index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured - end if - if (errflg == 0) then - if (.not. const_log) then - write(6, *) "ERROR: dyn_const2 is not a mass mixing_ratio" - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! Dry mixing ratio: - call const_props(index_ice)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get dry prop for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (.not. const_log) then - write(6, *) "ERROR: cloud ice mass_mixing_ratio is not dry" - errflg_final = -1 - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - ! Check wet mixing ratio for dynamic constituent 1 - call const_props(index_dyn1)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get dry prop for dyn_const1 index = ", index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (const_log) then - write(6, *) "ERROR: dyn_const1 is dry and should be wet" - errflg_final = -1 - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - call const_props(index_dyn1)%is_wet(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get wet prop for dyn_const1 index = ", index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (.not. const_log) then - write(6, *) "ERROR: dyn_const1 is not wet but should be" - errflg_final = -1 - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - ! Check moist mixing ratio for dynamic constituent 2 - call const_props(index_dyn2)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get dry prop for dyn_const2 index = ", index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (const_log) then - write(6, *) "ERROR: dyn_const2 is dry and should be moist" - errflg_final = -1 - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - call const_props(index_dyn2)%is_moist(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get moist prop for dyn_const2 index = ", index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (.not. const_log) then - write(6, *) "ERROR: dyn_const2 is not moist but should be" - errflg_final = -1 - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - ! Check dry mixing ratio for dynamic constituent 3 - call const_props(index_dyn3)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get dry prop for dyn_const3 index = ", index_dyn3, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (.not. const_log) then - write(6, *) "ERROR: dyn_const3 is not dry and should be" - errflg_final = -1 - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! ------------------- - - ! ------------------- - ! minimum value tests: - ! ------------------- - - ! Check that a constituent's minimum value defaults to zero: - call const_props(index_dyn2)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get minimum value for dyn_const2 index = ", index_dyn2, & - trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (check_value /= 0._kind_phys) then ! Should be zero - write(6, *) "ERROR: 'minimum' should default to zero for all ", & - "constituents unless set by host model or scheme metadata." - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! Check that a constituent instantiated with a specified minimum value - ! actually contains that minimum value property: - call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get minimum value for dyn_const1 index = ", index_dyn1, & - trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (check_value /= 1000._kind_phys) then !Should be 1000 - write(6, *) "ERROR: 'minimum' should give a value of 1000 ", & - "for dyn_const1, as was set during instantiation." - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! Check that setting a constituent's minimum value works - ! as expected: - call const_props(index_dyn1)%set_minimum(1._kind_phys, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to set minimum value for dyn_const1 index = ", index_dyn1, & - trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & - " trying to get minimum value for dyn_const1 index = ", & - index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - end if - if (errflg == 0) then - if (check_value /= 1._kind_phys) then ! Should now be one - write(6, *) "ERROR: 'set_minimum' did not set constituent", & - " minimum value correctly." - errflg_final = -1 ! Notify test script that a failure occurred - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! ---------------------- - ! molecular weight tests: - ! ---------------------- - - ! Check that a constituent instantiated with a specified molecular - ! weight actually contains that molecular weight property value: - call const_props(index)%molar_mass(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get molecular weight for specific humidity index = ", & - index, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (check_value /= 2000._kind_phys) then ! Should be 2000 - write(6, *) "ERROR: 'molar_mass' should give a value of 2000 ", & - "for specific humidity, as was set during instantiation." - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! Check that setting a constituent's molecular weight works - ! as expected: - call const_props(index_ice)%set_molar_mass(1._kind_phys, errflg, & - errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to set molecular weight for cld_ice index = ", index_ice, & - trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - call const_props(index_ice)%molar_mass(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & - " trying to get molecular weight for cld_ice index = ", & - index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - end if - if (errflg == 0) then - if (check_value /= 1._kind_phys) then ! Should be equal to one - write(6, *) "ERROR: 'set_molar_mass' did not set constituent", & - " molecular weight value correctly." - errflg_final = -1 ! Notify test script that a failure occurred - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! ------------------- - ! thermo-active tests: - ! ------------------- - - ! Check that being thermodynamically active defaults to False: - call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get thermo_active prop for cld_ice index = ", index_ice, & - trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (check) then ! Should be False - write(6, *) "ERROR: 'is_thermo_active' should default to False ", & - "for all constituents unless set by host model." - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! Check that setting a constituent to be thermodynamically active works - ! as expected: - call const_props(index_ice)%set_thermo_active(.true., errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to set thermo_active prop for cld_ice index = ", index_ice, & - trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & - " trying to get thermo_active prop for cld_ice index = ", & - index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - end if - if (errflg == 0) then - if (.not. check) then ! Should now be True - write(6, *) "ERROR: 'set_thermo_active' did not set", & - " thermo_active constituent property correctly." - errflg_final = -1 ! Notify test script that a failure occurred - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - ! ------------------- - - ! ------------------- - ! water-species tests: - ! ------------------- - - ! Check that being a water species defaults to False: - call const_props(index_liq)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to get water_species prop for cld_liq index = ", index_liq, & - trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (check) then ! Should be False - write(6, *) "ERROR: 'is_water_species' should default to False ", & - "for all constituents unless set by host model." - errflg_final = -1 ! Notify test script that a failure occured - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! Check that setting a constituent to be a water species works - ! as expected: - call const_props(index_liq)%set_water_species(.true., errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to set water_species prop for cld_liq index = ", index_liq, & - trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - call const_props(index_liq)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & - " trying to get water_species prop for cld_liq index = ", & - index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - end if - if (errflg == 0) then - if (.not. check) then ! Should now be True - write(6, *) "ERROR: 'set_water_species' did not set", & - " water_species constituent property correctly." - errflg_final = -1 ! Notify test script that a failure occurred - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - - ! Check that setting a constituent to be a water species via the - ! instantiate call works as expected - call const_props(index_dyn1)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & - "trying to get water_species prop for dyn_const1 index = ", & - index_dyn1, trim(errmsg) - end if - if (errflg == 0) then - if (.not. check) then ! Should now be True - write(6, *) "ERROR: 'water_species=.true. did not set", & - " water_species constituent property correctly" - errflg_final = -1 ! Notify test script that a failure occurred - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - call const_props(index_dyn2)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & - "trying to get water_species prop for dyn_const2 index = ", & - index_dyn2, trim(errmsg) - end if - if (errflg == 0) then - if (check) then ! Should now be False - write(6, *) "ERROR: 'water_species=.false. did not set", & - " water_species constituent property correctly" - errflg_final = -1 ! Notify test script that a failure occurred - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - ! ------------------- - - ! Check that setting a constituent's default value works as expected - call const_props(index_liq)%has_default(has_default, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to check for default for cld_liq index = ", index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (has_default) then - write(6, *) "ERROR: cloud liquid mass_mixing_ratio should not have default but does" - errflg_final = -1 ! Notify test script that a failure occurred - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - call const_props(index_ice)%has_default(has_default, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to check for default for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (.not. has_default) then - write(6, *) "ERROR: cloud ice mass_mixing_ratio should have default but doesn't" - errflg_final = -1 ! Notify test script that a failure occurred - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - call const_props(index_ice)%default_value(default_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & - "to grab default for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred - end if - if (errflg == 0) then - if (default_value /= 0.0_kind_phys) then - write(6, *) "ERROR: cloud ice mass_mixing_ratio default is ", default_value, & - " but should be 0.0" - errflg_final = -1 ! Notify test script that a failure occurred - end if - else - ! Reset error flag to continue testing other properties: - errflg = 0 - end if - ! ++++++++++++++++++++++++++++++++++ - - ! Set error flag to the "final" value, because any error - ! above will likely result in a large number of failures - ! below: - errflg = errflg_final - - ! Use the suite information to setup the run - do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_initialize( & - test_suites(sind)%suite_name, errmsg, errflg) - if (errflg /= 0) then - write(6, '(4a)') 'ERROR in initialize of ', & - trim(test_suites(sind)%suite_name), ': ', trim(errmsg) - exit - end if - end if - end do - - ! Check indices - call check_constituent_indices(test_scalar_const_index, test_const_indices, & - errmsg, errflg) - call check_errflg(subname // " check suite indices", errflg, errmsg, & - errflg_final) - - ! Loop over time steps - do time_step = 1, num_time_steps - ! Initialize the timestep - do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_timestep_initial( & - test_suites(sind)%suite_name, errmsg, errflg) - if (errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(errmsg) - end if - end if - end do - - do col_start = 1, ncols, 5 - if (errflg /= 0) then - continue - end if - col_end = min(col_start + 4, ncols) - - do sind = 1, num_suites - do index = 1, size(test_suites(sind)%suite_parts) - if (errflg == 0) then - call test_host_ccpp_physics_run( & - test_suites(sind)%suite_name, & - test_suites(sind)%suite_parts(index), & - col_start, col_end, errmsg, errflg) - if (errflg /= 0) then - write(6, '(5a)') trim(test_suites(sind)%suite_name), & - '/', trim(test_suites(sind)%suite_parts(index)),& - ': ', trim(errmsg) - exit - end if - end if - end do - end do - end do - ! Check indices - call check_constituent_indices(test_scalar_const_index, test_const_indices, & - errmsg, errflg) - call check_errflg(subname // " check suite indices", errflg, errmsg, & - errflg_final) - - do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_timestep_final( & - test_suites(sind)%suite_name, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(errmsg) - exit - end if - end do - - ! Run "dycore" - if (errflg == 0) then - call advect_constituents() - end if - end do ! End time step loop - - do sind = 1, num_suites - if (errflg == 0) then - call test_host_ccpp_physics_finalize( & - test_suites(sind)%suite_name, errmsg, errflg) - if (errflg /= 0) then - write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & - trim(errmsg) - write(6, '(2a)') 'An error occurred in ccpp_timestep_final, ', & - 'Exiting...' - exit - end if - end if - end do - - if (errflg == 0) then - ! Run finished without error, check answers - if (compare_data(num_advected)) then - write(6, *) 'Answers are correct!' - errflg = 0 - else - write(6, *) 'Answers are not correct!' - errflg = -1 - end if - end if - - ! Make sure "final" flag is non-zero if "errflg" is: - if (errflg /= 0) then - errflg_final = -1 ! Notify test script that a failure occured - end if - - ! Set return value to False if any errors were found: - retval = errflg_final == 0 - - end subroutine test_host - -end module test_prog diff --git a/test/advection_test/test_host.meta b/test/advection_test/test_host.meta deleted file mode 100644 index 5d861764..00000000 --- a/test/advection_test/test_host.meta +++ /dev/null @@ -1,38 +0,0 @@ -[ccpp-table-properties] - name = suite_info - type = ddt -[ccpp-arg-table] - name = suite_info - type = ddt - -[ccpp-table-properties] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True -[ col_end ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - protected = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer diff --git a/test/advection_test/test_host_data.F90 b/test/advection_test/test_host_data.F90 deleted file mode 100644 index f360ad79..00000000 --- a/test/advection_test/test_host_data.F90 +++ /dev/null @@ -1,96 +0,0 @@ -module test_host_data - - use ccpp_kinds, only: kind_phys - - implicit none - - !> \section arg_table_physics_state Argument Table - !! \htmlinclude arg_table_physics_state.html - type physics_state - real(kind=kind_phys), allocatable :: ps(:) ! surface pressure - real(kind=kind_phys), allocatable :: temp(:, :) ! temperature - real(kind=kind_phys), dimension(:, :, :), pointer :: q => null() ! constituent array - end type physics_state - - !> \section arg_table_test_host_data Argument Table - !! \htmlinclude arg_table_test_host_data.html - integer, public, parameter :: num_consts = 3 - character(len=32), public, parameter :: std_name_array(num_consts) = (/ & - 'specific_humidity ', & - 'cloud_liquid_dry_mixing_ratio', & - 'cloud_ice_dry_mixing_ratio ' /) - character(len=32), public, parameter :: const_std_name = std_name_array(1) - - integer :: const_inds(num_consts) = -1 ! test array access from suite - integer :: const_index = -1 ! test scalar access from suite - - public :: allocate_physics_state - public :: check_constituent_indices - -contains - - subroutine check_constituent_indices(test_index, test_indices, errmsg, errflg) - ! Check constituent indices against what was found by suite - ! indices are passed in rather than looked up to avoid a dependency loop - ! Dummy arguments - integer, intent(in) :: test_index ! scalar const index from host - integer, intent(in) :: test_indices(:) ! array_test_indices from host - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! Local variable - integer :: indx - integer :: emstrt - - errflg = 0 - errmsg = '' - if (test_index /= const_index) then - emstrt = len_trim(errmsg) + 1 - write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_index_check for ', & - const_std_name, test_index, ' /= ', const_index - errflg = errflg + 1 - end if - do indx = 1, num_consts - if (test_indices(indx) /= const_inds(indx)) then - emstrt = len_trim(errmsg) + 1 - if (len_trim(errmsg) > 0) then - write(errmsg(emstrt:), '(", ")') - emstrt = emstrt + 2 - end if - write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_indices_check for ', & - std_name_array(indx), test_indices(indx), ' /= ', const_inds(indx) - errflg = errflg + 1 - end if - end do - - ! Reset for next test - const_index = -1 - const_inds = -1 - - end subroutine check_constituent_indices - - subroutine allocate_physics_state(cols, levels, constituents, state) - integer, intent(in) :: cols - integer, intent(in) :: levels - real(kind=kind_phys), pointer :: constituents(:, :, :) - type(physics_state), intent(out) :: state - - if (allocated(state%ps)) then - deallocate(state%ps) - end if - allocate(state%ps(cols)) - state%ps = 0.0_kind_phys - if (allocated(state%temp)) then - deallocate(state%temp) - end if - allocate(state%temp(cols, levels)) - if (associated(state%q)) then - ! Do not deallocate (we do not own this array) - nullify(state%q) - end if - ! Point to the advected constituents array - state%q => constituents - - end subroutine allocate_physics_state - -end module test_host_data diff --git a/test/advection_test/test_host_data.meta b/test/advection_test/test_host_data.meta deleted file mode 100644 index a676f141..00000000 --- a/test/advection_test/test_host_data.meta +++ /dev/null @@ -1,70 +0,0 @@ -[ccpp-table-properties] - name = physics_state - type = ddt -[ccpp-arg-table] - name = physics_state - type = ddt -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_dimension) -[ Temp ] - standard_name = temperature - units = K - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys -[ q ] - standard_name = constituent_mixing_ratio - state_variable = true - type = real - kind = kind_phys - units = kg kg-1 moist or dry air depending on type - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) -[ q(:,:,index_of_water_vapor_specific_humidity) ] - standard_name = water_vapor_specific_humidity - state_variable = true - type = real - kind = kind_phys - units = kg kg-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - -[ccpp-table-properties] - name = test_host_data - type = module -[ccpp-arg-table] - name = test_host_data - type = module -[ num_consts ] - standard_name = banana_array_dim - long_name = Size of test_banana_name_array - units = 1 - dimensions = () - type = integer -[ std_name_array ] - standard_name = test_banana_name_array - type = character | kind = len=32 - units = count - dimensions = (banana_array_dim) - protected = true -[ const_std_name ] - standard_name = test_banana_name - type = character | kind = len=32 - units = 1 - dimensions = () - protected = true -[ const_inds ] - standard_name = test_banana_constituent_indices - long_name = Array of constituent indices - units = 1 - dimensions = (banana_array_dim) - protected = true - type = integer -[ const_index ] - standard_name = test_banana_constituent_index - long_name = Constituent index - units = 1 - dimensions = () - type = integer diff --git a/test/advection_test/test_host_mod.F90 b/test/advection_test/test_host_mod.F90 deleted file mode 100644 index 5099b9c1..00000000 --- a/test/advection_test/test_host_mod.F90 +++ /dev/null @@ -1,176 +0,0 @@ -module test_host_mod - - use ccpp_kinds, only: kind_phys - use test_host_data, only: physics_state, & - allocate_physics_state - - implicit none - public - - integer, parameter :: num_time_steps = 2 - real(kind=kind_phys), parameter :: tolerance = 1.0e-13_kind_phys - - !> \section arg_table_test_host_mod Argument Table - !! \htmlinclude arg_table_test_host_mod.html - !! - integer, parameter :: ncols = 10 - integer, parameter :: pver = 5 - integer, parameter :: pverp = pver + 1 - integer, protected :: ncnst = -1 - integer, protected :: index_qv = -1 - real(kind=kind_phys) :: dt - real(kind=kind_phys), parameter :: tfreeze = 273.15_kind_phys - type(physics_state) :: phys_state - integer :: num_model_times = -1 - integer, allocatable :: model_times(:) - - public :: init_data - public :: compare_data - public :: twist_array - - real(kind=kind_phys), private, allocatable :: check_vals(:, :, :) - real(kind=kind_phys), private :: check_temp(ncols, pver) - integer, private :: ind_liq = -1 - integer, private :: ind_ice = -1 - -contains - - subroutine init_data(constituent_array, index_qv_use, index_liq, index_ice, index_dyn) - - ! Dummy arguments - real(kind=kind_phys), pointer :: constituent_array(:, :, :) ! From host & suites - integer, intent(in) :: index_qv_use - integer, intent(in) :: index_liq - integer, intent(in) :: index_ice - integer, intent(in) :: index_dyn - - ! Local variables - integer :: col - integer :: lev - integer :: cind - integer :: itime - real(kind=kind_phys) :: qmax - real(kind=kind_phys), parameter :: inc = 0.1_kind_phys - - ! Allocate and initialize state - ! Temperature starts above freezing and decreases to -30C - ! water vapor is initialized in odd columns to different amounts - ncnst = size(constituent_array, 3) - call allocate_physics_state(ncols, pver, constituent_array, phys_state) - index_qv = index_qv_use - ind_liq = index_liq - ind_ice = index_ice - allocate(check_vals(ncols, pver, ncnst)) - check_vals(:, :, :) = 0.0_kind_phys - check_vals(:, :, index_dyn) = 1.0_kind_phys - do lev = 1, pver - phys_state%temp(:, lev) = tfreeze + (10.0_kind_phys * (lev - 3)) - qmax = real(lev, kind_phys) - do col = 1, ncols - if (mod(col, 2) == 1) then - phys_state%q(col, lev, index_qv) = qmax - else - phys_state%q(col, lev, index_qv) = 0.0_kind_phys - end if - end do - end do - check_vals(:, :, index_qv) = phys_state%q(:, :, index_qv) - check_temp(:, :) = phys_state%temp(:, :) - ! Do timestep 1 - do col = 1, ncols, 2 - check_temp(col, 1) = check_temp(col, 1) + 0.5_kind_phys - check_vals(col, 1, index_qv) = check_vals(col, 1, index_qv) - inc - check_vals(col, 1, ind_liq) = check_vals(col, 1, ind_liq) + inc - end do - do itime = 1, num_time_steps - do cind = 1, ncnst - call twist_array(check_vals(:, :, cind)) - end do - end do - - end subroutine init_data - - subroutine twist_array(array) - ! Dummy argument - real(kind=kind_phys), intent(inout) :: array(:, :) - - ! Local variables - integer :: icol, ilev ! Field coordinates - integer :: idir ! 'w' sign - integer :: levb, leve ! Starting and ending level indices - real(kind=kind_phys) :: last_val, next_val - - idir = 1 - leve = (pver * mod(ncols, 2)) + mod(ncols - 1, 2) - last_val = array(ncols, leve) - do icol = 1, ncols - levb = ((pver * (1 - idir)) + (1 + idir)) / 2 - leve = ((pver * (1 + idir)) + (1 - idir)) / 2 - do ilev = levb, leve, idir - next_val = array(icol, ilev) - array(icol, ilev) = last_val - last_val = next_val - end do - idir = -1 * idir - end do - - end subroutine twist_array - - logical function compare_data(ncnst) - - integer, intent(in) :: ncnst - - integer :: col - integer :: lev - integer :: cind - logical :: need_header - real(kind=kind_phys) :: check - real(kind=kind_phys) :: denom - - compare_data = .true. - - need_header = .true. - do lev = 1, pver - do col = 1, ncols - check = check_temp(col, lev) - if (abs((phys_state%temp(col, lev) - check) / check) > & - tolerance) then - if (need_header) then - write(6, '(" COL LEV T MIDPOINTS EXPECTED")') - need_header = .false. - end if - write(6, '(2i5,2(3x,es15.7))') col, lev, & - phys_state%temp(col, lev), check - compare_data = .false. - end if - end do - end do - ! Check constituents - need_header = .true. - do cind = 1, ncnst - do lev = 1, pver - do col = 1, ncols - check = check_vals(col, lev, cind) - if (check < tolerance) then - denom = 1.0_kind_phys - else - denom = check - end if - if (abs((phys_state%q(col, lev, cind) - check) / denom) > & - tolerance) then - if (need_header) then - write(6, '(2(2x,a),3x,a,10x,a,14x,a)') & - 'COL', 'LEV', 'C#', 'Q', 'EXPECTED' - need_header = .false. - end if - write(6, '(3i5,2(3x,es15.7))') col, lev, cind, & - phys_state%q(col, lev, cind), check - compare_data = .false. - end if - end do - end do - end do - - end function compare_data - -end module test_host_mod diff --git a/test/advection_test/test_host_mod.meta b/test/advection_test/test_host_mod.meta deleted file mode 100644 index 9f04a6fc..00000000 --- a/test/advection_test/test_host_mod.meta +++ /dev/null @@ -1,64 +0,0 @@ -[ccpp-table-properties] - name = test_host_mod - type = module -[ccpp-arg-table] - name = test_host_mod - type = module -[ ncols] - standard_name = horizontal_dimension - units = count - type = integer - protected = True - dimensions = () -[ pver ] - standard_name = vertical_layer_dimension - units = count - type = integer - protected = True - dimensions = () -[ pverP ] - standard_name = vertical_interface_dimension - type = integer - units = count - protected = True - dimensions = () -[ ncnst ] - standard_name = number_of_tracers - type = integer - units = count - protected = True - dimensions = () -[ index_qv ] - standard_name = index_of_water_vapor_specific_humidity - units = index - type = integer - protected = True - dimensions = () -[ dt ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real | kind = kind_phys -[ tfreeze ] - standard_name = water_temperature_at_freezing - long_name = Freezing temperature of water at sea level - units = K - dimensions = () - type = real | kind = kind_phys -[ phys_state ] - standard_name = physics_state_derived_type - long_name = Physics State DDT - type = physics_state - dimensions = () -[ num_model_times ] - standard_name = number_of_model_times - type = integer - units = count - dimensions = () -[ model_times ] - standard_name = model_times - units = seconds - dimensions = (number_of_model_times) - type = integer - allocatable = True diff --git a/test/capgen_test/CMakeLists.txt b/test/capgen_test/CMakeLists.txt deleted file mode 100644 index 49c75842..00000000 --- a/test/capgen_test/CMakeLists.txt +++ /dev/null @@ -1,95 +0,0 @@ - -#------------------------------------------------------------------------------ -# -# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES -# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) -# -#------------------------------------------------------------------------------ -set(SCHEME_FILES "setup_coeffs" "temp_set" "temp_adjust" "temp_calc_adjust") -set(SUITE_SCHEME_FILES "make_ddt" "environ_conditions" "temp_kinds") -set(HOST_FILES "test_host_data" "test_host_mod") -set(SUITE_FILES "ddt_suite.xml" "temp_suite.xml") -set(KIND_TYPE "kind_phys=REAL64") - -# HOST is the name of the executable we will build. -# We assume there are files ${HOST}.meta and ${HOST}.F90 in CMAKE_SOURCE_DIR -set(HOST "test_host") - -# By default, generated caps go in ccpp subdir -set(CCPP_CAP_FILES "${CMAKE_CURRENT_BINARY_DIR}/ccpp") - -# Create lists for Fortran and meta data files from file names -# Fortran files are not all in one directory -set(SCHEME_FORTRAN_FILES "") -foreach(sfile ${SCHEME_FILES}) - find_file(fort_file "${sfile}.F90" NO_CACHE - HINTS ${CMAKE_CURRENT_SOURCE_DIR} - HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir1 - HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir2) - list(APPEND SCHEME_FORTRAN_FILES ${fort_file}) - unset(fort_file) -endforeach() -find_file(fort_file "ddt2.F90" NO_CACHE - HINTS ${CMAKE_CURRENT_SOURCE_DIR}) -list(APPEND SCHEME_FORTRAN_FILES ${fort_file}) -unset(fort_file) -list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_META_FILES) -set(SUITE_SCHEME_FORTRAN_FILES "") -foreach(sfile ${SUITE_SCHEME_FILES}) - find_file(fort_file "${sfile}.F90" NO_CACHE - HINTS ${CMAKE_CURRENT_SOURCE_DIR} - HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir1 - HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir2 - HINTS ${CMAKE_CURRENT_SOURCE_DIR}/adjust) - list(APPEND SUITE_SCHEME_FORTRAN_FILES ${fort_file}) - unset(fort_file) -endforeach() - -list(TRANSFORM SUITE_SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SUITE_SCHEME_META_FILES) -list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE CAPGEN_HOST_FORTRAN_FILES) -list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE CAPGEN_HOST_METADATA_FILES) - -list(APPEND CAPGEN_HOST_METADATA_FILES "${HOST}.meta") - -ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${CAPGEN_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_META_FILES} ${SUITE_SCHEME_META_FILES} - SUITES ${SUITE_FILES} - HOST_NAME ${HOST} - KIND_TYPES ${KIND_TYPES} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Retrieve the list of Fortran files required for test host from datatable.xml and set to CCPP_CAPS_LIST -ccpp_datafile(DATATABLE "${CCPP_CAP_FILES}/datatable.xml" - REPORT_NAME "--ccpp-files") - -# Create test host library -add_library(CAPGEN_TESTLIB OBJECT ${SCHEME_FORTRAN_FILES} - ${SUITE_SCHEME_FORTRAN_FILES} - ${CAPGEN_HOST_FORTRAN_FILES} - ${CCPP_CAPS_LIST}) - -# Setup test executable with needed dependencies -add_executable(capgen_host_integration test_capgen_host_integration.F90 ${HOST}.F90) -if(OPENMP) - target_link_libraries(capgen_host_integration PRIVATE OpenMP::OpenMP_Fortran) -endif() -target_link_libraries(capgen_host_integration PRIVATE CAPGEN_TESTLIB test_utils) -target_include_directories(capgen_host_integration PRIVATE "$") - -# Add executable to be called with ctest -add_test(NAME ctest_capgen_host_integration_omp1 - COMMAND capgen_host_integration) - -add_test(NAME ctest_capgen_host_integration_omp2 - COMMAND capgen_host_integration) - -set_tests_properties(ctest_capgen_host_integration_omp1 - PROPERTIES - ENVIRONMENT "OMP_NUM_THREADS=1" -) - -set_tests_properties(ctest_capgen_host_integration_omp2 - PROPERTIES - ENVIRONMENT "OMP_NUM_THREADS=2" -) diff --git a/test/capgen_test/README.md b/test/capgen_test/README.md deleted file mode 100644 index bcc0e628..00000000 --- a/test/capgen_test/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Capgen Test - -Contains tests for overall capgen capabilities such as: -- Multiple suites -- Multiple groups -- General DDT usage -- DDT with undocumented DDT member variable -- Dimensions with `ccpp_constant_one:N` and just `N` -- Non-standard dimensions (not just horizontal and vertical) (including integer dimensions) -- Variables that should be promoted to suite level -- Dimensions that are set in the register phase and used to allocate module-level - interstitial variables - -## Building/Running - -To explicitly build/run the capgen test host, run: - -```bash -$ cmake --fresh -S -B -DCCPP_RUN_CAPGEN_TEST=ON -$ cd -$ make -$ ctest -``` diff --git a/test/capgen_test/adjust/temp_kinds.F90 b/test/capgen_test/adjust/temp_kinds.F90 deleted file mode 100644 index 3fb4cca4..00000000 --- a/test/capgen_test/adjust/temp_kinds.F90 +++ /dev/null @@ -1,12 +0,0 @@ -! Define a new Fortran kind for use within -! various temp_* test files. - -module temp_kinds - - implicit none - private - - integer, public, parameter :: temp_r8 = selected_real_kind(12) !8-byte real - integer, public, parameter :: temp_i8 = selected_int_kind(13) !8-byte integer - -end module temp_kinds diff --git a/test/capgen_test/capgen_test_reports.py b/test/capgen_test/capgen_test_reports.py deleted file mode 100644 index 3c683aab..00000000 --- a/test/capgen_test/capgen_test_reports.py +++ /dev/null @@ -1,152 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Test capgen database report python interface - - Assumptions: - - Command line arguments: build_dir database_filepath - - Usage: python test_reports ------------------------------------------------------------------------ -""" -import os -import unittest - -from test_stub import BaseTests - -_BUILD_DIR = os.path.join(os.path.abspath(os.environ['BUILD_DIR']), "test", "capgen_test") -_DATABASE = os.path.abspath(os.path.join(_BUILD_DIR, "ccpp", "datatable.xml")) - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_FRAMEWORK_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, os.pardir)) -_SCRIPTS_DIR = os.path.join(_FRAMEWORK_DIR, "scripts") -_SRC_DIR = os.path.join(_FRAMEWORK_DIR, "src") - -# Check data -_HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] -_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_ddt_suite_cap.F90"), - os.path.join(_BUILD_DIR, "ccpp", "ccpp_temp_suite_cap.F90")] -_UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), - os.path.join(_SRC_DIR, "ccpp_constituent_prop_mod.F90"), - os.path.join(_SRC_DIR, "ccpp_scheme_utils.F90"), - os.path.join(_SRC_DIR, "ccpp_hashable.F90"), - os.path.join(_SRC_DIR, "ccpp_hash_table.F90")] -_CCPP_FILES = _UTILITY_FILES + \ - [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90"), - os.path.join(_BUILD_DIR, "ccpp", "ccpp_ddt_suite_cap.F90"), - os.path.join(_BUILD_DIR, "ccpp", "ccpp_temp_suite_cap.F90")] -_DEPENDENCIES = [os.path.join(_TEST_DIR, "adjust", "qux.F90"), - os.path.join(_TEST_DIR, "adjust", "temp_kinds.F90"), - os.path.join(_TEST_DIR, "ddt2"), - os.path.join(_TEST_DIR, "bar.F90"), - os.path.join(_TEST_DIR, "foo.F90")] -_PROCESS_LIST = ["setter=temp_set", "adjusting=temp_calc_adjust"] -_MODULE_LIST = ["environ_conditions", "make_ddt", "setup_coeffs", "temp_adjust", - "temp_calc_adjust", "temp_set"] -_SUITE_LIST = ["ddt_suite", "temp_suite"] -_INPUT_VARS_DDT = ["model_times", "number_of_model_times", - "horizontal_loop_begin", "horizontal_loop_end", - "surface_air_pressure", "horizontal_dimension"] -_OUTPUT_VARS_DDT = ["ccpp_error_code", "ccpp_error_message", "model_times", - "surface_air_pressure", "number_of_model_times"] -_REQUIRED_VARS_DDT = _INPUT_VARS_DDT + _OUTPUT_VARS_DDT -_PROT_VARS_TEMP = ["horizontal_loop_begin", "horizontal_loop_end", - "horizontal_dimension", "vertical_layer_dimension", - "number_of_tracers", - "lower_bound_of_vertical_dimension_of_soil", - "upper_bound_of_vertical_dimension_of_soil", - "configuration_variable", - # Added for --debug - "index_of_water_vapor_specific_humidity", - "vertical_interface_dimension"] -_REQUIRED_VARS_TEMP = ["ccpp_error_code", "ccpp_error_message", - "potential_temperature", - "potential_temperature_at_interface", - "coefficients_for_interpolation", - "potential_temperature_increment", - "surface_air_pressure", "time_step_for_physics", - "water_vapor_specific_humidity", - "soil_levels", - "temperature_at_diagnostic_levels", - "array_variable_for_testing"] -_INPUT_VARS_TEMP = ["potential_temperature", - "potential_temperature_at_interface", - "coefficients_for_interpolation", - "potential_temperature_increment", - "surface_air_pressure", "time_step_for_physics", - "water_vapor_specific_humidity", - "soil_levels", - "temperature_at_diagnostic_levels", - "array_variable_for_testing"] -_OUTPUT_VARS_TEMP = ["ccpp_error_code", "ccpp_error_message", - "potential_temperature", - "potential_temperature_at_interface", - "coefficients_for_interpolation", - "surface_air_pressure", "water_vapor_specific_humidity", - "soil_levels", - "temperature_at_diagnostic_levels", - "array_variable_for_testing"] - - -class TestCapgenHostDataTables(unittest.TestCase, BaseTests.TestHostDataTables): - database = _DATABASE - host_files = _HOST_FILES - suite_files = _SUITE_FILES - utility_files = _UTILITY_FILES - ccpp_files = _CCPP_FILES - process_list = _PROCESS_LIST - module_list = _MODULE_LIST - dependencies = _DEPENDENCIES - suite_list = _SUITE_LIST - - -class CommandLineCapgenHostDatafileRequiredFiles(unittest.TestCase, BaseTests.TestHostCommandLineDataFiles): - database = _DATABASE - host_files = _HOST_FILES - suite_files = _SUITE_FILES - utility_files = _UTILITY_FILES - ccpp_files = _CCPP_FILES - process_list = _PROCESS_LIST - module_list = _MODULE_LIST - dependencies = _DEPENDENCIES - suite_list = _SUITE_LIST - datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" - - -class TestCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuite): - database = _DATABASE - required_vars = _REQUIRED_VARS_DDT - input_vars = _INPUT_VARS_DDT - output_vars = _OUTPUT_VARS_DDT - suite_name = "ddt_suite" - - -class CommandLineCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuiteCommandLine): - database = _DATABASE - required_vars = _REQUIRED_VARS_DDT - input_vars = _INPUT_VARS_DDT - output_vars = _OUTPUT_VARS_DDT - suite_name = "ddt_suite" - datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" - - -class TestCapgenTempSuite(unittest.TestCase, BaseTests.TestSuiteExcludeProtected): - database = _DATABASE - required_vars = _REQUIRED_VARS_TEMP + _PROT_VARS_TEMP - input_vars = _INPUT_VARS_TEMP + _PROT_VARS_TEMP - required_vars_excluding_protected = _REQUIRED_VARS_TEMP - input_vars_excluding_protected = _INPUT_VARS_TEMP - output_vars = _OUTPUT_VARS_TEMP - suite_name = "temp_suite" - - -class CommandLineCapgenTempSuite(unittest.TestCase, BaseTests.TestSuiteExcludeProtectedCommandLine): - database = _DATABASE - required_vars = _REQUIRED_VARS_TEMP + _PROT_VARS_TEMP - input_vars = _INPUT_VARS_TEMP + _PROT_VARS_TEMP - required_vars_excluding_protected = _REQUIRED_VARS_TEMP - input_vars_excluding_protected = _INPUT_VARS_TEMP - output_vars = _OUTPUT_VARS_TEMP - suite_name = "temp_suite" - datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" diff --git a/test/capgen_test/ddt2.F90 b/test/capgen_test/ddt2.F90 deleted file mode 100644 index ce560846..00000000 --- a/test/capgen_test/ddt2.F90 +++ /dev/null @@ -1,12 +0,0 @@ -module ddt2 - - use ccpp_kinds, only: kind_phys - - implicit none - - type ty_ddt2 - integer :: foo - real(kind=kind_phys) :: bar - end type ty_ddt2 - -end module ddt2 diff --git a/test/capgen_test/ddt_suite.xml b/test/capgen_test/ddt_suite.xml deleted file mode 100644 index 749bb3bc..00000000 --- a/test/capgen_test/ddt_suite.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - make_ddt - environ_conditions - - diff --git a/test/capgen_test/environ_conditions.meta b/test/capgen_test/environ_conditions.meta deleted file mode 100644 index f87e5039..00000000 --- a/test/capgen_test/environ_conditions.meta +++ /dev/null @@ -1,111 +0,0 @@ -[ccpp-table-properties] - name = environ_conditions - type = scheme - source_path = source_dir1 -[ccpp-arg-table] - name = environ_conditions_run - type = scheme -[ psurf ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = environ_conditions_init - type = scheme -[ nbox ] - standard_name = horizontal_dimension - type = integer - units = count - dimensions = () - intent = in -[ o3 ] - standard_name = ozone - units = ppmv - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = out -[ hno3 ] - standard_name = nitric_acid - units = ppmv - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = out -[ ntimes ] - standard_name = number_of_model_times - type = integer - units = count - dimensions = () - intent = out -[ model_times ] - standard_name = model_times - units = seconds - dimensions = (number_of_model_times) - type = integer - intent = out - allocatable = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = environ_conditions_finalize - type = scheme -[ ntimes ] - standard_name = number_of_model_times - type = integer - units = count - dimensions = () - intent = in -[ model_times ] - standard_name = model_times - units = seconds - dimensions = (number_of_model_times) - type = integer - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/capgen_test/make_ddt.F90 b/test/capgen_test/make_ddt.F90 deleted file mode 100644 index 429e8939..00000000 --- a/test/capgen_test/make_ddt.F90 +++ /dev/null @@ -1,142 +0,0 @@ -!Hello demonstration parameterization -! - -module make_ddt - - use ccpp_kinds, only: kind_phys - use ddt2, only: ty_ddt2 - - implicit none - private - - public :: make_ddt_init - public :: make_ddt_run - public :: make_ddt_timestep_final - public :: vmr_type - - type ty_ddt3 - integer :: dont_lose - integer :: your_head - integer :: to_gain_a_minute - integer :: you_need_your_head - integer :: your_brains_are_in_it - end type ty_ddt3 - - !> \section arg_table_vmr_type Argument Table - !! \htmlinclude arg_table_vmr_type.html - !! - type vmr_type - integer :: nvmr - real(kind=kind_phys), allocatable :: vmr_array(:, :) - type(ty_ddt2) :: error_maybe - type(ty_ddt3) :: burma_shave - end type vmr_type - -contains - - !> \section arg_table_make_ddt_run Argument Table - !! \htmlinclude arg_table_make_ddt_run.html - !! - subroutine make_ddt_run(cols, cole, o3, hno3, vmr, errmsg, errflg) - !---------------------------------------------------------------- - implicit none - !---------------------------------------------------------------- - - ! Dummy arguments - integer, intent(in) :: cols - integer, intent(in) :: cole - real(kind=kind_phys), intent(in) :: o3(:) - real(kind=kind_phys), intent(in) :: hno3(:) - type(vmr_type), intent(inout) :: vmr - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - ! Local variable - integer :: nbox - !---------------------------------------------------------------- - - errmsg = '' - errflg = 0 - - ! Check for correct threading behavior - nbox = cole - cols + 1 - if (size(o3) /= nbox) then - errflg = 1 - write(errmsg, '(2(a,i0))') 'SIZE(O3) = ', size(o3), ', should be ', nbox - else if (size(hno3) /= nbox) then - errflg = 1 - write(errmsg, '(2(a,i0))') 'SIZE(HNO3) = ', size(hno3), & - ', should be ', nbox - else - ! NOTE -- This is prototyping one approach to passing a large number of - ! chemical VMR values and is the predecssor for adding in methods and - ! maybe nesting DDTs (especially for aerosols) - vmr%vmr_array(cols:cole, 1) = o3(:) - vmr%vmr_array(cols:cole, 2) = hno3(:) - end if - - end subroutine make_ddt_run - - !> \section arg_table_make_ddt_init Argument Table - !! \htmlinclude arg_table_make_ddt_init.html - !! - subroutine make_ddt_init(nbox, vmr, errmsg, errflg) - - ! Dummy arguments - integer, intent(in) :: nbox - type(vmr_type), intent(out) :: vmr - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine initializes the vmr array - vmr%nvmr = 2 - allocate(vmr%vmr_array(nbox, vmr%nvmr)) - - errmsg = '' - errflg = 0 - - end subroutine make_ddt_init - - !> \section arg_table_make_ddt_timestep_final Argument Table - !! \htmlinclude arg_table_make_ddt_timestep_final.html - !! - subroutine make_ddt_timestep_final(ncols, vmr, errmsg, errflg) - - ! Dummy arguments - integer, intent(in) :: ncols - type(vmr_type), intent(in) :: vmr - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - ! Local variables - integer :: index - real(kind=kind_phys) :: rind - - errmsg = '' - errflg = 0 - - ! This routine checks the array values in vmr - if (size(vmr%vmr_array, 1) /= ncols) then - errflg = 1 - write(errmsg, '(2(a,i0))') 'VMR%VMR_ARRAY first dimension size is, ', & - size(vmr%vmr_array, 1), ', should be, ', ncols - else - do index = 1, ncols - rind = real(index, kind_phys) - if (vmr%vmr_array(index, 1) /= rind * 1.e-6_kind_phys) then - errflg = 1 - write(errmsg, '(a,i0,2(a,e12.4))') 'O3(', index, ') = ', & - vmr%vmr_array(index, 1), ', should be, ', & - rind * 1.e-6_kind_phys - exit - else if (vmr%vmr_array(index, 2) /= rind * 1.e-9_kind_phys) then - errflg = 1 - write(errmsg, '(a,i0,2(a,e12.4))') 'HNO3(', index, ') = ', & - vmr%vmr_array(index, 2), ', should be, ', & - rind * 1.e-9_kind_phys - exit - end if - end do - end if - - end subroutine make_ddt_timestep_final - -end module make_ddt diff --git a/test/capgen_test/make_ddt.meta b/test/capgen_test/make_ddt.meta deleted file mode 100644 index 2f3eeaa5..00000000 --- a/test/capgen_test/make_ddt.meta +++ /dev/null @@ -1,128 +0,0 @@ -[ccpp-table-properties] - name = vmr_type - type = ddt - dependencies = ddt2 -[ccpp-arg-table] - name = vmr_type - type = ddt -[ nvmr ] - standard_name = number_of_chemical_species - units = count - dimensions = () - type = integer -[ vmr_array ] - standard_name = array_of_volume_mixing_ratios - units = ppmv - dimensions = (horizontal_dimension, number_of_chemical_species) - type = real - kind = kind_phys -[ccpp-table-properties] - name = make_ddt - type = scheme -[ccpp-arg-table] - name = make_ddt_run - type = scheme -[ cols ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - intent = in -[ cole ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - intent = in -[ O3 ] - standard_name = ozone - units = ppmv - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ HNO3 ] - standard_name = nitric_acid - units = ppmv - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = vmr_type - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = make_ddt_init - type = scheme -[ nbox ] - standard_name = horizontal_dimension - type = integer - units = count - dimensions = () - intent = in -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = vmr_type - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = make_ddt_timestep_final - type = scheme -[ ncols ] - standard_name = horizontal_dimension - type = integer - units = count - dimensions = () - intent = in -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = vmr_type - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/capgen_test/setup_coeffs.F90 b/test/capgen_test/setup_coeffs.F90 deleted file mode 100644 index 09c7fcc1..00000000 --- a/test/capgen_test/setup_coeffs.F90 +++ /dev/null @@ -1,24 +0,0 @@ -module setup_coeffs - use ccpp_kinds, only: kind_phys - implicit none - - public :: setup_coeffs_timestep_init - -contains - !> \section arg_table_setup_coeffs_timestep_init Argument Table - !! \htmlinclude arg_table_setup_coeffs_timestep_init.html - !! - subroutine setup_coeffs_timestep_init(coeffs, errmsg, errflg) - - real(kind=kind_phys), intent(inout) :: coeffs(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - coeffs(:) = 1._kind_phys - - end subroutine setup_coeffs_timestep_init - -end module setup_coeffs diff --git a/test/capgen_test/setup_coeffs.meta b/test/capgen_test/setup_coeffs.meta deleted file mode 100644 index 8d0fc5f4..00000000 --- a/test/capgen_test/setup_coeffs.meta +++ /dev/null @@ -1,29 +0,0 @@ -[ccpp-table-properties] - name = setup_coeffs - type = scheme -[ccpp-arg-table] - name = setup_coeffs_timestep_init - type = scheme -[ coeffs ] - standard_name = coefficients_for_interpolation - long_name = coefficients for interpolation - units = none - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/capgen_test/source_dir1/environ_conditions.F90 b/test/capgen_test/source_dir1/environ_conditions.F90 deleted file mode 100644 index 2d63366e..00000000 --- a/test/capgen_test/source_dir1/environ_conditions.F90 +++ /dev/null @@ -1,96 +0,0 @@ -module environ_conditions - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: environ_conditions_init - public :: environ_conditions_run - public :: environ_conditions_finalize - - integer, parameter :: input_model_times = 3 - integer, parameter :: input_model_values(input_model_times) = (/ 31, 37, 41 /) - -contains - - !> \section arg_table_environ_conditions_run Argument Table - !! \htmlinclude arg_table_environ_conditions_run.html - !! - subroutine environ_conditions_run(psurf, errmsg, errflg) - - ! This routine currently does nothing -- should update values - - real(kind=kind_phys), intent(in) :: psurf(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - end subroutine environ_conditions_run - - !> \section arg_table_environ_conditions_init Argument Table - !! \htmlinclude arg_table_environ_conditions_init.html - !! - subroutine environ_conditions_init(nbox, o3, hno3, ntimes, model_times, & - errmsg, errflg) - - integer, intent(in) :: nbox - real(kind=kind_phys), intent(out) :: o3(:) - real(kind=kind_phys), intent(out) :: hno3(:) - integer, intent(out) :: ntimes - integer, allocatable, intent(out) :: model_times(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: i, j - - errmsg = '' - errflg = 0 - - ! This may be replaced with MusicBox json environmental conditions reader??? - - do i = 1, nbox - o3(i) = real(i, kind_phys) * 1.e-6_kind_phys - hno3(i) = real(i, kind_phys) * 1.e-9_kind_phys - end do - - ntimes = input_model_times - allocate(model_times(ntimes)) - model_times = input_model_values - - end subroutine environ_conditions_init - - !> \section arg_table_environ_conditions_finalize Argument Table - !! \htmlinclude arg_table_environ_conditions_finalize.html - !! - subroutine environ_conditions_finalize(ntimes, model_times, errmsg, errflg) - - integer, intent(in) :: ntimes - integer, intent(in) :: model_times(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine checks the size and values of model_times - if (ntimes /= input_model_times) then - errflg = 1 - write(errmsg, '(2(a,i0))') 'ntimes mismatch, ', ntimes, ' should be ', & - input_model_times - else if (size(model_times) /= input_model_times) then - errflg = 1 - write(errmsg, '(2(a,i0))') 'model_times size mismatch, ', & - size(model_times), ' should be ', input_model_times - else if (any(model_times /= input_model_values)) then - errflg = 1 - write(errmsg, *) 'model_times mismatch, ', & - model_times, ' should be ', input_model_values - else - errmsg = '' - errflg = 0 - end if - - end subroutine environ_conditions_finalize - -end module environ_conditions diff --git a/test/capgen_test/source_dir2/temp_set.F90 b/test/capgen_test/source_dir2/temp_set.F90 deleted file mode 100644 index be54b80c..00000000 --- a/test/capgen_test/source_dir2/temp_set.F90 +++ /dev/null @@ -1,124 +0,0 @@ -!Test 3D parameterization -! - -module temp_set - - use ccpp_kinds, only: kind_phys, & - kind_temp - - implicit none - private - - public :: temp_set_init - public :: temp_set_timestep_initialize - public :: temp_set_run - public :: temp_set_finalize - -contains - - !> \section arg_table_temp_set_run Argument Table - !! \htmlinclude arg_table_temp_set_run.html - !! - subroutine temp_set_run(ncol, lev, timestep, temp_level, temp_diag, temp, ps, & - to_promote, promote_pcnst, slev_lbound, soil_levs, var_array, errmsg, errflg) - !---------------------------------------------------------------- - implicit none - !---------------------------------------------------------------- - - integer, intent(in) :: ncol, lev, slev_lbound - real(kind=kind_phys), intent(out) :: temp(:, :) - real(kind=kind_phys), intent(in) :: timestep - real(kind=kind_phys), intent(in) :: ps(:) - real(kind=kind_phys), intent(inout) :: temp_level(:, :) - real(kind=kind_phys), intent(inout) :: temp_diag(:, :) - real(kind=kind_phys), intent(inout) :: soil_levs(slev_lbound:) - real(kind=kind_phys), intent(inout) :: var_array(:, :, :, :) - real(kind=kind_temp), intent(out) :: to_promote(:, :) - real(kind=kind_phys), intent(out) :: promote_pcnst(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - integer :: ilev - - integer :: col_index - integer :: lev_index - real(kind=kind_phys) :: internal_scalar_var - - errmsg = '' - errflg = 0 - - ilev = size(temp_level, 2) - if (ilev /= (lev + 1)) then - errflg = 1 - errmsg = 'Invalid value for ilev, must be lev+1' - return - end if - - do col_index = 1, ncol - do lev_index = 1, lev - temp(col_index, lev_index) = (temp_level(col_index, lev_index) & - + temp_level(col_index, lev_index + 1)) / 2.0_kind_phys - end do - end do - - var_array(:, :, :, :) = 1._kind_phys - - ! - internal_scalar_var = soil_levs(slev_lbound) - internal_scalar_var = soil_levs(0) - - end subroutine temp_set_run - - !> \section arg_table_temp_set_init Argument Table - !! \htmlinclude arg_table_temp_set_init.html - !! - subroutine temp_set_init(temp_inc_in, fudge, temp_inc_set, errmsg, errflg) - - real(kind=kind_phys), intent(in) :: temp_inc_in - real(kind=kind_phys), intent(in) :: fudge - real(kind=kind_phys), intent(out) :: temp_inc_set - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - temp_inc_set = temp_inc_in - - errmsg = '' - errflg = 0 - - end subroutine temp_set_init - - !> \section arg_table_temp_set_timestep_initialize Argument Table - !! \htmlinclude arg_table_temp_set_timestep_initialize.html - !! - subroutine temp_set_timestep_initialize(ncol, temp_inc, temp_level, & - errmsg, errflg) - - integer, intent(in) :: ncol - real(kind=kind_phys), intent(in) :: temp_inc - real(kind=kind_phys), intent(inout) :: temp_level(:, :) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - temp_level = temp_level + temp_inc - - end subroutine temp_set_timestep_initialize - - !> \section arg_table_temp_set_finalize Argument Table - !! \htmlinclude arg_table_temp_set_finalize.html - !! - subroutine temp_set_finalize(errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_set_finalize - -end module temp_set diff --git a/test/capgen_test/temp_adjust.F90 b/test/capgen_test/temp_adjust.F90 deleted file mode 100644 index e8ac281d..00000000 --- a/test/capgen_test/temp_adjust.F90 +++ /dev/null @@ -1,127 +0,0 @@ -! Test parameterization with no vertical level -! - -module temp_adjust - - use ccpp_kinds, only: kind_phys, & - kind_temp - - implicit none - private - - public :: temp_adjust_register - public :: temp_adjust_init - public :: temp_adjust_run - public :: temp_adjust_finalize - - logical :: module_level_config = .false. - -contains - - !> \section arg_table_temp_adjust_register Argument Table - !! \htmlinclude arg_table_temp_adjust_register.hml - !! - subroutine temp_adjust_register(config_var, errmsg, errflg) - logical, intent(in) :: config_var - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - module_level_config = config_var - errflg = 0 - errmsg = '' - - end subroutine temp_adjust_register - - !> \section arg_table_temp_adjust_run Argument Table - !! \htmlinclude arg_table_temp_adjust_run.html - !! - subroutine temp_adjust_run(foo, timestep, interstitial_var, temp_prev, temp_layer, qv, ps, & - to_promote, promote_pcnst, errmsg, errflg, innie, outie, optsie) - - integer, intent(in) :: foo - real(kind=kind_phys), intent(in) :: timestep - real(kind=kind_phys), intent(inout), optional :: qv(:, :) - real(kind=kind_phys), intent(inout) :: ps(:) - ! codee format off - REAL(kind_phys), intent(in) :: temp_prev(:,:) - REAL(kind_phys), intent(inout) :: temp_layer(:,:) - ! codee format on - real(kind=kind_temp), intent(in) :: to_promote(:, :) - real(kind=kind_phys), intent(in) :: promote_pcnst(:) - integer, intent(out) :: interstitial_var(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - real(kind=kind_phys), optional, intent(in) :: innie - real(kind=kind_phys), optional, intent(out) :: outie - real(kind=kind_phys), optional, intent(inout) :: optsie - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - interstitial_var = 6 - if (size(interstitial_var) /= 3) then - errflg = 1 - errmsg = 'interstitial variable not allocated properly!' - return - end if - - if (.not. module_level_config) then - ! do nothing - return - end if - - do col_index = 1, foo - temp_layer(col_index, :) = temp_layer(col_index, :) + temp_prev(col_index, :) - if (present(qv)) qv(col_index, :) = qv(col_index, :) + 1.0_kind_phys - end do - if (present(innie) .and. present(outie) .and. present(optsie)) then - outie = innie * optsie - optsie = optsie + 1.0_kind_phys - end if - - end subroutine temp_adjust_run - - !> \section arg_table_temp_adjust_init Argument Table - !! \htmlinclude arg_table_temp_adjust_init.html - !! - subroutine temp_adjust_init(errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_adjust_init - - !> \section arg_table_temp_adjust_finalize Argument Table - !! \htmlinclude arg_table_temp_adjust_finalize.html - !! - subroutine temp_adjust_finalize(interstitial_var, errmsg, errflg) - - integer, intent(in) :: interstitial_var(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - if (size(interstitial_var) /= 3) then - errflg = 1 - errmsg = 'interstitial variable not allocated properly!' - return - end if - if (interstitial_var(1) /= 6) then - errflg = 1 - errmsg = 'interstitial variable not set properly!' - end if - - end subroutine temp_adjust_finalize - -end module temp_adjust diff --git a/test/capgen_test/temp_adjust.meta b/test/capgen_test/temp_adjust.meta deleted file mode 100644 index 63e7fcc1..00000000 --- a/test/capgen_test/temp_adjust.meta +++ /dev/null @@ -1,156 +0,0 @@ -[ccpp-table-properties] - name = temp_adjust - type = scheme - kind_spec = temp_kinds:kind_temp=>temp_r8 - dependencies = qux.F90, temp_kinds.F90 - dependencies_path = adjust -[ccpp-arg-table] - name = temp_adjust_register - type = scheme -[ config_var ] - standard_name = configuration_variable - type = logical - units = none - dimensions = () - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_adjust_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ interstitial_var ] - standard_name = output_only_interstitial_variable - units = 1 - dimensions = (dimension_for_interstitial_variable) - type = integer - intent = out -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout - diagnostic_name = temperature -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout - diagnostic_name_fixed = Q - optional = True -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = inout -[ to_promote ] - standard_name = promote_this_variable_to_suite - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_temp - intent = in -[ promote_pcnst ] - standard_name = promote_this_variable_with_no_horizontal_dimension - units = K - dimensions = (number_of_tracers) - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_adjust_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_adjust_finalize - type = scheme -[ interstitial_var ] - standard_name = output_only_interstitial_variable - units = 1 - dimensions = (dimension_for_interstitial_variable) - type = integer - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/capgen_test/temp_calc_adjust.F90 b/test/capgen_test/temp_calc_adjust.F90 deleted file mode 100644 index 7c423669..00000000 --- a/test/capgen_test/temp_calc_adjust.F90 +++ /dev/null @@ -1,111 +0,0 @@ -!Test parameterization with no vertical level and hanging intent(out) variable -! - -module temp_calc_adjust - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: temp_calc_adjust_register - public :: temp_calc_adjust_init - public :: temp_calc_adjust_run - public :: temp_calc_adjust_finalize - -contains - -! codee format off -!> \section arg_table_temp_calc_adjust_register Argument Table -!! \htmlinclude arg_table_temp_calc_adjust_register.html -!! - SUBROUTINE temp_calc_adjust_register(dim_inter, errmsg, errflg) -! codee format on - integer, intent(out) :: dim_inter - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errflg = 0 - errmsg = '' - dim_inter = 3 - end subroutine temp_calc_adjust_register - - !> \section arg_table_temp_calc_adjust_run Argument Table - !! \htmlinclude arg_table_temp_calc_adjust_run.html - !! - subroutine temp_calc_adjust_run(nbox, timestep, temp_level, temp_calc, & - errmsg, errflg) - - integer, intent(in) :: nbox - real(kind=kind_phys), intent(in) :: timestep - real(kind=kind_phys), intent(in) :: temp_level(:, :) - real(kind=kind_phys), intent(out) :: temp_calc(:, :) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - real(kind=kind_phys) :: bar = 1.0_kind_phys - - errmsg = '' - errflg = 0 - - call temp_calc_adjust_nested_subroutine(temp_calc) - if (check_foo()) then - call foo(bar) - end if - - contains - - elemental subroutine temp_calc_adjust_nested_subroutine(temp) - - real(kind=kind_phys), intent(out) :: temp - !------------------------------------------------------------- - - temp = 1.0_kind_phys - - end subroutine temp_calc_adjust_nested_subroutine - - subroutine foo(bar) - real(kind=kind_phys), intent(inout) :: bar - bar = bar + 1.0_kind_phys - - end subroutine foo - - logical function check_foo() - check_foo = .true. - end function check_foo - - end subroutine temp_calc_adjust_run - - !> \section arg_table_temp_calc_adjust_init Argument Table - !! \htmlinclude arg_table_temp_calc_adjust_init.html - !! - subroutine temp_calc_adjust_init(errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_calc_adjust_init - - !> \section arg_table_temp_calc_adjust_finalize Argument Table - !! \htmlinclude arg_table_temp_calc_adjust_finalize.html - !! - subroutine temp_calc_adjust_finalize(errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_calc_adjust_finalize - -end module temp_calc_adjust diff --git a/test/capgen_test/temp_calc_adjust.meta b/test/capgen_test/temp_calc_adjust.meta deleted file mode 100644 index f795da63..00000000 --- a/test/capgen_test/temp_calc_adjust.meta +++ /dev/null @@ -1,111 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = foo.F90, bar.F90 -[ccpp-arg-table] - name = temp_calc_adjust_register - type = scheme -[ dim_inter ] - standard_name = dimension_for_interstitial_variable - type = integer - units = count - dimensions = () - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme - process = adjusting -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_level ] - standard_name = potential_temperature_at_interface - units = K - dimensions = (ccpp_constant_one:horizontal_loop_extent, vertical_interface_dimension) - type = real - kind = kind_phys - intent = in -[ temp_calc ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_calc_adjust_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_calc_adjust_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/capgen_test/temp_set.meta b/test/capgen_test/temp_set.meta deleted file mode 100644 index 42bbb194..00000000 --- a/test/capgen_test/temp_set.meta +++ /dev/null @@ -1,214 +0,0 @@ -[ccpp-table-properties] - name = temp_set - type = scheme - source_path = source_dir2 - kind_spec = temp_kinds:kind_temp=>temp_r8 - dependencies = temp_kinds.F90 - dependencies_path = adjust -[ccpp-arg-table] - name = temp_set_run - type = scheme - process = setter -[ ncol ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ lev ] - standard_name = vertical_layer_dimension - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_level ] - standard_name = potential_temperature_at_interface - units = K - dimensions = (ccpp_constant_one:horizontal_loop_extent, vertical_interface_dimension) - type = real - kind = kind_phys - intent = inout -[ temp_diag ] - standard_name = temperature_at_diagnostic_levels - units = K - dimensions = (horizontal_loop_extent, 6) - type = real - kind = kind_phys - intent = inout -[ temp ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = out -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = in -[ to_promote ] - standard_name = promote_this_variable_to_suite - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_temp - intent = out -[ promote_pcnst ] - standard_name = promote_this_variable_with_no_horizontal_dimension - units = K - dimensions = (number_of_tracers) - type = real - kind = kind_phys - intent = out -[ slev_lbound ] - standard_name = lower_bound_of_vertical_dimension_of_soil - type = integer - units = count - dimensions = () - intent = in -[ soil_levs ] - standard_name = soil_levels - long_name = soil levels - units = cm - dimensions = (lower_bound_of_vertical_dimension_of_soil:upper_bound_of_vertical_dimension_of_soil) - type = real - kind = kind_phys - intent = inout -[ var_array ] - standard_name = array_variable_for_testing - long_name = array variable for testing - units = none - dimensions = (horizontal_loop_extent,2,4,6) - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -# Init -[ccpp-arg-table] - name = temp_set_init - type = scheme -[ temp_inc_in ] - standard_name = potential_temperature_increment - long_name = Per time step potential temperature increment - units = K - dimensions = () - type = real - kind = kind_phys - intent = in -[ fudge ] - standard_name = random_fudge_factor - long_name = Ignore this - units = 1 - dimensions = () - type = real - kind = kind_phys - intent = in - default_value = 1.0_kind_phys -[ temp_inc_set ] - standard_name = test_potential_temperature_increment - long_name = Per time step potential temperature increment - units = K - dimensions = () - type = real - kind = kind_phys - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -# Timestep Initialization -[ccpp-arg-table] - name = temp_set_timestep_initialize - type = scheme -[ ncol ] - standard_name = horizontal_dimension - type = integer - units = count - dimensions = () - intent = in -[ temp_inc ] - standard_name = test_potential_temperature_increment - long_name = Per time step potential temperature increment - units = K - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_level ] - standard_name = potential_temperature_at_interface - units = K - dimensions = (horizontal_dimension, vertical_interface_dimension) - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -# Finalize -[ccpp-arg-table] - name = temp_set_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/capgen_test/temp_suite.xml b/test/capgen_test/temp_suite.xml deleted file mode 100644 index 7a4795c4..00000000 --- a/test/capgen_test/temp_suite.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - setup_coeffs - temp_set - - - temp_calc_adjust - temp_adjust - - diff --git a/test/capgen_test/test_capgen_host_integration.F90 b/test/capgen_test/test_capgen_host_integration.F90 deleted file mode 100644 index 7f964178..00000000 --- a/test/capgen_test/test_capgen_host_integration.F90 +++ /dev/null @@ -1,89 +0,0 @@ -program test - use test_prog, only: test_host, & - suite_info, & - cm, & - cs - - implicit none - - character(len=cs), target :: test_parts1(2) = (/ 'physics1 ', & - 'physics2 ' /) - character(len=cs), target :: test_parts2(1) = (/ 'data_prep ' /) - character(len=cm), target :: test_invars1(10) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'potential_temperature_increment ', & - 'soil_levels ', & - 'temperature_at_diagnostic_levels ', & - 'time_step_for_physics ', & - 'array_variable_for_testing ' /) - character(len=cm), target :: test_outvars1(10) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'soil_levels ', & - 'temperature_at_diagnostic_levels ', & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'array_variable_for_testing ' /) - character(len=cm), target :: test_reqvars1(12) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'potential_temperature_increment ', & - 'time_step_for_physics ', & - 'soil_levels ', & - 'temperature_at_diagnostic_levels ', & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'array_variable_for_testing ' /) - - character(len=cm), target :: test_invars2(3) = (/ & - 'model_times ', & - 'number_of_model_times ', & - 'surface_air_pressure ' /) - - character(len=cm), target :: test_outvars2(5) = (/ & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'model_times ', & - 'surface_air_pressure ', & - 'number_of_model_times ' /) - - character(len=cm), target :: test_reqvars2(5) = (/ & - 'model_times ', & - 'number_of_model_times ', & - 'surface_air_pressure ', & - 'ccpp_error_code ', & - 'ccpp_error_message ' /) - type(suite_info) :: test_suites(2) - logical :: run_okay - - ! Setup expected test suite info - test_suites(1)%suite_name = 'temp_suite' - test_suites(1)%suite_parts => test_parts1 - test_suites(1)%suite_input_vars => test_invars1 - test_suites(1)%suite_output_vars => test_outvars1 - test_suites(1)%suite_required_vars => test_reqvars1 - test_suites(2)%suite_name = 'ddt_suite' - test_suites(2)%suite_parts => test_parts2 - test_suites(2)%suite_input_vars => test_invars2 - test_suites(2)%suite_output_vars => test_outvars2 - test_suites(2)%suite_required_vars => test_reqvars2 - - call test_host(run_okay, test_suites) - - if (run_okay) then - stop 0 - else - stop -1 - end if - -end program test diff --git a/test/capgen_test/test_host.F90 b/test/capgen_test/test_host.F90 deleted file mode 100644 index 258f0d91..00000000 --- a/test/capgen_test/test_host.F90 +++ /dev/null @@ -1,306 +0,0 @@ -module test_prog - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public test_host - - ! Public data and interfaces - integer, public, parameter :: cs = 16 - integer, public, parameter :: cm = 36 - - !> \section arg_table_suite_info Argument Table - !! \htmlinclude arg_table_suite_info.html - !! - type, public :: suite_info - character(len=cs) :: suite_name = '' - character(len=cs), pointer :: suite_parts(:) => null() - character(len=cm), pointer :: suite_input_vars(:) => null() - character(len=cm), pointer :: suite_output_vars(:) => null() - character(len=cm), pointer :: suite_required_vars(:) => null() - end type suite_info - -contains - - logical function check_suite(test_suite) - use test_host_ccpp_cap, only: ccpp_physics_suite_part_list - use test_host_ccpp_cap, only: ccpp_physics_suite_variables - use test_utils, only: check_list - - ! Dummy argument - type(suite_info), intent(in) :: test_suite - ! Local variables - integer :: sind - logical :: check - integer :: errflg - character(len=512) :: errmsg - character(len=128), allocatable :: test_list(:) - - check_suite = .true. - write(6, *) "Checking suite ", trim(test_suite%suite_name) - ! First, check the suite parts - call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_parts, 'part names', & - suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check the input variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.true., output_vars=.false.) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_input_vars, & - 'input variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check the output variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.false., output_vars=.true.) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_output_vars, & - 'output variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check all required variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_required_vars, & - 'required variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - end function check_suite - - !> \section arg_table_test_host Argument Table - !! \htmlinclude arg_table_test_host.html - !! - subroutine test_host(retval, test_suites) - -#ifdef _OPENMP - use omp_lib -#endif - use test_host_mod, only: ncols, & - num_time_steps - use test_host_ccpp_cap, only: test_host_ccpp_physics_register - use test_host_ccpp_cap, only: test_host_ccpp_physics_initialize - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_initial - use test_host_ccpp_cap, only: test_host_ccpp_physics_run - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_final - use test_host_ccpp_cap, only: test_host_ccpp_physics_finalize - use test_host_ccpp_cap, only: ccpp_physics_suite_list - use test_host_mod, only: init_data, & - compare_data, & - check_model_times - use test_utils, only: check_list - - type(suite_info), intent(in) :: test_suites(:) - logical, intent(out) :: retval - - logical :: check - integer :: col_start, col_end - integer :: thread_num, num_threads - integer :: index, sind - integer :: time_step - integer :: num_suites - character(len=128), allocatable :: suite_names(:) - character(len=512) :: errmsg - integer :: errflg - - ! Initialize our 'data' - call init_data() - - ! Gather and test the inspection routines - num_suites = size(test_suites) - call ccpp_physics_suite_list(suite_names) - retval = check_list(suite_names, test_suites(:)%suite_name, & - 'suite names') - write(6, *) 'Available suites are:' - do index = 1, size(suite_names) - do sind = 1, num_suites - if (trim(test_suites(sind)%suite_name) == & - trim(suite_names(index))) then - exit - end if - end do - write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & - ' = test_suites(', sind, ')' - end do - if (retval) then - do sind = 1, num_suites - check = check_suite(test_suites(sind)) - retval = retval .and. check - end do - end if - !!! Return here if any check failed - if (.not. retval) then - return - end if - - ! Use the suite information to call the register phase - do sind = 1, num_suites - call test_host_ccpp_physics_register(test_suites(sind)%suite_name, & - errmsg, errflg) - if (errflg /= 0) then - write(6, '(4a)') 'ERROR in register of ', & - trim(test_suites(sind)%suite_name), ': ', trim(errmsg) - end if - end do - ! Use the suite information to setup the run - do sind = 1, num_suites - call test_host_ccpp_physics_initialize(test_suites(sind)%suite_name, & - errmsg, errflg) - if (errflg /= 0) then - write(6, '(4a)') 'ERROR in initialize of ', & - trim(test_suites(sind)%suite_name), ': ', trim(errmsg) - end if - end do - ! Loop over time steps - do time_step = 1, num_time_steps - ! Initialize the timestep - do sind = 1, num_suites - if (errflg /= 0) then - exit - end if - if (errflg == 0) then - call test_host_ccpp_physics_timestep_initial( & - test_suites(sind)%suite_name, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(errmsg) - exit - end if - if (errflg /= 0) then - exit - end if - end do - - run_phase_if_no_error: if (errflg == 0) then -#ifdef _OPENMP - num_threads = omp_get_max_threads() -#else - num_threads = 1 -#endif - !$OMP parallel num_threads (num_threads) & - !$OMP default (none) & - !$OMP shared (num_threads, num_suites, test_suites) & - !$OMP private (thread_num, col_start, col_end, errmsg) & - !$OMP reduction (+:errflg) -#ifdef _OPENMP - thread_num = omp_get_thread_num() -#else - thread_num = 0 -#endif - !$OMP do - do col_start = 1, ncols, 5 - if (errflg /= 0) then - continue - end if - col_end = min(col_start + 4, ncols) - do sind = 1, num_suites - if (errflg /= 0) then - continue - end if - do index = 1, size(test_suites(sind)%suite_parts) - if (errflg /= 0) then - continue - end if - write(0, '(a,i0,a,i0,5a,i0,a,i0)') 'Thread ', thread_num, '/', num_threads, & - ': calling run phase for suite ', trim(test_suites(sind)%suite_name), & - ' part ', trim(test_suites(sind)%suite_parts(index)), & - ' columns ', col_start, ':', col_end - call test_host_ccpp_physics_run( & - test_suites(sind)%suite_name, & - test_suites(sind)%suite_parts(index), & - col_start, col_end, errmsg, errflg) - if (errflg /= 0) then - write(6, '(5a)') trim(test_suites(sind)%suite_name), & - '/', trim(test_suites(sind)%suite_parts(index)), & - ': ', trim(errmsg) - end if - end do - end do - end do - !$OMP end do - !$OMP end parallel - end if run_phase_if_no_error - - do sind = 1, num_suites - if (errflg /= 0) then - exit - end if - if (errflg == 0) then - call test_host_ccpp_physics_timestep_final( & - test_suites(sind)%suite_name, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(errmsg) - exit - end if - end do - end do ! End time step loop - - do sind = 1, num_suites - if (errflg /= 0) then - exit - end if - if (errflg == 0) then - call test_host_ccpp_physics_finalize( & - test_suites(sind)%suite_name, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & - trim(errmsg) - write(6, '(2a)') 'An error occurred in ccpp_timestep_final, ', & - 'Exiting...' - exit - end if - end do - - if (errflg == 0) then - ! Run finished without error, check answers - if (.not. check_model_times()) then - write(6, *) 'Model times error!' - errflg = -1 - else if (compare_data()) then - write(6, *) 'Answers are correct!' - errflg = 0 - else - write(6, *) 'Answers are not correct!' - errflg = -1 - end if - end if - - retval = errflg == 0 - - end subroutine test_host - -end module test_prog diff --git a/test/capgen_test/test_host.meta b/test/capgen_test/test_host.meta deleted file mode 100644 index 5d861764..00000000 --- a/test/capgen_test/test_host.meta +++ /dev/null @@ -1,38 +0,0 @@ -[ccpp-table-properties] - name = suite_info - type = ddt -[ccpp-arg-table] - name = suite_info - type = ddt - -[ccpp-table-properties] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True -[ col_end ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - protected = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer diff --git a/test/capgen_test/test_host_data.F90 b/test/capgen_test/test_host_data.F90 deleted file mode 100644 index 32c421a4..00000000 --- a/test/capgen_test/test_host_data.F90 +++ /dev/null @@ -1,60 +0,0 @@ -module test_host_data - - use ccpp_kinds, only: kind_phys - - implicit none - private - - !> \section arg_table_physics_state Argument Table - !! \htmlinclude arg_table_physics_state.html - type physics_state - real(kind=kind_phys), dimension(:), allocatable :: & - ps, & ! surface pressure - soil_levs ! soil temperature (cm) - real(kind=kind_phys), dimension(:, :), allocatable :: & - u, & ! zonal wind (m/s) - v, & ! meridional wind (m/s) - pmid ! midpoint pressure (Pa) - real(kind=kind_phys), dimension(:, :, :), allocatable :: & - q ! constituent mixing ratio (kg/kg moist or dry air depending on type) - end type physics_state - - public :: physics_state - public :: allocate_physics_state - -contains - - subroutine allocate_physics_state(cols, levels, constituents, lbnd_slev, ubnd_slev, state) - integer, intent(in) :: cols - integer, intent(in) :: levels - integer, intent(in) :: constituents - integer, intent(in) :: lbnd_slev, ubnd_slev - type(physics_state), intent(out) :: state - - if (allocated(state%ps)) then - deallocate(state%ps) - end if - allocate(state%ps(cols)) - if (allocated(state%u)) then - deallocate(state%u) - end if - allocate(state%u(cols, levels)) - if (allocated(state%v)) then - deallocate(state%v) - end if - allocate(state%v(cols, levels)) - if (allocated(state%pmid)) then - deallocate(state%pmid) - end if - allocate(state%pmid(cols, levels)) - if (allocated(state%q)) then - deallocate(state%q) - end if - allocate(state%q(cols, levels, constituents)) - if (allocated(state%soil_levs)) then - deallocate(state%soil_levs) - end if - allocate(state%soil_levs(lbnd_slev:ubnd_slev)) - - end subroutine allocate_physics_state -end module test_host_data diff --git a/test/capgen_test/test_host_data.meta b/test/capgen_test/test_host_data.meta deleted file mode 100644 index 0e73c060..00000000 --- a/test/capgen_test/test_host_data.meta +++ /dev/null @@ -1,59 +0,0 @@ -[ccpp-table-properties] - name = physics_state - type = ddt -[ccpp-arg-table] - name = physics_state - type = ddt -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_dimension) -[ u ] - standard_name = eastward_wind - long_name = Zonal wind - state_variable = true - type = real - kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) -[ v ] - standard_name = northward_wind - long_name = Meridional wind - state_variable = true - type = real - kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) -[ pmid ] - standard_name = air_pressure - long_name = Midpoint air pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_dimension, vertical_layer_dimension) -[ soil_levs ] - standard_name = soil_levels - long_name = soil levels - units = cm - dimensions = (lower_bound_of_vertical_dimension_of_soil:upper_bound_of_vertical_dimension_of_soil) - type = real - kind = kind_phys -[ q ] - standard_name = constituent_mixing_ratio - state_variable = true - type = real - kind = kind_phys - units = kg kg-1 moist or dry air depending on type - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) -[ q(:,:,index_of_water_vapor_specific_HUMidity) ] - standard_name = water_vapor_specific_humidity - state_variable = true - type = real - kind = kind_phys - units = kg kg-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - active = (index_of_water_vapor_specific_humidity > 0) diff --git a/test/capgen_test/test_host_mod.F90 b/test/capgen_test/test_host_mod.F90 deleted file mode 100644 index 48ee959b..00000000 --- a/test/capgen_test/test_host_mod.F90 +++ /dev/null @@ -1,164 +0,0 @@ -module test_host_mod - - use ccpp_kinds, only: kind_phys - use test_host_data, only: physics_state, & - allocate_physics_state - - implicit none - public - - !> \section arg_table_test_host_mod Argument Table - !! \htmlinclude arg_table_test_host_host.html - !! - integer, parameter :: ncols = 10 - integer, parameter :: pver = 5 - integer, parameter :: pverp = 6 - integer, parameter :: pcnst = 2 - integer, parameter :: slevs = 4 - integer, parameter :: slev_lbound = -3 - integer, parameter :: slev_ubound = 0 - integer, parameter :: diagdimstart = 2 - integer, parameter :: index_qv = 1 - logical, parameter :: config_var = .true. - real(kind=kind_phys), allocatable :: temp_midpoints(:, :) - real(kind=kind_phys) :: temp_interfaces(ncols, pverp) - real(kind=kind_phys) :: temp_diag(ncols, 6) - real(kind=kind_phys) :: coeffs(ncols) - real(kind=kind_phys) :: var_array(ncols, 2, 4, 6) - real(kind=kind_phys), dimension(diagdimstart:ncols, diagdimstart:pver) :: & - diag1, & - diag2 - real(kind=kind_phys) :: dt - real(kind=kind_phys), parameter :: temp_inc = 0.05_kind_phys - type(physics_state) :: phys_state - integer :: num_model_times = -1 - integer, allocatable :: model_times(:) - - integer, parameter :: num_time_steps = 2 - real(kind=kind_phys), parameter :: tolerance = 1.0e-13_kind_phys - real(kind=kind_phys) :: tint_save(ncols, pverp) - - public :: init_data - public :: compare_data - public :: check_model_times - -contains - - subroutine init_data() - - integer :: col - integer :: lev - integer :: cind - integer :: offsize - - ! Allocate and initialize temperature - allocate(temp_midpoints(ncols, pver)) - temp_midpoints = 0.0_kind_phys - cind = 1 - do lev = 1, pverp - offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) - do col = 1, ncols - temp_interfaces(col, lev) = real(offsize + col, kind=kind_phys) - tint_save(col, lev) = temp_interfaces(col, lev) - end do - end do - ! Allocate and initialize state - call allocate_physics_state(ncols, pver, pcnst, slev_lbound, slev_ubound, phys_state) - do cind = 1, pcnst - do lev = 1, pver - offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) - do col = 1, ncols - phys_state%q(col, lev, cind) = real(offsize + col, kind=kind_phys) - end do - end do - end do - - end subroutine init_data - - logical function check_model_times() - - check_model_times = (num_model_times > 0) - if (check_model_times) then - check_model_times = (size(model_times) == num_model_times) - if (.not. check_model_times) then - write(6, '(2(a,i0))') 'model_times size mismatch, ', & - size(model_times), ' should be ', num_model_times - end if - else - write(6, '(a,i0,a)') 'num_model_times mismatch, ', num_model_times, & - ' should be greater than zero' - end if - - end function check_model_times - - logical function compare_data() - - integer :: col - integer :: lev - integer :: cind - integer :: offsize - logical :: need_header - real(kind=kind_phys) :: avg - integer, parameter :: cincrements(pcnst) = (/ 1, 0 /) - real(kind=kind_phys) :: total_test - real(kind=kind_phys), parameter :: total_ref = 6730.0_kind_phys - - compare_data = .true. - - total_test = 0.0_kind_phys - need_header = .true. - do lev = 1, pver - do col = 1, ncols - avg = (tint_save(col, lev) + tint_save(col, lev + 1)) - avg = 1.0_kind_phys + (avg / 2.0_kind_phys) - avg = avg + (temp_inc * num_time_steps) - total_test = total_test + avg - if (abs((temp_midpoints(col, lev) - avg) / avg) > tolerance) then - if (need_header) then - write(6, '(" COL LEV T MIDPOINTS EXPECTED")') - need_header = .false. - end if - write(6, '(2i5,2(3x,es15.7))') col, lev, & - temp_midpoints(col, lev), avg - compare_data = .false. - end if - end do - end do - ! Check constituents - need_header = .true. - do cind = 1, pcnst - do lev = 1, pver - offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) - do col = 1, ncols - avg = real(offsize + col + (cincrements(cind) * num_time_steps), & - kind=kind_phys) - total_test = total_test + avg - if (abs((phys_state%q(col, lev, cind) - avg) / avg) > & - tolerance) then - if (need_header) then - write(6, '(2(2x,a),3x,a,10x,a,14x,a)') & - 'COL', 'LEV', 'C#', 'Q', 'EXPECTED' - need_header = .false. - end if - write(6, '(3i5,2(3x,es15.7))') col, lev, cind, & - phys_state%q(col, lev, cind), avg - compare_data = .false. - end if - end do - end do - end do - if (abs((total_test - total_ref) / total_ref) > tolerance) then - write(6, '(a,e12.4)') 'TOTAL REFERENCE: ', total_ref - write(6, '(a,e12.4)') 'TOTAL TEST: ', total_test - write(6, '(2(a,e12.4))') 'REL.DIFF > TOLERANCE:', & - abs((total_test - total_ref) / total_ref), ' >', tolerance - compare_data = .false. - else - write(0, '(a,e12.4)') 'TOTAL REFERENCE: ', total_ref - write(0, '(a,e12.4)') 'TOTAL TEST: ', total_test - write(0, '(2(a,e12.4))') 'REL.DIFF < TOLERANCE:', & - abs((total_test - total_ref) / total_ref), ' <', tolerance - end if - end function compare_data - -end module test_host_mod diff --git a/test/capgen_test/test_host_mod.meta b/test/capgen_test/test_host_mod.meta deleted file mode 100644 index 08627af0..00000000 --- a/test/capgen_test/test_host_mod.meta +++ /dev/null @@ -1,133 +0,0 @@ -[ccpp-table-properties] - name = test_host_mod - type = module -[ccpp-arg-table] - name = test_host_mod - type = module -[ index_qv ] - standard_name = index_of_water_vapor_specific_HUMidity - units = index - type = integer - protected = True - dimensions = () -[ config_var ] - standard_name = configuration_variable - units = none - type = logical - protected = True - dimensions = () -[ ncols] - standard_name = horizontal_dimension - units = count - type = integer - protected = True - dimensions = () -[ pver ] - standard_name = vertical_layer_dimension - units = count - type = integer - protected = True - dimensions = () -[ pverP ] - standard_name = vertical_interface_dimension - type = integer - units = count - protected = True - dimensions = () -[ pcnst ] - standard_name = number_of_tracers - type = integer - units = count - protected = True - dimensions = () -[ slevs ] - standard_name = vertical_dimension_of_soil - type = integer - units = count - protected = True - dimensions = () -[ slev_lbound] - standard_name = lower_bound_of_vertical_dimension_of_soil - type = integer - units = count - protected = True - dimensions = () -[ slev_ubound] - standard_name = upper_bound_of_vertical_dimension_of_soil - type = integer - units = count - protected = True - dimensions = () -[ DiagDimStart ] - standard_name = first_index_of_diag_fields - type = integer - units = count - protected = True - dimensions = () -[ temp_midpoints ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys -[ temp_interfaces ] - standard_name = potential_temperature_at_interface - units = K - dimensions = (horizontal_dimension, vertical_interface_dimension) - type = real | kind = kind_phys -[ temp_diag ] - standard_name = temperature_at_diagnostic_levels - units = K - dimensions = (horizontal_dimension, 6) - type = real | kind = kind_phys -[ diag1 ] - standard_name = diagnostic_stuff_type_1 - long_name = This is just a test field - units = K - dimensions = (first_index_of_diag_fields:horizontal_dimension, first_index_of_diag_fields:vertical_layer_dimension) - type = real | kind = kind_phys -[ diag2 ] - standard_name = diagnostic_stuff_type_2 - long_name = This is just a test field - units = K - dimensions = (first_index_of_diag_fields: horizontal_dimension, first_index_of_diag_fields :vertical_layer_dimension) - type = real | kind = kind_phys -[ dt ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real | kind = kind_phys -[ temp_inc ] - standard_name = potential_temperature_increment - long_name = Per time step potential temperature increment - units = K - dimensions = () - type = real | kind = kind_phys -[ phys_state ] - standard_name = physics_state_derived_type - long_name = Physics State DDT - type = physics_state - dimensions = () -[ num_model_times ] - standard_name = number_of_model_times - type = integer - units = count - dimensions = () -[ model_times ] - standard_name = model_times - units = seconds - dimensions = (number_of_model_times) - type = integer - allocatable = True -[ coeffs ] - standard_name = coefficients_for_interpolation - long_name = coefficients for interpolation - units = none - dimensions = (horizontal_dimension) - type = real | kind = kind_phys -[ var_array ] - standard_name = array_variable_for_testing - long_name = array variable for testing - units = none - dimensions = (horizontal_dimension,2,4,6) - type = real | kind = kind_phys diff --git a/test/ddthost_test/CMakeLists.txt b/test/ddthost_test/CMakeLists.txt deleted file mode 100644 index 5516277b..00000000 --- a/test/ddthost_test/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ - -#------------------------------------------------------------------------------ -# -# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES -# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) -# -#------------------------------------------------------------------------------ -set(SCHEME_FILES "setup_coeffs" "temp_set" "temp_adjust" "temp_calc_adjust") -set(SUITE_SCHEME_FILES "make_ddt" "environ_conditions") -set(HOST_FILES "test_host_data" "test_host_mod" "host_ccpp_ddt") -set(SUITE_FILES "ddt_suite.xml" "temp_suite.xml") -# HOST is the name of the executable we will build. -# We assume there are files ${HOST}.meta and ${HOST}.F90 in CMAKE_SOURCE_DIR -set(HOST "test_host") - -# By default, generated caps go in ccpp subdir -set(CCPP_CAP_FILES "${CMAKE_CURRENT_BINARY_DIR}/ccpp") - -# Create lists for Fortran and meta data files from file names -list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) -list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_META_FILES) -list(TRANSFORM SUITE_SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SUITE_SCHEME_FORTRAN_FILES) -list(TRANSFORM SUITE_SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SUITE_SCHEME_META_FILES) -list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE DDT_HOST_FORTRAN_FILES) -list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE DDT_HOST_METADATA_FILES) - -list(APPEND DDT_HOST_METADATA_FILES "${HOST}.meta") - -# Run ccpp_capgen -ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${DDT_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_META_FILES} ${SUITE_SCHEME_META_FILES} - SUITES ${SUITE_FILES} - HOST_NAME ${HOST} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Retrieve the list of Fortran files required for test host from datatable.xml and set to CCPP_CAPS_LIST -ccpp_datafile(DATATABLE "${CCPP_CAP_FILES}/datatable.xml" - REPORT_NAME "--ccpp-files") - -# Create test host library -add_library(DDT_TESTLIB OBJECT ${SCHEME_FORTRAN_FILES} - ${SUITE_SCHEME_FORTRAN_FILES} - ${DDT_HOST_FORTRAN_FILES} - ${CCPP_CAPS_LIST}) - -# Setup test executable with needed dependencies -add_executable(ddt_host_integration test_ddt_host_integration.F90 ${HOST}.F90) -target_link_libraries(ddt_host_integration PRIVATE DDT_TESTLIB test_utils) -target_include_directories(ddt_host_integration PRIVATE "$") - -# Add executable to be called with ctest -add_test(NAME ctest_ddt_host_integration COMMAND ddt_host_integration) diff --git a/test/ddthost_test/README.md b/test/ddthost_test/README.md deleted file mode 100644 index 9c90d754..00000000 --- a/test/ddthost_test/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# DDT Host Test - -Contains tests to exercise more DDT functionality: -- Passing around and modifying a DDT -- Making DDT in host model & using it in CCPP-ized physics code - -## Building/Running - -To explicitly build/run the ddt test host, run: - -```bash -$ cmake --fresh -S -B -DCCPP_RUN_DDT_HOST_TEST=ON -$ cd -$ make -$ ctest -``` diff --git a/test/ddthost_test/ddt_suite.xml b/test/ddthost_test/ddt_suite.xml deleted file mode 100644 index 749bb3bc..00000000 --- a/test/ddthost_test/ddt_suite.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - make_ddt - environ_conditions - - diff --git a/test/ddthost_test/ddthost_test_reports.py b/test/ddthost_test/ddthost_test_reports.py deleted file mode 100644 index 612cbbbf..00000000 --- a/test/ddthost_test/ddthost_test_reports.py +++ /dev/null @@ -1,139 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Test DDT host database report python interface - - Assumptions: - - Command line arguments: build_dir database_filepath - - Usage: python test_reports ------------------------------------------------------------------------ -""" -import os -import unittest - -from test_stub import BaseTests - -_BUILD_DIR = os.path.join(os.path.abspath(os.environ['BUILD_DIR']), "test", "ddthost_test") -_DATABASE = os.path.abspath(os.path.join(_BUILD_DIR, "ccpp", "datatable.xml")) - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_FRAMEWORK_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, os.pardir)) -_SCRIPTS_DIR = os.path.join(_FRAMEWORK_DIR, "scripts") -_SRC_DIR = os.path.join(_FRAMEWORK_DIR, "src") - - -# Check data -_HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] -_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_ddt_suite_cap.F90"), - os.path.join(_BUILD_DIR, "ccpp", "ccpp_temp_suite_cap.F90")] -_UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), - os.path.join(_SRC_DIR, "ccpp_constituent_prop_mod.F90"), - os.path.join(_SRC_DIR, "ccpp_scheme_utils.F90"), - os.path.join(_SRC_DIR, "ccpp_hashable.F90"), - os.path.join(_SRC_DIR, "ccpp_hash_table.F90")] -_CCPP_FILES = _UTILITY_FILES + \ - [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90"), - os.path.join(_BUILD_DIR, "ccpp", "ccpp_ddt_suite_cap.F90"), - os.path.join(_BUILD_DIR, "ccpp", "ccpp_temp_suite_cap.F90")] -_DEPENDENCIES = [os.path.join(_TEST_DIR, "adjust", "qux.F90"), - os.path.join(_TEST_DIR, "bar.F90"), - os.path.join(_TEST_DIR, "foo.F90")] -_PROCESS_LIST = ["setter=temp_set", "adjusting=temp_calc_adjust"] -_MODULE_LIST = ["environ_conditions", "make_ddt", "setup_coeffs", "temp_adjust", - "temp_calc_adjust", "temp_set"] -_SUITE_LIST = ["ddt_suite", "temp_suite"] -_INPUT_VARS_DDT = ["model_times", "number_of_model_times", - "horizontal_loop_begin", "horizontal_loop_end", - "surface_air_pressure", "horizontal_dimension", - "host_standard_ccpp_type"] -_OUTPUT_VARS_DDT = ["ccpp_error_code", "ccpp_error_message", "model_times", - "number_of_model_times", "surface_air_pressure"] -_REQUIRED_VARS_DDT = _INPUT_VARS_DDT + _OUTPUT_VARS_DDT -_PROT_VARS_TEMP = ["horizontal_loop_begin", "horizontal_loop_end", - "horizontal_dimension", "vertical_layer_dimension", - "number_of_tracers", - # Added for --debug - "index_of_water_vapor_specific_humidity", - "vertical_interface_dimension"] -_REQUIRED_VARS_TEMP = ["ccpp_error_code", "ccpp_error_message", - "potential_temperature", - "potential_temperature_at_interface", - "coefficients_for_interpolation", - "potential_temperature_increment", - "surface_air_pressure", "time_step_for_physics", - "water_vapor_specific_humidity"] -_INPUT_VARS_TEMP = ["potential_temperature", - "potential_temperature_at_interface", - "coefficients_for_interpolation", - "potential_temperature_increment", - "surface_air_pressure", "time_step_for_physics", - "water_vapor_specific_humidity"] -_OUTPUT_VARS_TEMP = ["ccpp_error_code", "ccpp_error_message", - "potential_temperature", - "potential_temperature_at_interface", - "coefficients_for_interpolation", - "surface_air_pressure", "water_vapor_specific_humidity"] - -class TestDdtHostDataTables(unittest.TestCase, BaseTests.TestHostDataTables): - database = _DATABASE - host_files = _HOST_FILES - suite_files = _SUITE_FILES - utility_files = _UTILITY_FILES - ccpp_files = _CCPP_FILES - process_list = _PROCESS_LIST - module_list = _MODULE_LIST - dependencies = _DEPENDENCIES - suite_list = _SUITE_LIST - - -class CommandLineDdtHostDatafileRequiredFiles(unittest.TestCase, BaseTests.TestHostCommandLineDataFiles): - database = _DATABASE - host_files = _HOST_FILES - suite_files = _SUITE_FILES - utility_files = _UTILITY_FILES - ccpp_files = _CCPP_FILES - process_list = _PROCESS_LIST - module_list = _MODULE_LIST - dependencies = _DEPENDENCIES - suite_list = _SUITE_LIST - datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" - - -class TestDdtSuite(unittest.TestCase, BaseTests.TestSuite): - database = _DATABASE - required_vars = _REQUIRED_VARS_DDT - input_vars = _INPUT_VARS_DDT - output_vars = _OUTPUT_VARS_DDT - suite_name = "ddt_suite" - - -class CommandLineDdtSuite(unittest.TestCase, BaseTests.TestSuiteCommandLine): - database = _DATABASE - required_vars = _REQUIRED_VARS_DDT - input_vars = _INPUT_VARS_DDT - output_vars = _OUTPUT_VARS_DDT - suite_name = "ddt_suite" - datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" - - -class TestDdtTempSuite(unittest.TestCase, BaseTests.TestSuiteExcludeProtected): - database = _DATABASE - required_vars = _REQUIRED_VARS_TEMP + _PROT_VARS_TEMP - input_vars = _INPUT_VARS_TEMP + _PROT_VARS_TEMP - required_vars_excluding_protected = _REQUIRED_VARS_TEMP - input_vars_excluding_protected = _INPUT_VARS_TEMP - output_vars = _OUTPUT_VARS_TEMP - suite_name = "temp_suite" - - -class CommandLineDdtTempSuite(unittest.TestCase, BaseTests.TestSuiteExcludeProtectedCommandLine): - database = _DATABASE - required_vars = _REQUIRED_VARS_TEMP + _PROT_VARS_TEMP - input_vars = _INPUT_VARS_TEMP + _PROT_VARS_TEMP - required_vars_excluding_protected = _REQUIRED_VARS_TEMP - input_vars_excluding_protected = _INPUT_VARS_TEMP - output_vars = _OUTPUT_VARS_TEMP - suite_name = "temp_suite" - datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" diff --git a/test/ddthost_test/environ_conditions.F90 b/test/ddthost_test/environ_conditions.F90 deleted file mode 100644 index 2d63366e..00000000 --- a/test/ddthost_test/environ_conditions.F90 +++ /dev/null @@ -1,96 +0,0 @@ -module environ_conditions - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: environ_conditions_init - public :: environ_conditions_run - public :: environ_conditions_finalize - - integer, parameter :: input_model_times = 3 - integer, parameter :: input_model_values(input_model_times) = (/ 31, 37, 41 /) - -contains - - !> \section arg_table_environ_conditions_run Argument Table - !! \htmlinclude arg_table_environ_conditions_run.html - !! - subroutine environ_conditions_run(psurf, errmsg, errflg) - - ! This routine currently does nothing -- should update values - - real(kind=kind_phys), intent(in) :: psurf(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - end subroutine environ_conditions_run - - !> \section arg_table_environ_conditions_init Argument Table - !! \htmlinclude arg_table_environ_conditions_init.html - !! - subroutine environ_conditions_init(nbox, o3, hno3, ntimes, model_times, & - errmsg, errflg) - - integer, intent(in) :: nbox - real(kind=kind_phys), intent(out) :: o3(:) - real(kind=kind_phys), intent(out) :: hno3(:) - integer, intent(out) :: ntimes - integer, allocatable, intent(out) :: model_times(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: i, j - - errmsg = '' - errflg = 0 - - ! This may be replaced with MusicBox json environmental conditions reader??? - - do i = 1, nbox - o3(i) = real(i, kind_phys) * 1.e-6_kind_phys - hno3(i) = real(i, kind_phys) * 1.e-9_kind_phys - end do - - ntimes = input_model_times - allocate(model_times(ntimes)) - model_times = input_model_values - - end subroutine environ_conditions_init - - !> \section arg_table_environ_conditions_finalize Argument Table - !! \htmlinclude arg_table_environ_conditions_finalize.html - !! - subroutine environ_conditions_finalize(ntimes, model_times, errmsg, errflg) - - integer, intent(in) :: ntimes - integer, intent(in) :: model_times(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine checks the size and values of model_times - if (ntimes /= input_model_times) then - errflg = 1 - write(errmsg, '(2(a,i0))') 'ntimes mismatch, ', ntimes, ' should be ', & - input_model_times - else if (size(model_times) /= input_model_times) then - errflg = 1 - write(errmsg, '(2(a,i0))') 'model_times size mismatch, ', & - size(model_times), ' should be ', input_model_times - else if (any(model_times /= input_model_values)) then - errflg = 1 - write(errmsg, *) 'model_times mismatch, ', & - model_times, ' should be ', input_model_values - else - errmsg = '' - errflg = 0 - end if - - end subroutine environ_conditions_finalize - -end module environ_conditions diff --git a/test/ddthost_test/environ_conditions.meta b/test/ddthost_test/environ_conditions.meta deleted file mode 100644 index 894e0e92..00000000 --- a/test/ddthost_test/environ_conditions.meta +++ /dev/null @@ -1,110 +0,0 @@ -[ccpp-table-properties] - name = environ_conditions - type = scheme -[ccpp-arg-table] - name = environ_conditions_run - type = scheme -[ psurf ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = environ_conditions_init - type = scheme -[ nbox ] - standard_name = horizontal_dimension - type = integer - units = count - dimensions = () - intent = in -[ o3 ] - standard_name = ozone - units = ppmv - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = out -[ hno3 ] - standard_name = nitric_acid - units = ppmv - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = out -[ ntimes ] - standard_name = number_of_model_times - type = integer - units = count - dimensions = () - intent = out -[ model_times ] - standard_name = model_times - units = seconds - dimensions = (number_of_model_times) - type = integer - intent = out - allocatable = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = environ_conditions_finalize - type = scheme -[ ntimes ] - standard_name = number_of_model_times - type = integer - units = count - dimensions = () - intent = in -[ model_times ] - standard_name = model_times - units = seconds - dimensions = (number_of_model_times) - type = integer - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/ddthost_test/host_ccpp_ddt.F90 b/test/ddthost_test/host_ccpp_ddt.F90 deleted file mode 100644 index b60c81af..00000000 --- a/test/ddthost_test/host_ccpp_ddt.F90 +++ /dev/null @@ -1,16 +0,0 @@ -module host_ccpp_ddt - - implicit none - private - - !> \section arg_table_ccpp_info_t Argument Table - !! \htmlinclude arg_table_ccpp_info_t.html - !! - type, public :: ccpp_info_t - integer :: col_start ! horizontal_loop_begin - integer :: col_end ! horizontal_loop_end - character(len=512) :: errmsg ! ccpp_error_message - integer :: errflg ! ccpp_error_code - end type ccpp_info_t - -end module host_ccpp_ddt diff --git a/test/ddthost_test/host_ccpp_ddt.meta b/test/ddthost_test/host_ccpp_ddt.meta deleted file mode 100644 index 56dca845..00000000 --- a/test/ddthost_test/host_ccpp_ddt.meta +++ /dev/null @@ -1,31 +0,0 @@ -[ccpp-table-properties] - name = ccpp_info_t - type = ddt -[ccpp-arg-table] - name = ccpp_info_t - type = ddt -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True -[ col_end ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - protected = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer diff --git a/test/ddthost_test/make_ddt.F90 b/test/ddthost_test/make_ddt.F90 deleted file mode 100644 index a0de4177..00000000 --- a/test/ddthost_test/make_ddt.F90 +++ /dev/null @@ -1,133 +0,0 @@ -!Hello demonstration parameterization -! - -module make_ddt - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: make_ddt_init - public :: make_ddt_run - public :: make_ddt_timestep_final - public :: vmr_type - - !> \section arg_table_vmr_type Argument Table - !! \htmlinclude arg_table_vmr_type.html - !! - type vmr_type - integer :: nvmr - real(kind=kind_phys), allocatable :: vmr_array(:, :) - end type vmr_type - -contains - - !> \section arg_table_make_ddt_run Argument Table - !! \htmlinclude arg_table_make_ddt_run.html - !! - subroutine make_ddt_run(cols, cole, o3, hno3, vmr, errmsg, errflg) - !---------------------------------------------------------------- - implicit none - !---------------------------------------------------------------- - - ! Dummy arguments - integer, intent(in) :: cols - integer, intent(in) :: cole - real(kind=kind_phys), intent(in) :: o3(:) - real(kind=kind_phys), intent(in) :: hno3(:) - type(vmr_type), intent(inout) :: vmr - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - ! Local variable - integer :: nbox - !---------------------------------------------------------------- - - errmsg = '' - errflg = 0 - - ! Check for correct threading behavior - nbox = cole - cols + 1 - if (size(o3) /= nbox) then - errflg = 1 - write(errmsg, '(2(a,i0))') 'SIZE(O3) = ', size(o3), ', should be ', nbox - else if (size(hno3) /= nbox) then - errflg = 1 - write(errmsg, '(2(a,i0))') 'SIZE(HNO3) = ', size(hno3), & - ', should be ', nbox - else - ! NOTE -- This is prototyping one approach to passing a large number of - ! chemical VMR values and is the predecessor for adding in methods and - ! maybe nesting DDTs (especially for aerosols) - vmr%vmr_array(cols:cole, 1) = o3(:) - vmr%vmr_array(cols:cole, 2) = hno3(:) - end if - - end subroutine make_ddt_run - - !> \section arg_table_make_ddt_init Argument Table - !! \htmlinclude arg_table_make_ddt_init.html - !! - subroutine make_ddt_init(nbox, ccpp_info, vmr, errmsg, errflg) - use host_ccpp_ddt, only: ccpp_info_t - - ! Dummy arguments - integer, intent(in) :: nbox - type(ccpp_info_t), intent(in) :: ccpp_info - type(vmr_type), intent(out) :: vmr - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine initializes the vmr array - vmr%nvmr = 2 - allocate(vmr%vmr_array(nbox, vmr%nvmr)) - - errmsg = '' - errflg = 0 - - end subroutine make_ddt_init - - !> \section arg_table_make_ddt_timestep_final Argument Table - !! \htmlinclude arg_table_make_ddt_timestep_final.html - !! - subroutine make_ddt_timestep_final(ncols, vmr, errmsg, errflg) - - ! Dummy arguments - integer, intent(in) :: ncols - type(vmr_type), intent(in) :: vmr - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - ! Local variables - integer :: index - real(kind=kind_phys) :: rind - - errmsg = '' - errflg = 0 - - ! This routine checks the array values in vmr - if (size(vmr%vmr_array, 1) /= ncols) then - errflg = 1 - write(errmsg, '(2(a,i0))') 'VMR%VMR_ARRAY first dimension size is, ', & - size(vmr%vmr_array, 1), ', should be, ', ncols - else - do index = 1, ncols - rind = real(index, kind_phys) - if (vmr%vmr_array(index, 1) /= rind * 1.e-6_kind_phys) then - errflg = 1 - write(errmsg, '(a,i0,2(a,e12.4))') 'O3(', index, ') = ', & - vmr%vmr_array(index, 1), ', should be, ', & - rind * 1.e-6_kind_phys - exit - else if (vmr%vmr_array(index, 2) /= rind * 1.e-9_kind_phys) then - errflg = 1 - write(errmsg, '(a,i0,2(a,e12.4))') 'HNO3(', index, ') = ', & - vmr%vmr_array(index, 2), ', should be, ', & - rind * 1.e-9_kind_phys - exit - end if - end do - end if - - end subroutine make_ddt_timestep_final - -end module make_ddt diff --git a/test/ddthost_test/make_ddt.meta b/test/ddthost_test/make_ddt.meta deleted file mode 100644 index 4998e917..00000000 --- a/test/ddthost_test/make_ddt.meta +++ /dev/null @@ -1,132 +0,0 @@ -[ccpp-table-properties] - name = vmr_type - type = ddt -[ccpp-arg-table] - name = vmr_type - type = ddt -[ nvmr ] - standard_name = number_of_chemical_species - units = count - dimensions = () - type = integer -[ vmr_array ] - standard_name = array_of_volume_mixing_ratios - units = ppmv - dimensions = (horizontal_dimension, number_of_chemical_species) - type = real - kind = kind_phys -[ccpp-table-properties] - name = make_ddt - type = scheme -[ccpp-arg-table] - name = make_ddt_run - type = scheme -[ cols ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - intent = in -[ cole ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - intent = in -[ O3 ] - standard_name = ozone - units = ppmv - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ HNO3 ] - standard_name = nitric_acid - units = ppmv - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = vmr_type - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = make_ddt_init - type = scheme -[ nbox ] - standard_name = horizontal_dimension - type = integer - units = count - dimensions = () - intent = in -[ ccpp_info ] - standard_name = host_standard_ccpp_type - type = ccpp_info_t - dimensions = () - intent = in -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = vmr_type - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = make_ddt_timestep_final - type = scheme -[ ncols ] - standard_name = horizontal_dimension - type = integer - units = count - dimensions = () - intent = in -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = vmr_type - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/ddthost_test/setup_coeffs.F90 b/test/ddthost_test/setup_coeffs.F90 deleted file mode 100644 index 09c7fcc1..00000000 --- a/test/ddthost_test/setup_coeffs.F90 +++ /dev/null @@ -1,24 +0,0 @@ -module setup_coeffs - use ccpp_kinds, only: kind_phys - implicit none - - public :: setup_coeffs_timestep_init - -contains - !> \section arg_table_setup_coeffs_timestep_init Argument Table - !! \htmlinclude arg_table_setup_coeffs_timestep_init.html - !! - subroutine setup_coeffs_timestep_init(coeffs, errmsg, errflg) - - real(kind=kind_phys), intent(inout) :: coeffs(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - coeffs(:) = 1._kind_phys - - end subroutine setup_coeffs_timestep_init - -end module setup_coeffs diff --git a/test/ddthost_test/setup_coeffs.meta b/test/ddthost_test/setup_coeffs.meta deleted file mode 100644 index 8d0fc5f4..00000000 --- a/test/ddthost_test/setup_coeffs.meta +++ /dev/null @@ -1,29 +0,0 @@ -[ccpp-table-properties] - name = setup_coeffs - type = scheme -[ccpp-arg-table] - name = setup_coeffs_timestep_init - type = scheme -[ coeffs ] - standard_name = coefficients_for_interpolation - long_name = coefficients for interpolation - units = none - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/ddthost_test/temp_adjust.F90 b/test/ddthost_test/temp_adjust.F90 deleted file mode 100644 index 4ef3655d..00000000 --- a/test/ddthost_test/temp_adjust.F90 +++ /dev/null @@ -1,84 +0,0 @@ -! Test parameterization with no vertical level -! - -module temp_adjust - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: temp_adjust_init - public :: temp_adjust_run - public :: temp_adjust_finalize - -contains - - !> \section arg_table_temp_adjust_run Argument Table - !! \htmlinclude arg_table_temp_adjust_run.html - !! - subroutine temp_adjust_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - to_promote, promote_pcnst, errmsg, errflg, innie, outie, optsie) - - integer, intent(in) :: foo - real(kind=kind_phys), intent(in) :: timestep - real(kind=kind_phys), intent(inout), optional :: qv(:, :) - real(kind=kind_phys), intent(inout) :: ps(:) - real(kind=kind_phys), intent(in) :: temp_prev(:, :) - real(kind=kind_phys), intent(inout) :: temp_layer(:, :) - real(kind=kind_phys), intent(in) :: to_promote(:, :) - real(kind=kind_phys), intent(in) :: promote_pcnst(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - real(kind=kind_phys), optional, intent(in) :: innie - real(kind=kind_phys), optional, intent(out) :: outie - real(kind=kind_phys), optional, intent(inout) :: optsie - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index, :) = temp_layer(col_index, :) + temp_prev(col_index, :) - if (present(qv)) qv(col_index, :) = qv(col_index, :) + 1.0_kind_phys - end do - if (present(innie) .and. present(outie) .and. present(optsie)) then - outie = innie * optsie - optsie = optsie + 1.0_kind_phys - end if - - end subroutine temp_adjust_run - - !> \section arg_table_temp_adjust_init Argument Table - !! \htmlinclude arg_table_temp_adjust_init.html - !! - subroutine temp_adjust_init(errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_adjust_init - - !> \section arg_table_temp_adjust_finalize Argument Table - !! \htmlinclude arg_table_temp_adjust_finalize.html - !! - subroutine temp_adjust_finalize(errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_adjust_finalize - -end module temp_adjust diff --git a/test/ddthost_test/temp_adjust.meta b/test/ddthost_test/temp_adjust.meta deleted file mode 100644 index a67cef8a..00000000 --- a/test/ddthost_test/temp_adjust.meta +++ /dev/null @@ -1,119 +0,0 @@ -[ccpp-table-properties] - name = temp_adjust - type = scheme - dependencies = qux.F90 - dependencies_path = adjust -[ccpp-arg-table] - name = temp_adjust_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout - diagnostic_name = temperature -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout - diagnostic_name_fixed = Q - optional = True -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = inout -[ to_promote ] - standard_name = promote_this_variable_to_suite - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = in -[ promote_pcnst ] - standard_name = promote_this_variable_with_no_horizontal_dimension - units = K - dimensions = (number_of_tracers) - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_adjust_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_adjust_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/ddthost_test/temp_calc_adjust.F90 b/test/ddthost_test/temp_calc_adjust.F90 deleted file mode 100644 index 4c2d7ece..00000000 --- a/test/ddthost_test/temp_calc_adjust.F90 +++ /dev/null @@ -1,95 +0,0 @@ -!Test parameterization with no vertical level and hanging intent(out) variable -! - -module temp_calc_adjust - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: temp_calc_adjust_init - public :: temp_calc_adjust_run - public :: temp_calc_adjust_finalize - -contains - - !> \section arg_table_temp_calc_adjust_run Argument Table - !! \htmlinclude arg_table_temp_calc_adjust_run.html - !! - subroutine temp_calc_adjust_run(nbox, timestep, temp_level, temp_calc, & - errmsg, errflg) - - integer, intent(in) :: nbox - real(kind=kind_phys), intent(in) :: timestep - real(kind=kind_phys), intent(in) :: temp_level(:, :) - real(kind=kind_phys), intent(out) :: temp_calc(:, :) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - real(kind=kind_phys) :: bar = 1.0_kind_phys - - errmsg = '' - errflg = 0 - - call temp_calc_adjust_nested_subroutine(temp_calc) - if (check_foo()) then - call foo(bar) - end if - - contains - - elemental subroutine temp_calc_adjust_nested_subroutine(temp) - - real(kind=kind_phys), intent(out) :: temp - !------------------------------------------------------------- - - temp = 1.0_kind_phys - - end subroutine temp_calc_adjust_nested_subroutine - - subroutine foo(bar) - real(kind=kind_phys), intent(inout) :: bar - bar = bar + 1.0_kind_phys - - end subroutine foo - - logical function check_foo() - check_foo = .true. - end function check_foo - - end subroutine temp_calc_adjust_run - - !> \section arg_table_temp_calc_adjust_init Argument Table - !! \htmlinclude arg_table_temp_calc_adjust_init.html - !! - subroutine temp_calc_adjust_init(errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_calc_adjust_init - - !> \section arg_table_temp_calc_adjust_finalize Argument Table - !! \htmlinclude arg_table_temp_calc_adjust_finalize.html - !! - subroutine temp_calc_adjust_finalize(errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_calc_adjust_finalize - -end module temp_calc_adjust diff --git a/test/ddthost_test/temp_calc_adjust.meta b/test/ddthost_test/temp_calc_adjust.meta deleted file mode 100644 index 2a5279a4..00000000 --- a/test/ddthost_test/temp_calc_adjust.meta +++ /dev/null @@ -1,87 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = foo.F90, bar.F90 -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme - process = adjusting -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_level ] - standard_name = potential_temperature_at_interface - units = K - dimensions = (ccpp_constant_one:horizontal_loop_extent, vertical_interface_dimension) - type = real - kind = kind_phys - intent = in -[ temp_calc ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_calc_adjust_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_calc_adjust_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/ddthost_test/temp_set.F90 b/test/ddthost_test/temp_set.F90 deleted file mode 100644 index ce1c32ed..00000000 --- a/test/ddthost_test/temp_set.F90 +++ /dev/null @@ -1,113 +0,0 @@ -!Test 3D parameterization -! - -module temp_set - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: temp_set_init - public :: temp_set_timestep_initialize - public :: temp_set_run - public :: temp_set_finalize - -contains - - !> \section arg_table_temp_set_run Argument Table - !! \htmlinclude arg_table_temp_set_run.html - !! - subroutine temp_set_run(ncol, lev, timestep, temp_level, temp, ps, & - to_promote, promote_pcnst, errmsg, errflg) - !---------------------------------------------------------------- - implicit none - !---------------------------------------------------------------- - - integer, intent(in) :: ncol, lev - real(kind=kind_phys), intent(out) :: temp(:, :) - real(kind=kind_phys), intent(in) :: timestep - real(kind=kind_phys), intent(in) :: ps(:) - real(kind=kind_phys), intent(inout) :: temp_level(:, :) - real(kind=kind_phys), intent(out) :: to_promote(:, :) - real(kind=kind_phys), intent(out) :: promote_pcnst(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - integer :: ilev - - integer :: col_index - integer :: lev_index - - errmsg = '' - errflg = 0 - - ilev = size(temp_level, 2) - if (ilev /= (lev + 1)) then - errflg = 1 - errmsg = 'Invalid value for ilev, must be lev+1' - return - end if - - do col_index = 1, ncol - do lev_index = 1, lev - temp(col_index, lev_index) = (temp_level(col_index, lev_index) & - + temp_level(col_index, lev_index + 1)) / 2.0_kind_phys - end do - end do - - end subroutine temp_set_run - - !> \section arg_table_temp_set_init Argument Table - !! \htmlinclude arg_table_temp_set_init.html - !! - subroutine temp_set_init(temp_inc_in, fudge, temp_inc_set, errmsg, errflg) - - real(kind=kind_phys), intent(in) :: temp_inc_in - real(kind=kind_phys), intent(in) :: fudge - real(kind=kind_phys), intent(out) :: temp_inc_set - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - temp_inc_set = temp_inc_in - - errmsg = '' - errflg = 0 - - end subroutine temp_set_init - - !> \section arg_table_temp_set_timestep_initialize Argument Table - !! \htmlinclude arg_table_temp_set_timestep_initialize.html - !! - subroutine temp_set_timestep_initialize(ncol, temp_inc, temp_level, & - errmsg, errflg) - - integer, intent(in) :: ncol - real(kind=kind_phys), intent(in) :: temp_inc - real(kind=kind_phys), intent(inout) :: temp_level(:, :) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - temp_level = temp_level + temp_inc - - end subroutine temp_set_timestep_initialize - - !> \section arg_table_temp_set_finalize Argument Table - !! \htmlinclude arg_table_temp_set_finalize.html - !! - subroutine temp_set_finalize(errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_set_finalize - -end module temp_set diff --git a/test/ddthost_test/temp_set.meta b/test/ddthost_test/temp_set.meta deleted file mode 100644 index b6c403ce..00000000 --- a/test/ddthost_test/temp_set.meta +++ /dev/null @@ -1,181 +0,0 @@ -[ccpp-table-properties] - name = temp_set - type = scheme -[ccpp-arg-table] - name = temp_set_run - type = scheme - process = setter -[ ncol ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ lev ] - standard_name = vertical_layer_dimension - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_level ] - standard_name = potential_temperature_at_interface - units = K - dimensions = (ccpp_constant_one:horizontal_loop_extent, vertical_interface_dimension) - type = real - kind = kind_phys - intent = inout -[ temp ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = out -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = in -[ to_promote ] - standard_name = promote_this_variable_to_suite - units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - type = real - kind = kind_phys - intent = out -[ promote_pcnst ] - standard_name = promote_this_variable_with_no_horizontal_dimension - units = K - dimensions = (number_of_tracers) - type = real - kind = kind_phys - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -# Init -[ccpp-arg-table] - name = temp_set_init - type = scheme -[ temp_inc_in ] - standard_name = potential_temperature_increment - long_name = Per time step potential temperature increment - units = K - dimensions = () - type = real - kind = kind_phys - intent = in -[ fudge ] - standard_name = random_fudge_factor - long_name = Ignore this - units = 1 - dimensions = () - type = real - kind = kind_phys - intent = in - default_value = 1.0_kind_phys -[ temp_inc_set ] - standard_name = test_potential_temperature_increment - long_name = Per time step potential temperature increment - units = K - dimensions = () - type = real - kind = kind_phys - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -# Timestep Initialization -[ccpp-arg-table] - name = temp_set_timestep_initialize - type = scheme -[ ncol ] - standard_name = horizontal_dimension - type = integer - units = count - dimensions = () - intent = in -[ temp_inc ] - standard_name = test_potential_temperature_increment - long_name = Per time step potential temperature increment - units = K - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_level ] - standard_name = potential_temperature_at_interface - units = K - dimensions = (horizontal_dimension, vertical_interface_dimension) - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -# Finalize -[ccpp-arg-table] - name = temp_set_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/ddthost_test/temp_suite.xml b/test/ddthost_test/temp_suite.xml deleted file mode 100644 index 7a4795c4..00000000 --- a/test/ddthost_test/temp_suite.xml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - setup_coeffs - temp_set - - - temp_calc_adjust - temp_adjust - - diff --git a/test/ddthost_test/test_ddt_host_integration.F90 b/test/ddthost_test/test_ddt_host_integration.F90 deleted file mode 100644 index c3cef458..00000000 --- a/test/ddthost_test/test_ddt_host_integration.F90 +++ /dev/null @@ -1,82 +0,0 @@ -program test - use test_prog, only: test_host, & - suite_info, & - cm, & - cs - - implicit none - - character(len=cs), target :: test_parts1(2) = (/ 'physics1 ', & - 'physics2 ' /) - character(len=cs), target :: test_parts2(1) = (/ 'data_prep ' /) - character(len=cm), target :: test_invars1(7) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'potential_temperature_increment ', & - 'time_step_for_physics ' /) - character(len=cm), target :: test_outvars1(7) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'ccpp_error_code ', & - 'ccpp_error_message ' /) - character(len=cm), target :: test_reqvars1(9) = (/ & - 'potential_temperature ', & - 'potential_temperature_at_interface ', & - 'coefficients_for_interpolation ', & - 'surface_air_pressure ', & - 'water_vapor_specific_humidity ', & - 'potential_temperature_increment ', & - 'time_step_for_physics ', & - 'ccpp_error_code ', & - 'ccpp_error_message ' /) - - character(len=cm), target :: test_invars2(4) = (/ & - 'model_times ', & - 'number_of_model_times ', & - 'surface_air_pressure ', & - 'host_standard_ccpp_type ' /) - - character(len=cm), target :: test_outvars2(5) = (/ & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'model_times ', & - 'surface_air_pressure ', & - 'number_of_model_times ' /) - - character(len=cm), target :: test_reqvars2(6) = (/ & - 'model_times ', & - 'number_of_model_times ', & - 'surface_air_pressure ', & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'host_standard_ccpp_type ' /) - type(suite_info) :: test_suites(2) - logical :: run_okay - - ! Setup expected test suite info - test_suites(1)%suite_name = 'temp_suite' - test_suites(1)%suite_parts => test_parts1 - test_suites(1)%suite_input_vars => test_invars1 - test_suites(1)%suite_output_vars => test_outvars1 - test_suites(1)%suite_required_vars => test_reqvars1 - test_suites(2)%suite_name = 'ddt_suite' - test_suites(2)%suite_parts => test_parts2 - test_suites(2)%suite_input_vars => test_invars2 - test_suites(2)%suite_output_vars => test_outvars2 - test_suites(2)%suite_required_vars => test_reqvars2 - - call test_host(run_okay, test_suites) - - if (run_okay) then - stop 0 - else - stop -1 - end if - -end program test diff --git a/test/ddthost_test/test_host.F90 b/test/ddthost_test/test_host.F90 deleted file mode 100644 index ebe175d9..00000000 --- a/test/ddthost_test/test_host.F90 +++ /dev/null @@ -1,273 +0,0 @@ -module test_prog - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public test_host - - ! Public data and interfaces - integer, public, parameter :: cs = 16 - integer, public, parameter :: cm = 36 - - !> \section arg_table_suite_info Argument Table - !! \htmlinclude arg_table_suite_info.html - !! - type, public :: suite_info - character(len=cs) :: suite_name = '' - character(len=cs), pointer :: suite_parts(:) => null() - character(len=cm), pointer :: suite_input_vars(:) => null() - character(len=cm), pointer :: suite_output_vars(:) => null() - character(len=cm), pointer :: suite_required_vars(:) => null() - end type suite_info - -contains - - logical function check_suite(test_suite) - use test_host_ccpp_cap, only: ccpp_physics_suite_part_list - use test_host_ccpp_cap, only: ccpp_physics_suite_variables - use test_utils, only: check_list - - ! Dummy argument - type(suite_info), intent(in) :: test_suite - ! Local variables - integer :: sind - logical :: check - integer :: errflg - character(len=512) :: errmsg - character(len=128), allocatable :: test_list(:) - - check_suite = .true. - write(6, *) "Checking suite ", trim(test_suite%suite_name) - ! First, check the suite parts - call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_parts, 'part names', & - suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check the input variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.true., output_vars=.false.) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_input_vars, & - 'input variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check the output variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.false., output_vars=.true.) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_output_vars, & - 'output variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check all required variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_required_vars, & - 'required variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - end function check_suite - - !> \section arg_table_test_host Argument Table - !! \htmlinclude arg_table_test_host.html - !! - subroutine test_host(retval, test_suites) - - use host_ccpp_ddt, only: ccpp_info_t - use test_host_mod, only: ncols, & - num_time_steps - use test_host_ccpp_cap, only: test_host_ccpp_physics_initialize - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_initial - use test_host_ccpp_cap, only: test_host_ccpp_physics_run - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_final - use test_host_ccpp_cap, only: test_host_ccpp_physics_finalize - use test_host_ccpp_cap, only: ccpp_physics_suite_list - use test_host_mod, only: init_data, & - compare_data, & - check_model_times - use test_utils, only: check_list - - type(suite_info), intent(in) :: test_suites(:) - logical, intent(out) :: retval - - logical :: check - integer :: col_start - integer :: index, sind - integer :: time_step - integer :: num_suites - character(len=128), allocatable :: suite_names(:) - type(ccpp_info_t) :: ccpp_info - - ! Initialize our 'data' - call init_data() - - ! Gather and test the inspection routines - num_suites = size(test_suites) - call ccpp_physics_suite_list(suite_names) - retval = check_list(suite_names, test_suites(:)%suite_name, & - 'suite names') - write(6, *) 'Available suites are:' - do index = 1, size(suite_names) - do sind = 1, num_suites - if (trim(test_suites(sind)%suite_name) == & - trim(suite_names(index))) then - exit - end if - end do - write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & - ' = test_suites(', sind, ')' - end do - if (retval) then - do sind = 1, num_suites - check = check_suite(test_suites(sind)) - retval = retval .and. check - end do - end if - !!! Return here if any check failed - if (.not. retval) then - return - end if - - ! Use the suite information to setup the run - do sind = 1, num_suites - call test_host_ccpp_physics_initialize(test_suites(sind)%suite_name, & - ccpp_info) - if (ccpp_info%errflg /= 0) then - write(6, '(4a)') 'ERROR in initialize of ', & - trim(test_suites(sind)%suite_name), ': ', trim(ccpp_info%errmsg) - end if - end do - ! Loop over time steps - do time_step = 1, num_time_steps - ! Initialize the timestep - do sind = 1, num_suites - if (ccpp_info%errflg /= 0) then - exit - end if - if (ccpp_info%errflg == 0) then - call test_host_ccpp_physics_timestep_initial( & - test_suites(sind)%suite_name, ccpp_info) - end if - if (ccpp_info%errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(ccpp_info%errmsg) - exit - end if - if (ccpp_info%errflg /= 0) then - exit - end if - end do - - do col_start = 1, ncols, 5 - if (ccpp_info%errflg /= 0) then - exit - end if - ccpp_info%col_start = col_start - ccpp_info%col_end = min(col_start + 4, ncols) - - do sind = 1, num_suites - if (ccpp_info%errflg /= 0) then - exit - end if - do index = 1, size(test_suites(sind)%suite_parts) - if (ccpp_info%errflg /= 0) then - exit - end if - if (ccpp_info%errflg == 0) then - call test_host_ccpp_physics_run( & - test_suites(sind)%suite_name, & - test_suites(sind)%suite_parts(index), & - ccpp_info) - end if - if (ccpp_info%errflg /= 0) then - write(6, '(5a)') trim(test_suites(sind)%suite_name), & - '/', trim(test_suites(sind)%suite_parts(index)), & - ': ', trim(ccpp_info%errmsg) - exit - end if - end do - end do - end do - - do sind = 1, num_suites - if (ccpp_info%errflg /= 0) then - exit - end if - if (ccpp_info%errflg == 0) then - call test_host_ccpp_physics_timestep_final( & - test_suites(sind)%suite_name, ccpp_info) - end if - if (ccpp_info%errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(ccpp_info%errmsg) - exit - end if - end do - end do ! End time step loop - - do sind = 1, num_suites - if (ccpp_info%errflg /= 0) then - exit - end if - if (ccpp_info%errflg == 0) then - call test_host_ccpp_physics_finalize( & - test_suites(sind)%suite_name, ccpp_info) - end if - if (ccpp_info%errflg /= 0) then - write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & - trim(ccpp_info%errmsg) - write(6, '(2a)') 'An error occurred in ccpp_timestep_final, ', & - 'Exiting...' - exit - end if - end do - - if (ccpp_info%errflg == 0) then - ! Run finished without error, check answers - if (.not. check_model_times()) then - write(6, *) 'Model times error!' - ccpp_info%errflg = -1 - else if (compare_data()) then - write(6, *) 'Answers are correct!' - ccpp_info%errflg = 0 - else - write(6, *) 'Answers are not correct!' - ccpp_info%errflg = -1 - end if - end if - - retval = ccpp_info%errflg == 0 - - end subroutine test_host - -end module test_prog diff --git a/test/ddthost_test/test_host.meta b/test/ddthost_test/test_host.meta deleted file mode 100644 index 82fdc462..00000000 --- a/test/ddthost_test/test_host.meta +++ /dev/null @@ -1,18 +0,0 @@ -[ccpp-table-properties] - name = suite_info - type = ddt -[ccpp-arg-table] - name = suite_info - type = ddt - -[ccpp-table-properties] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host -[ ccpp ] - standard_name = host_standard_ccpp_type - type = ccpp_info_t - dimensions = () - protected = False diff --git a/test/ddthost_test/test_host_data.F90 b/test/ddthost_test/test_host_data.F90 deleted file mode 100644 index 88812719..00000000 --- a/test/ddthost_test/test_host_data.F90 +++ /dev/null @@ -1,51 +0,0 @@ -module test_host_data - - use ccpp_kinds, only: kind_phys - - !> \section arg_table_physics_state Argument Table - !! \htmlinclude arg_table_physics_state.html - type physics_state - real(kind=kind_phys), dimension(:), allocatable :: & - ps ! surface pressure - real(kind=kind_phys), dimension(:, :), allocatable :: & - u, & ! zonal wind (m/s) - v, & ! meridional wind (m/s) - pmid ! midpoint pressure (Pa) - - real(kind=kind_phys), dimension(:, :, :), allocatable :: & - q ! constituent mixing ratio (kg/kg moist or dry air depending on type) - end type physics_state - - public allocate_physics_state - -contains - - subroutine allocate_physics_state(cols, levels, constituents, state) - integer, intent(in) :: cols - integer, intent(in) :: levels - integer, intent(in) :: constituents - type(physics_state), intent(out) :: state - - if (allocated(state%ps)) then - deallocate(state%ps) - end if - allocate(state%ps(cols)) - if (allocated(state%u)) then - deallocate(state%u) - end if - allocate(state%u(cols, levels)) - if (allocated(state%v)) then - deallocate(state%v) - end if - allocate(state%v(cols, levels)) - if (allocated(state%pmid)) then - deallocate(state%pmid) - end if - allocate(state%pmid(cols, levels)) - if (allocated(state%q)) then - deallocate(state%q) - end if - allocate(state%q(cols, levels, constituents)) - - end subroutine allocate_physics_state -end module test_host_data diff --git a/test/ddthost_test/test_host_data.meta b/test/ddthost_test/test_host_data.meta deleted file mode 100644 index df4b92b4..00000000 --- a/test/ddthost_test/test_host_data.meta +++ /dev/null @@ -1,52 +0,0 @@ -[ccpp-table-properties] - name = physics_state - type = ddt -[ccpp-arg-table] - name = physics_state - type = ddt -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_dimension) -[ u ] - standard_name = eastward_wind - long_name = Zonal wind - state_variable = true - type = real - kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) -[ v ] - standard_name = northward_wind - long_name = Meridional wind - state_variable = true - type = real - kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) -[ pmid ] - standard_name = air_pressure - long_name = Midpoint air pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_dimension, vertical_layer_dimension) -[ q ] - standard_name = constituent_mixing_ratio - state_variable = true - type = real - kind = kind_phys - units = kg kg-1 moist or dry air depending on type - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) -[ q(:,:,index_of_water_vapor_specific_humidity) ] - standard_name = water_vapor_specific_humidity - state_variable = true - type = real - kind = kind_phys - units = kg kg-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - active = (index_of_water_vapor_specific_humidity > 0) diff --git a/test/ddthost_test/test_host_mod.F90 b/test/ddthost_test/test_host_mod.F90 deleted file mode 100644 index 02eb4991..00000000 --- a/test/ddthost_test/test_host_mod.F90 +++ /dev/null @@ -1,141 +0,0 @@ -module test_host_mod - - use ccpp_kinds, only: kind_phys - use test_host_data, only: physics_state, & - allocate_physics_state - - implicit none - public - - !> \section arg_table_test_host_mod Argument Table - !! \htmlinclude arg_table_test_host_host.html - !! - integer, parameter :: ncols = 10 - integer, parameter :: pver = 5 - integer, parameter :: pverp = 6 - integer, parameter :: pcnst = 2 - integer, parameter :: diagdimstart = 2 - integer, parameter :: index_qv = 1 - real(kind=kind_phys), allocatable :: temp_midpoints(:, :) - real(kind=kind_phys) :: temp_interfaces(ncols, pverp) - real(kind=kind_phys) :: coeffs(ncols) - real(kind=kind_phys), dimension(diagdimstart:ncols, diagdimstart:pver) :: & - diag1, & - diag2 - real(kind=kind_phys) :: dt - real(kind=kind_phys), parameter :: temp_inc = 0.05_kind_phys - type(physics_state) :: phys_state - integer :: num_model_times = -1 - integer, allocatable :: model_times(:) - - integer, parameter :: num_time_steps = 2 - real(kind=kind_phys), parameter :: tolerance = 1.0e-13_kind_phys - real(kind=kind_phys) :: tint_save(ncols, pverp) - - public :: init_data - public :: compare_data - public :: check_model_times - -contains - - subroutine init_data() - - integer :: col - integer :: lev - integer :: cind - integer :: offsize - - ! Allocate and initialize temperature - allocate(temp_midpoints(ncols, pver)) - temp_midpoints = 0.0_kind_phys - do lev = 1, pverp - offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) - do col = 1, ncols - temp_interfaces(col, lev) = real(offsize + col, kind=kind_phys) - tint_save(col, lev) = temp_interfaces(col, lev) - end do - end do - ! Allocate and initialize state - call allocate_physics_state(ncols, pver, pcnst, phys_state) - do cind = 1, pcnst - do lev = 1, pver - offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) - do col = 1, ncols - phys_state%q(col, lev, cind) = real(offsize + col, kind=kind_phys) - end do - end do - end do - - end subroutine init_data - - logical function check_model_times() - - check_model_times = (num_model_times > 0) - if (check_model_times) then - check_model_times = (size(model_times) == num_model_times) - if (.not. check_model_times) then - write(6, '(2(a,i0))') 'model_times size mismatch, ', & - size(model_times), ' should be ', num_model_times - end if - else - write(6, '(a,i0,a)') 'num_model_times mismatch, ', num_model_times, & - ' should be greater than zero' - end if - - end function check_model_times - - logical function compare_data() - - integer :: col - integer :: lev - integer :: cind - integer :: offsize - logical :: need_header - real(kind=kind_phys) :: avg - integer, parameter :: cincrements(pcnst) = (/ 1, 0 /) - - compare_data = .true. - - need_header = .true. - do lev = 1, pver - do col = 1, ncols - avg = (tint_save(col, lev) + tint_save(col, lev + 1)) - avg = 1.0_kind_phys + (avg / 2.0_kind_phys) - avg = avg + (temp_inc * num_time_steps) - if (abs((temp_midpoints(col, lev) - avg) / avg) > tolerance) then - if (need_header) then - write(6, '(" COL LEV T MIDPOINTS EXPECTED")') - need_header = .false. - end if - write(6, '(2i5,2(3x,es15.7))') col, lev, & - temp_midpoints(col, lev), avg - compare_data = .false. - end if - end do - end do - ! Check constituents - need_header = .true. - do cind = 1, pcnst - do lev = 1, pver - offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) - do col = 1, ncols - avg = real(offsize + col + (cincrements(cind) * num_time_steps), & - kind=kind_phys) - if (abs((phys_state%q(col, lev, cind) - avg) / avg) > & - tolerance) then - if (need_header) then - write(6, '(2(2x,a),3x,a,10x,a,14x,a)') & - 'COL', 'LEV', 'C#', 'Q', 'EXPECTED' - need_header = .false. - end if - write(6, '(3i5,2(3x,es15.7))') col, lev, cind, & - phys_state%q(col, lev, cind), avg - compare_data = .false. - end if - end do - end do - end do - - end function compare_data - -end module test_host_mod diff --git a/test/ddthost_test/test_host_mod.meta b/test/ddthost_test/test_host_mod.meta deleted file mode 100644 index a450ee67..00000000 --- a/test/ddthost_test/test_host_mod.meta +++ /dev/null @@ -1,98 +0,0 @@ -[ccpp-table-properties] - name = test_host_mod - type = module -[ccpp-arg-table] - name = test_host_mod - type = module -[ index_qv ] - standard_name = index_of_water_vapor_specific_humidity - units = index - type = integer - protected = True - dimensions = () -[ ncols] - standard_name = horizontal_dimension - units = count - type = integer - protected = True - dimensions = () -[ pver ] - standard_name = vertical_layer_dimension - units = count - type = integer - protected = True - dimensions = () -[ pverP ] - standard_name = vertical_interface_dimension - type = integer - units = count - protected = True - dimensions = () -[ pcnst ] - standard_name = number_of_tracers - type = integer - units = count - protected = True - dimensions = () -[ DiagDimStart ] - standard_name = first_index_of_diag_fields - type = integer - units = count - protected = True - dimensions = () -[ temp_midpoints ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys -[ temp_interfaces ] - standard_name = potential_temperature_at_interface - units = K - dimensions = (horizontal_dimension, vertical_interface_dimension) - type = real | kind = kind_phys -[ diag1 ] - standard_name = diagnostic_stuff_type_1 - long_name = This is just a test field - units = K - dimensions = (first_index_of_diag_fields:horizontal_dimension, first_index_of_diag_fields:vertical_layer_dimension) - type = real | kind = kind_phys -[ diag2 ] - standard_name = diagnostic_stuff_type_2 - long_name = This is just a test field - units = K - dimensions = (first_index_of_diag_fields: horizontal_dimension, first_index_of_diag_fields :vertical_layer_dimension) - type = real | kind = kind_phys -[ dt ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real | kind = kind_phys -[ temp_inc ] - standard_name = potential_temperature_increment - long_name = Per time step potential temperature increment - units = K - dimensions = () - type = real | kind = kind_phys -[ phys_state ] - standard_name = physics_state_derived_type - long_name = Physics State DDT - type = physics_state - dimensions = () -[ num_model_times ] - standard_name = number_of_model_times - type = integer - units = count - dimensions = () -[ model_times ] - standard_name = model_times - units = seconds - dimensions = (number_of_model_times) - type = integer - allocatable = True -[ coeffs ] - standard_name = coefficients_for_interpolation - long_name = coefficients for interpolation - units = none - dimensions = (horizontal_dimension) - type = real | kind = kind_phys \ No newline at end of file diff --git a/test/hash_table_tests/Makefile b/test/hash_table_tests/Makefile deleted file mode 100644 index 5db42275..00000000 --- a/test/hash_table_tests/Makefile +++ /dev/null @@ -1,38 +0,0 @@ -SHELL = /bin/sh - -INCFLAG = -I -INCPATH += $(INCFLAG). -FCFLAGS += -g - -SRCPATH = ../../src -HASHPATH = $(SRCPATH) - -HASHOBJS = ccpp_hashable.o ccpp_hash_table.o - -# Make sure we have a log file -ifeq ($(LOGFILE),) -LOGFILE := ccpp_test.log -endif - -# TARGETS - -ccpp_hashable.o: $(HASHPATH)/ccpp_hashable.F90 - @echo "${FC} -c ${FCFLAGS} ${INCPATH} $^" 2>&1 >> $(LOGFILE) - @${FC} -c ${FCFLAGS} ${INCPATH} $^ 2>&1 >> $(LOGFILE) - -ccpp_hash_table.o: $(HASHPATH)/ccpp_hash_table.F90 - @echo "${FC} -c ${FCFLAGS} ${INCPATH} $^" 2>&1 >> $(LOGFILE) - @${FC} -c ${FCFLAGS} ${INCPATH} $^ 2>&1 >> $(LOGFILE) - -test_hash_table: test_hash.F90 $(HASHOBJS) - @echo "${FC} ${FCFLAGS} ${INCPATH} -o $@ $^" 2>&1 >> $(LOGFILE) - @${FC} ${FCFLAGS} ${INCPATH} -o $@ $^ 2>&1 >> $(LOGFILE) - -test: test_hash_table - @echo "Run Hash Table Tests" - @./test_hash_table - -# CLEAN -clean: - @rm -f *.o *.mod ccpp_test.log - @rm -f test_hash_table diff --git a/test/hash_table_tests/test_hash.F90 b/test/hash_table_tests/test_hash.F90 deleted file mode 100644 index b7faa074..00000000 --- a/test/hash_table_tests/test_hash.F90 +++ /dev/null @@ -1,218 +0,0 @@ -module test_hash_utils - use ccpp_hashable, only: ccpp_hashable_char_t - - implicit none - private - - public :: test_table - - integer, parameter, public :: max_terrs = 16 - - type, public :: hash_object_t - type(ccpp_hashable_char_t), pointer :: item => null() - end type hash_object_t - - private add_error - -contains - - subroutine add_error(msg, num_errs, errors) - ! Dummy arguments - character(len=*), intent(in) :: msg - integer, intent(inout) :: num_errs - character(len=*), intent(inout) :: errors(:) - - if (num_errs < max_terrs) then - num_errs = num_errs + 1 - write(errors(num_errs), *) trim(msg) - end if - - end subroutine add_error - - subroutine test_table(hash_table, table_size, num_tests, num_errs, errors) - use ccpp_hash_table, only: ccpp_hash_table_t, & - ccpp_hash_iterator_t - use ccpp_hashable, only: ccpp_hashable_t, & - new_hashable_char - - ! Dummy arguments - type(ccpp_hash_table_t), target, intent(inout) :: hash_table - integer, intent(in) :: table_size - integer, intent(out) :: num_tests - integer, intent(out) :: num_errs - character(len=*), intent(inout) :: errors(:) - ! Local variables - integer, parameter :: num_test_entries = 4 - integer, parameter :: key_len = 10 - character(len=key_len) :: hash_names(num_test_entries) = (/ & - 'foo ', 'bar ', 'foobar ', 'big daddy ' /) - logical :: hash_found(num_test_entries) - - type(hash_object_t) :: hash_chars(num_test_entries) - class(ccpp_hashable_t), pointer :: test_ptr => null() - type(ccpp_hash_iterator_t) :: hash_iter - character(len=key_len) :: test_key - character(len=len(errors(1))) :: errmsg - integer :: index - - write(6, '(a,i0)') "Testing hash table, size = ", table_size - num_tests = 0 - num_errs = 0 - ! Make sure hash table is *not* initialized - if (hash_table%is_initialized()) then - call add_error("Error: hash table initialized too early", & - num_errs, errors) - end if - num_tests = num_tests + 1 - ! Initialize hash table - call hash_table%initialize(table_size) - ! Make sure hash table is *is* initialized - if (.not. hash_table%is_initialized()) then - call add_error("Error: hash table *not* initialized", num_errs, errors) - end if - num_tests = num_tests + 1 - do index = 1, num_test_entries - call new_hashable_char(hash_names(index), hash_chars(index)%item) - call hash_table%add_hash_key(hash_chars(index)%item, & - errmsg=errors(num_errs + 1)) - if (len_trim(errors(num_errs + 1)) > 0) then - num_errs = num_errs + 1 - end if - if (num_errs > max_terrs) then - exit - end if - end do - - if (num_errs == 0) then - ! We have populated the table, let's do some tests - ! First, make sure we can find existing entries - do index = 1, num_test_entries - test_ptr => hash_table%table_value(hash_names(index), & - errmsg=errors(num_errs + 1)) - if (len_trim(errors(num_errs + 1)) > 0) then - num_errs = num_errs + 1 - else if (trim(test_ptr%key()) /= trim(hash_names(index))) then - num_errs = num_errs + 1 - write(errmsg, *) "ERROR: Found '", trim(test_ptr%key()), & - "', expected '", trim(hash_names(index)), "'" - call add_error(trim(errmsg), num_errs, errors) - end if - if (num_errs > max_terrs) then - exit - end if - end do - num_tests = num_tests + 1 - ! Next, make sure we do not find a non-existent entry - test_ptr => hash_table%table_value(trim(hash_names(1)) // '_oops', & - errmsg=errors(num_errs + 1)) - if (len_trim(errors(num_errs + 1)) == 0) then - write(errmsg, *) "ERROR: Found an entry for '", & - trim(hash_names(1)) // '_oops', "'" - call add_error(trim(errmsg), num_errs, errors) - end if - num_tests = num_tests + 1 - ! Make sure we get an error if we try to add a duplicate key - call hash_table%add_hash_key(hash_chars(2)%item, & - errmsg=errors(num_errs + 1)) - if (len_trim(errors(num_errs + 1)) == 0) then - num_errs = num_errs + 1 - write(errors(num_errs), *) & - "ERROR: Allowed duplicate entry for '", & - hash_chars(2)%item%key(), "'" - end if - num_tests = num_tests + 1 - ! Check that the total number of table entries is correct - if (hash_table%num_values() /= num_test_entries) then - write(errmsg, '(2(a,i0))') "ERROR: Wrong table value count, ", & - hash_table%num_values(), ', should be ', num_test_entries - call add_error(errmsg, num_errs, errors) - end if - num_tests = num_tests + 1 - ! Test iteration through hash table - hash_found(:) = .false. - call hash_iter%initialize(hash_table) - num_tests = num_tests + 1 - do - if (hash_iter%valid()) then - test_key = hash_iter%key() - index = 1 - do - if (trim(test_key) == trim(hash_names(index))) then - hash_found(index) = .true. - exit - else if (index >= num_test_entries) then - write(errmsg, '(3a)') & - "ERROR: Unexpected table entry, '", & - trim(test_key), "'" - call add_error(errmsg, num_errs, errors) - end if - index = index + 1 - end do - call hash_iter%next() - else - exit - end if - end do - call hash_iter%finalize() - if (any(.not. hash_found)) then - write(errmsg, '(a,i0,a)') "ERROR: ", & - count(.not. hash_found), " test keys not found in table." - call add_error(errmsg, num_errs, errors) - end if - end if - ! Finally, clear the hash table (should deallocate everything) - call hash_table%clear() - ! Make sure hash table is *not* initialized - if (hash_table%is_initialized()) then - call add_error("Error: hash table initialized after clear", & - num_errs, errors) - end if - num_tests = num_tests + 1 - ! Cleanup - do index = 1, num_test_entries - deallocate(hash_chars(index)%item) - end do - - end subroutine test_table - -end module test_hash_utils - -program test_hash - use ccpp_hash_table, only: ccpp_hash_table_t - use test_hash_utils, only: test_table, & - max_terrs - - integer, parameter :: num_table_sizes = 5 - integer, parameter :: max_errs = max_terrs * num_table_sizes - integer, parameter :: err_size = 128 - integer, parameter :: test_sizes(num_table_sizes) = (/ & - 0, 1, 2, 4, 20 /) - - type(ccpp_hash_table_t), target :: hash_table - integer :: index - integer :: errcnt = 0 - integer :: num_tests = 0 - integer :: total_errcnt = 0 - integer :: total_tests = 0 - character(len=err_size) :: errors(max_errs) - - errors = '' - do index = 1, num_table_sizes - call test_table(hash_table, test_sizes(index), num_tests, errcnt, & - errors(total_errcnt + 1:)) - total_tests = total_tests + num_tests - total_errcnt = total_errcnt + errcnt - end do - - if (total_errcnt > 0) then - write(6, '(a,i0,a)') 'FAIL, ', total_errcnt, ' errors found' - do index = 1, total_errcnt - write(6, *) trim(errors(index)) - end do - stop 1 - else - write(6, '(a,i0,a)') "All ", total_tests, " hash table tests passed!" - stop 0 - end if - -end program test_hash diff --git a/test/nested_suite_test/CMakeLists.txt b/test/nested_suite_test/CMakeLists.txt deleted file mode 100644 index c55d9bed..00000000 --- a/test/nested_suite_test/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ - -#------------------------------------------------------------------------------ -# -# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES -# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) -# -#------------------------------------------------------------------------------ -set(SCHEME_FILES "effr_calc" "effrs_calc" "effr_diag" "effr_pre" "effr_post" "rad_lw" "rad_sw") -set(HOST_FILES "module_rad_ddt" "test_host_data" "test_host_mod") -set(SUITE_FILES "main_suite.xml") -# HOST is the name of the executable we will build. -# We assume there are files ${HOST}.meta and ${HOST}.F90 in CMAKE_SOURCE_DIR -set(HOST "test_host") - -# By default, generated caps go in ccpp subdir -set(CCPP_CAP_FILES "${CMAKE_CURRENT_BINARY_DIR}/ccpp") - -# Create lists for Fortran and meta data files from file names -list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) -list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_META_FILES) -list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE NESTED_SUITE_HOST_FORTRAN_FILES) -list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE NESTED_SUITE_HOST_METADATA_FILES) - -list(APPEND NESTED_SUITE_HOST_METADATA_FILES "${HOST}.meta") - -# Run ccpp_capgen -ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${NESTED_SUITE_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_META_FILES} - SUITES ${SUITE_FILES} - HOST_NAME ${HOST} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Retrieve the list of Fortran files required for test host from datatable.xml and set to CCPP_CAPS_LIST -ccpp_datafile(DATATABLE "${CCPP_CAP_FILES}/datatable.xml" - REPORT_NAME "--ccpp-files") - -# Create test host library -add_library(NESTED_SUITE_TESTLIB OBJECT ${SCHEME_FORTRAN_FILES} - ${NESTED_SUITE_HOST_FORTRAN_FILES} - ${CCPP_CAPS_LIST}) - -# Setup test executable with needed dependencies -add_executable(nested_suite_host_integration test_nested_suite_integration.F90 ${HOST}.F90) -target_link_libraries(nested_suite_host_integration PRIVATE NESTED_SUITE_TESTLIB test_utils) -target_include_directories(nested_suite_host_integration PRIVATE "$") - -# Add executable to be called with ctest -add_test(NAME ctest_nested_suite_host_integration COMMAND nested_suite_host_integration) diff --git a/test/nested_suite_test/README.md b/test/nested_suite_test/README.md deleted file mode 100644 index d3c3182f..00000000 --- a/test/nested_suite_test/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Nested Suite Test - -Tests the capability to process nested suites: -- Inherited from the variable compatibility test as of 2025/10/01 - - Perform same tests as variable compatibility test at that date -- Parse new XML schema 2.0 -- Expand nested suites at the group level and inside groups - -## Building/Running - -To explicitly build/run the nested suite test host, run: - -```bash -$ cmake --fresh -S -B -DCCPP_RUN_NESTED_SUITE_TEST=ON -$ cd -$ make -$ ctest -``` diff --git a/test/nested_suite_test/ccpp_kinds.F90 b/test/nested_suite_test/ccpp_kinds.F90 deleted file mode 100644 index 2eed03c9..00000000 --- a/test/nested_suite_test/ccpp_kinds.F90 +++ /dev/null @@ -1,27 +0,0 @@ -! -! This work (Common Community Physics Package Framework), identified by -! NOAA, NCAR, CU/CIRES, is free of known copyright restrictions and is -! placed in the public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -!> -!! @brief Auto-generated kinds for CCPP -!! -! -module ccpp_kinds - - use iso_fortran_env, only: & - kind_phys => real64 - - implicit none - private - - public :: kind_phys - -end module ccpp_kinds diff --git a/test/nested_suite_test/effr_calc.F90 b/test/nested_suite_test/effr_calc.F90 deleted file mode 100644 index b8fc43ed..00000000 --- a/test/nested_suite_test/effr_calc.F90 +++ /dev/null @@ -1,84 +0,0 @@ -!Test unit conversions for intent in, inout, out variables -! - -module effr_calc - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: effr_calc_run, effr_calc_init - -contains - !> \section arg_table_effr_calc_init Argument Table - !! \htmlinclude arg_table_effr_calc_init.html - !! - subroutine effr_calc_init(scheme_order, errmsg, errflg) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(inout) :: scheme_order - - errmsg = '' - errflg = 0 - - if (scheme_order /= 2) then - errflg = 1 - errmsg = 'ERROR: effr_calc_init() needs to be called second' - return - else - scheme_order = scheme_order + 1 - end if - - end subroutine effr_calc_init - - !> \section arg_table_effr_calc_run Argument Table - !! \htmlinclude arg_table_effr_calc_run.html - !! - subroutine effr_calc_run(ncol, nlev, effrr_in, effrg_in, ncg_in, nci_out, & - effrl_inout, effri_out, effrs_inout, ncl_out, & - has_graupel, scalar_var, tke_inout, tke2_inout, & - errmsg, errflg) - - integer, intent(in) :: ncol - integer, intent(in) :: nlev - real(kind=kind_phys), intent(in) :: effrr_in(:, :) - real(kind=kind_phys), intent(in), optional :: effrg_in(:, :) - real(kind=kind_phys), intent(in), optional :: ncg_in(:, :) - real(kind=kind_phys), intent(out), optional :: nci_out(:, :) - real(kind=kind_phys), intent(inout) :: effrl_inout(:, :) - real(kind=kind_phys), intent(out), optional :: effri_out(:, :) - real(kind=8), intent(inout) :: effrs_inout(:, :) - logical, intent(in) :: has_graupel - real(kind=kind_phys), intent(inout) :: scalar_var - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - real(kind=kind_phys), intent(out), optional :: ncl_out(:, :) - real(kind=kind_phys), intent(inout) :: tke_inout - real(kind=kind_phys), intent(inout) :: tke2_inout - - !---------------------------------------------------------------- - - real(kind=kind_phys), parameter :: re_qc_min = 2.5 ! microns - real(kind=kind_phys), parameter :: re_qc_max = 50. ! microns - real(kind=kind_phys), parameter :: re_qi_avg = 75. ! microns - real(kind=kind_phys) :: effrr_local(ncol, nlev) - real(kind=kind_phys) :: effrg_local(ncol, nlev) - real(kind=kind_phys) :: ncg_in_local(ncol, nlev) - real(kind=kind_phys) :: nci_out_local(ncol, nlev) - - errmsg = '' - errflg = 0 - - effrr_local = effrr_in - if (present(effrg_in)) effrg_local = effrg_in - if (present(ncg_in)) ncg_in_local = ncg_in - if (present(nci_out)) nci_out_local = nci_out - effrl_inout = min(max(effrl_inout, re_qc_min), re_qc_max) - if (present(effri_out)) effri_out = re_qi_avg - effrs_inout = effrs_inout + (10.0 / 6.0) ! in micrometer - scalar_var = 2.0 ! in km - - end subroutine effr_calc_run - -end module effr_calc diff --git a/test/nested_suite_test/effr_calc.meta b/test/nested_suite_test/effr_calc.meta deleted file mode 100644 index c3733f13..00000000 --- a/test/nested_suite_test/effr_calc.meta +++ /dev/null @@ -1,163 +0,0 @@ -[ccpp-table-properties] - name = effr_calc - type = scheme - dependencies = -######################################################################## -[ccpp-arg-table] - name = effr_calc_init - type = scheme -[ scheme_order ] - standard_name = scheme_order_in_suite - long_name = scheme order in suite definition file - units = None - dimensions = () - type = integer - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -######################################################################## -[ccpp-arg-table] - name = effr_calc_run - type = scheme -[ ncol ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ nlev ] - standard_name = vertical_layer_dimension - type = integer - units = count - dimensions = () - intent = in -[effrr_in] - standard_name = effective_radius_of_stratiform_cloud_rain_particle - long_name = effective radius of cloud rain particle in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - top_at_one = True -[effrg_in] - standard_name = effective_radius_of_stratiform_cloud_graupel - long_name = effective radius of cloud graupel in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - optional = True -[ncg_in] - standard_name = cloud_graupel_number_concentration - long_name = number concentration of cloud graupel - units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - optional = True -[nci_out] - standard_name = cloud_ice_number_concentration - long_name = number concentration of cloud ice - units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = out - optional = True -[effrl_inout] - standard_name = effective_radius_of_stratiform_cloud_liquid_water_particle - long_name = effective radius of cloud liquid water particle in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout -[effri_out] - standard_name = effective_radius_of_stratiform_cloud_ice_particle - long_name = effective radius of cloud ice water particle in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = out - optional = True -[effrs_inout] - standard_name = effective_radius_of_stratiform_cloud_snow_particle - long_name = effective radius of cloud snow particle in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = 8 - intent = inout - top_at_one = True -[ncl_out] - standard_name = cloud_liquid_number_concentration - long_name = number concentration of cloud liquid - units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = out - optional = True -[has_graupel] - standard_name = flag_indicating_cloud_microphysics_has_graupel - long_name = flag indicating that the cloud microphysics produces graupel - units = flag - dimensions = () - type = logical - intent = in -[ scalar_var ] - standard_name = scalar_variable_for_testing - long_name = scalar variable for testing - units = km - dimensions = () - type = real - kind = kind_phys - intent = inout -[ tke_inout ] - standard_name = turbulent_kinetic_energy - long_name = turbulent_kinetic_energy - units = m2 s-2 - dimensions = () - type = real - kind = kind_phys - intent = inout -[ tke2_inout ] - standard_name = turbulent_kinetic_energy2 - long_name = turbulent_kinetic_energy2 - units = m+2 s-2 - dimensions = () - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/nested_suite_test/effr_diag.F90 b/test/nested_suite_test/effr_diag.F90 deleted file mode 100644 index 75da29c7..00000000 --- a/test/nested_suite_test/effr_diag.F90 +++ /dev/null @@ -1,68 +0,0 @@ -!Test unit conversions for intent in, inout, out variables -! - -module effr_diag - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: effr_diag_run, effr_diag_init - -contains - - !> \section arg_table_effr_diag_init Argument Table - !! \htmlinclude arg_table_effr_diag_init.html - !! - subroutine effr_diag_init(scheme_order, errmsg, errflg) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(inout) :: scheme_order - - errmsg = '' - errflg = 0 - - if (scheme_order /= 4) then - errflg = 1 - errmsg = 'ERROR: effr_diag_init() needs to be called fourth' - return - else - scheme_order = scheme_order + 1 - end if - - end subroutine effr_diag_init - - !> \section arg_table_effr_diag_run Argument Table - !! \htmlinclude arg_table_effr_diag_run.html - !! - subroutine effr_diag_run(effrr_in, scalar_var, errmsg, errflg) - - real(kind=kind_phys), intent(in) :: effrr_in(:, :) - integer, intent(in) :: scalar_var - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - real(kind=kind_phys) :: effrr_min, effrr_max - - errmsg = '' - errflg = 0 - - call cmp_effr_diag(effrr_in, effrr_min, effrr_max) - - if (scalar_var /= 380) then - errmsg = 'ERROR: effr_diag_run(): scalar_var should be 380' - errflg = 1 - end if - end subroutine effr_diag_run - - subroutine cmp_effr_diag(effr, effr_min, effr_max) - real(kind=kind_phys), intent(in) :: effr(:, :) - real(kind=kind_phys), intent(out) :: effr_min, effr_max - - ! Do some diagnostic calcualtions... - effr_min = minval(effr) - effr_max = maxval(effr) - - end subroutine cmp_effr_diag -end module effr_diag diff --git a/test/nested_suite_test/effr_diag.meta b/test/nested_suite_test/effr_diag.meta deleted file mode 100644 index 9e0e4fc2..00000000 --- a/test/nested_suite_test/effr_diag.meta +++ /dev/null @@ -1,65 +0,0 @@ -[ccpp-table-properties] - name = effr_diag - type = scheme - dependencies = -######################################################################## -[ccpp-arg-table] - name = effr_diag_init - type = scheme -[ scheme_order ] - standard_name = scheme_order_in_suite - long_name = scheme order in suite definition file - units = None - dimensions = () - type = integer - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -######################################################################## -[ccpp-arg-table] - name = effr_diag_run - type = scheme -[effrr_in] - standard_name = effective_radius_of_stratiform_cloud_rain_particle - long_name = effective radius of cloud rain particle in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - top_at_one = True -[ scalar_var ] - standard_name = scalar_variable_for_testing_c - long_name = unused scalar variable C - units = m - dimensions = () - type = integer - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/nested_suite_test/effr_post.F90 b/test/nested_suite_test/effr_post.F90 deleted file mode 100644 index 01357350..00000000 --- a/test/nested_suite_test/effr_post.F90 +++ /dev/null @@ -1,61 +0,0 @@ -!Test unit conversions for intent in, inout, out variables -! - -module effr_post - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: effr_post_run, effr_post_init - -contains - - !> \section arg_table_effr_post_init Argument Table - !! \htmlinclude arg_table_effr_post_init.html - !! - subroutine effr_post_init(scheme_order, errmsg, errflg) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(inout) :: scheme_order - - errmsg = '' - errflg = 0 - - if (scheme_order /= 3) then - errflg = 1 - errmsg = 'ERROR: effr_post_init() needs to be called third' - return - else - scheme_order = scheme_order + 1 - end if - - end subroutine effr_post_init - - !> \section arg_table_effr_post_run Argument Table - !! \htmlinclude arg_table_effr_post_run.html - !! - subroutine effr_post_run(effrr_inout, scalar_var, errmsg, errflg) - - real(kind=kind_phys), intent(inout) :: effrr_inout(:, :) - real(kind=kind_phys), intent(in) :: scalar_var - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - real(kind=kind_phys) :: effrr_min, effrr_max - - errmsg = '' - errflg = 0 - - ! Do some post-processing on effrr... - effrr_inout(:, :) = effrr_inout(:, :) * 1._kind_phys - - if (scalar_var /= 1013.0) then - errmsg = 'ERROR: effr_post_run(): scalar_var should be 1013.0' - errflg = 1 - end if - - end subroutine effr_post_run - -end module effr_post diff --git a/test/nested_suite_test/effr_post.meta b/test/nested_suite_test/effr_post.meta deleted file mode 100644 index 721582a6..00000000 --- a/test/nested_suite_test/effr_post.meta +++ /dev/null @@ -1,65 +0,0 @@ -[ccpp-table-properties] - name = effr_post - type = scheme - dependencies = -######################################################################## -[ccpp-arg-table] - name = effr_post_init - type = scheme -[ scheme_order ] - standard_name = scheme_order_in_suite - long_name = scheme order in suite definition file - units = None - dimensions = () - type = integer - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -######################################################################## -[ccpp-arg-table] - name = effr_post_run - type = scheme -[effrr_inout] - standard_name = effective_radius_of_stratiform_cloud_rain_particle - long_name = effective radius of cloud rain particle in micrometer - units = m - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout -[ scalar_var ] - standard_name = scalar_variable_for_testing_b - long_name = unused scalar variable B - units = m - dimensions = () - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/nested_suite_test/effr_pre.F90 b/test/nested_suite_test/effr_pre.F90 deleted file mode 100644 index a2fe2f5c..00000000 --- a/test/nested_suite_test/effr_pre.F90 +++ /dev/null @@ -1,60 +0,0 @@ -!Test unit conversions for intent in, inout, out variables -! - -module mod_effr_pre - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: effr_pre_run, effr_pre_init - -contains - !> \section arg_table_effr_pre_init Argument Table - !! \htmlinclude arg_table_effr_pre_init.html - !! - subroutine effr_pre_init(scheme_order, errmsg, errflg) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(inout) :: scheme_order - - errmsg = '' - errflg = 0 - - if (scheme_order /= 1) then - errflg = 1 - errmsg = 'ERROR: effr_pre_init() needs to be called first' - return - else - scheme_order = scheme_order + 1 - end if - - end subroutine effr_pre_init - - !> \section arg_table_effr_pre_run Argument Table - !! \htmlinclude arg_table_effr_pre_run.html - !! - subroutine effr_pre_run(effrr_inout, scalar_var, errmsg, errflg) - - real(kind=kind_phys), intent(inout) :: effrr_inout(:, :) - real(kind=kind_phys), intent(in) :: scalar_var - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - real(kind=kind_phys) :: effrr_min, effrr_max - - errmsg = '' - errflg = 0 - - ! Do some pre-processing on effrr... - effrr_inout(:, :) = effrr_inout(:, :) * 1._kind_phys - - if (scalar_var /= 273.15) then - errmsg = 'ERROR: effr_pre_run(): scalar_var should be 273.15' - errflg = 1 - end if - - end subroutine effr_pre_run - -end module mod_effr_pre diff --git a/test/nested_suite_test/effr_pre.meta b/test/nested_suite_test/effr_pre.meta deleted file mode 100644 index 251b4175..00000000 --- a/test/nested_suite_test/effr_pre.meta +++ /dev/null @@ -1,66 +0,0 @@ -[ccpp-table-properties] - name = effr_pre - type = scheme - module_name = mod_effr_pre - dependencies = -######################################################################## -[ccpp-arg-table] - name = effr_pre_init - type = scheme -[ scheme_order ] - standard_name = scheme_order_in_suite - long_name = scheme order in suite definition file - units = None - dimensions = () - type = integer - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -######################################################################## -[ccpp-arg-table] - name = effr_pre_run - type = scheme -[effrr_inout] - standard_name = effective_radius_of_stratiform_cloud_rain_particle - long_name = effective radius of cloud rain particle in micrometer - units = m - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout -[ scalar_var ] - standard_name = scalar_variable_for_testing_a - long_name = unused scalar variable A - units = m - dimensions = () - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/nested_suite_test/effrs_calc.F90 b/test/nested_suite_test/effrs_calc.F90 deleted file mode 100644 index 3aa8d196..00000000 --- a/test/nested_suite_test/effrs_calc.F90 +++ /dev/null @@ -1,32 +0,0 @@ -!Test unit conversions for intent in, inout, out variables -! - -module effrs_calc - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: effrs_calc_run - -contains - !> \section arg_table_effrs_calc_run Argument Table - !! \htmlinclude arg_table_effrs_calc_run.html - !! - subroutine effrs_calc_run(effrs_inout, errmsg, errflg) - - real(kind=kind_phys), intent(inout) :: effrs_inout(:, :) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - !---------------------------------------------------------------- - - errmsg = '' - errflg = 0 - - effrs_inout = effrs_inout + (10.E-6_kind_phys / 3._kind_phys) ! in meters - - end subroutine effrs_calc_run - -end module effrs_calc diff --git a/test/nested_suite_test/effrs_calc.meta b/test/nested_suite_test/effrs_calc.meta deleted file mode 100644 index 9ce7b88e..00000000 --- a/test/nested_suite_test/effrs_calc.meta +++ /dev/null @@ -1,25 +0,0 @@ -[ccpp-table-properties] - name = effrs_calc - type = scheme - -[ccpp-arg-table] - name = effrs_calc_run - type = scheme -[ effrs_inout ] - standard_name = effective_radius_of_stratiform_cloud_snow_particle - units = m - type = real | kind = kind_phys - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - units = none - type = character | kind = len=512 - dimensions = () - intent = out -[ errflg ] - standard_name = ccpp_error_code - units = 1 - type = integer - dimensions = () - intent = out diff --git a/test/nested_suite_test/main_suite.xml b/test/nested_suite_test/main_suite.xml deleted file mode 100644 index a319ec47..00000000 --- a/test/nested_suite_test/main_suite.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - effr_pre - - - effr_calc - - - effr_post - - - - - - diff --git a/test/nested_suite_test/module_rad_ddt.F90 b/test/nested_suite_test/module_rad_ddt.F90 deleted file mode 100644 index 6e992250..00000000 --- a/test/nested_suite_test/module_rad_ddt.F90 +++ /dev/null @@ -1,23 +0,0 @@ -module mod_rad_ddt - use ccpp_kinds, only: kind_phys - implicit none - - public ty_rad_lw, ty_rad_sw - - !> \section arg_table_ty_rad_lw Argument Table - !! \htmlinclude arg_table_ty_rad_lw.html - !! - type ty_rad_lw - real(kind=kind_phys) :: sfc_up_lw - real(kind=kind_phys) :: sfc_down_lw - end type ty_rad_lw - - !> \section arg_table_ty_rad_sw Argument Table - !! \htmlinclude arg_table_ty_rad_sw.html - !! - type ty_rad_sw - real(kind=kind_phys), pointer :: sfc_up_sw(:) => null() - real(kind=kind_phys), pointer :: sfc_down_sw(:) => null() - end type ty_rad_sw - -end module mod_rad_ddt diff --git a/test/nested_suite_test/module_rad_ddt.meta b/test/nested_suite_test/module_rad_ddt.meta deleted file mode 100644 index c4792547..00000000 --- a/test/nested_suite_test/module_rad_ddt.meta +++ /dev/null @@ -1,40 +0,0 @@ -[ccpp-table-properties] - name = ty_rad_lw - type = ddt - dependencies = - module_name = mod_rad_ddt -[ccpp-arg-table] - name = ty_rad_lw - type = ddt -[ sfc_up_lw ] - standard_name = surface_upwelling_longwave_radiation_flux - units = W m2 - dimensions = () - type = real - kind = kind_phys -[ sfc_down_lw ] - standard_name = surface_downwelling_longwave_radiation_flux - units = W m2 - dimensions = () - type = real - kind = kind_phys - -[ccpp-table-properties] - name = ty_rad_sw - type = ddt - module_name = mod_rad_ddt -[ccpp-arg-table] - name = ty_rad_sw - type = ddt -[ sfc_up_sw ] - standard_name = surface_upwelling_shortwave_radiation_flux - units = W m2 - dimensions = (horizontal_dimension) - type = real - kind = kind_phys -[ sfc_down_sw ] - standard_name = surface_downwelling_shortwave_radiation_flux - units = W m2 - dimensions = (horizontal_dimension) - type = real - kind = kind_phys diff --git a/test/nested_suite_test/rad_lw.F90 b/test/nested_suite_test/rad_lw.F90 deleted file mode 100644 index ded4861f..00000000 --- a/test/nested_suite_test/rad_lw.F90 +++ /dev/null @@ -1,35 +0,0 @@ -module rad_lw - use ccpp_kinds, only: kind_phys - use mod_rad_ddt, only: ty_rad_lw - - implicit none - private - - public :: rad_lw_run - -contains - - !> \section arg_table_rad_lw_run Argument Table - !! \htmlinclude arg_table_rad_lw_run.html - !! - subroutine rad_lw_run(ncol, fluxlw, errmsg, errflg) - - integer, intent(in) :: ncol - type(ty_rad_lw), intent(inout) :: fluxlw(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! Locals - integer :: icol - - errmsg = '' - errflg = 0 - - do icol = 1, ncol - fluxlw(icol)%sfc_up_lw = 300._kind_phys - fluxlw(icol)%sfc_down_lw = 50._kind_phys - end do - - end subroutine rad_lw_run - -end module rad_lw diff --git a/test/nested_suite_test/rad_lw.meta b/test/nested_suite_test/rad_lw.meta deleted file mode 100644 index 883edf1b..00000000 --- a/test/nested_suite_test/rad_lw.meta +++ /dev/null @@ -1,35 +0,0 @@ -[ccpp-table-properties] - name = rad_lw - type = scheme - dependencies = module_rad_ddt.F90 -[ccpp-arg-table] - name = rad_lw_run - type = scheme -[ ncol ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[fluxLW] - standard_name = longwave_radiation_fluxes - long_name = longwave radiation fluxes - units = W m-2 - dimensions = (horizontal_loop_extent) - type = ty_rad_lw - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/nested_suite_test/rad_sw.F90 b/test/nested_suite_test/rad_sw.F90 deleted file mode 100644 index 64756217..00000000 --- a/test/nested_suite_test/rad_sw.F90 +++ /dev/null @@ -1,35 +0,0 @@ -module rad_sw - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: rad_sw_run - -contains - - !> \section arg_table_rad_sw_run Argument Table - !! \htmlinclude arg_table_rad_sw_run.html - !! - subroutine rad_sw_run(ncol, sfc_up_sw, sfc_down_sw, errmsg, errflg) - - integer, intent(in) :: ncol - real(kind=kind_phys), intent(inout) :: sfc_up_sw(:) - real(kind=kind_phys), intent(inout) :: sfc_down_sw(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! Locals - integer :: icol - - errmsg = '' - errflg = 0 - - do icol = 1, ncol - sfc_up_sw(icol) = 100._kind_phys - sfc_down_sw(icol) = 400._kind_phys - end do - - end subroutine rad_sw_run - -end module rad_sw diff --git a/test/nested_suite_test/rad_sw.meta b/test/nested_suite_test/rad_sw.meta deleted file mode 100644 index d88b9acc..00000000 --- a/test/nested_suite_test/rad_sw.meta +++ /dev/null @@ -1,41 +0,0 @@ -[ccpp-table-properties] - name = rad_sw - type = scheme -[ccpp-arg-table] - name = rad_sw_run - type = scheme -[ ncol ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ sfc_up_sw ] - standard_name = surface_upwelling_shortwave_radiation_flux - units = W m2 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ sfc_down_sw ] - standard_name = surface_downwelling_shortwave_radiation_flux - units = W m2 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/nested_suite_test/radiation2_suite.xml b/test/nested_suite_test/radiation2_suite.xml deleted file mode 100644 index e20b81e8..00000000 --- a/test/nested_suite_test/radiation2_suite.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - effrs_calc - - effr_diag - - diff --git a/test/nested_suite_test/radiation3_subsuite.xml b/test/nested_suite_test/radiation3_subsuite.xml deleted file mode 100644 index 346db62d..00000000 --- a/test/nested_suite_test/radiation3_subsuite.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - rad_sw - - diff --git a/test/nested_suite_test/radiation3_suite.xml b/test/nested_suite_test/radiation3_suite.xml deleted file mode 100644 index 89e5bc13..00000000 --- a/test/nested_suite_test/radiation3_suite.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/test/nested_suite_test/radiation4_suite.xml b/test/nested_suite_test/radiation4_suite.xml deleted file mode 100644 index d3df4fb9..00000000 --- a/test/nested_suite_test/radiation4_suite.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - rad_lw - - diff --git a/test/nested_suite_test/test_host.F90 b/test/nested_suite_test/test_host.F90 deleted file mode 100644 index 67c7a1ac..00000000 --- a/test/nested_suite_test/test_host.F90 +++ /dev/null @@ -1,264 +0,0 @@ -module test_prog - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public test_host - - ! Public data and interfaces - integer, public, parameter :: cs = 32 - integer, public, parameter :: cm = 60 - - !> \section arg_table_suite_info Argument Table - !! \htmlinclude arg_table_suite_info.html - !! - type, public :: suite_info - character(len=cs) :: suite_name = '' - character(len=cs), pointer :: suite_parts(:) => null() - character(len=cm), pointer :: suite_input_vars(:) => null() - character(len=cm), pointer :: suite_output_vars(:) => null() - character(len=cm), pointer :: suite_required_vars(:) => null() - end type suite_info - -contains - - logical function check_suite(test_suite) - use test_host_ccpp_cap, only: ccpp_physics_suite_part_list - use test_host_ccpp_cap, only: ccpp_physics_suite_variables - use test_utils, only: check_list - - ! Dummy argument - type(suite_info), intent(in) :: test_suite - ! Local variables - integer :: sind - logical :: check - integer :: errflg - character(len=512) :: errmsg - character(len=128), allocatable :: test_list(:) - - check_suite = .true. - write(6, *) "Checking suite ", trim(test_suite%suite_name) - ! First, check the suite parts - call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_parts, 'part names', & - suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check the input variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.true., output_vars=.false.) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_input_vars, & - 'input variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check the output variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.false., output_vars=.true.) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_output_vars, & - 'output variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check all required variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_required_vars, & - 'required variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - end function check_suite - - !> \section arg_table_test_host Argument Table - !! \htmlinclude arg_table_test_host.html - !! - subroutine test_host(retval, test_suites) - - use test_host_mod, only: ncols - use test_host_ccpp_cap, only: test_host_ccpp_physics_initialize - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_initial - use test_host_ccpp_cap, only: test_host_ccpp_physics_run - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_final - use test_host_ccpp_cap, only: test_host_ccpp_physics_finalize - use test_host_ccpp_cap, only: ccpp_physics_suite_list - use test_host_mod, only: init_data, & - compare_data - use test_utils, only: check_list - - type(suite_info), intent(in) :: test_suites(:) - logical, intent(out) :: retval - - logical :: check - integer :: col_start, col_end - integer :: index, sind - integer :: num_suites - character(len=128), allocatable :: suite_names(:) - character(len=512) :: errmsg - integer :: errflg - - ! Initialize our 'data' - call init_data() - - ! Gather and test the inspection routines - num_suites = size(test_suites) - call ccpp_physics_suite_list(suite_names) - retval = check_list(suite_names, test_suites(:)%suite_name, & - 'suite names') - write(6, *) 'Available suites are:' - do index = 1, size(suite_names) - do sind = 1, num_suites - if (trim(test_suites(sind)%suite_name) == & - trim(suite_names(index))) then - exit - end if - end do - write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & - ' = test_suites(', sind, ')' - end do - if (retval) then - do sind = 1, num_suites - check = check_suite(test_suites(sind)) - retval = retval .and. check - end do - end if - !!! Return here if any check failed - if (.not. retval) then - return - end if - - ! Use the suite information to setup the run - do sind = 1, num_suites - call test_host_ccpp_physics_initialize(test_suites(sind)%suite_name, & - errmsg, errflg) - if (errflg /= 0) then - write(6, '(4a)') 'ERROR in initialize of ', & - trim(test_suites(sind)%suite_name), ': ', trim(errmsg) - end if - end do - - ! Initialize the timestep - do sind = 1, num_suites - if (errflg /= 0) then - exit - end if - if (errflg == 0) then - call test_host_ccpp_physics_timestep_initial( & - test_suites(sind)%suite_name, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(errmsg) - exit - end if - if (errflg /= 0) then - exit - end if - end do - - do col_start = 1, ncols, 5 - if (errflg /= 0) then - exit - end if - col_end = min(col_start + 4, ncols) - - do sind = 1, num_suites - if (errflg /= 0) then - exit - end if - do index = 1, size(test_suites(sind)%suite_parts) - if (errflg /= 0) then - exit - end if - if (errflg == 0) then - call test_host_ccpp_physics_run( & - test_suites(sind)%suite_name, & - test_suites(sind)%suite_parts(index), & - col_start, col_end, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(5a)') trim(test_suites(sind)%suite_name), & - '/', trim(test_suites(sind)%suite_parts(index)), & - ': ', trim(errmsg) - exit - end if - end do - end do - end do - - do sind = 1, num_suites - if (errflg /= 0) then - exit - end if - if (errflg == 0) then - call test_host_ccpp_physics_timestep_final( & - test_suites(sind)%suite_name, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(errmsg) - exit - end if - end do - - do sind = 1, num_suites - if (errflg /= 0) then - exit - end if - if (errflg == 0) then - call test_host_ccpp_physics_finalize( & - test_suites(sind)%suite_name, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & - trim(errmsg) - write(6, '(2a)') 'An error occurred in ccpp_timestep_final, ', & - 'Exiting...' - exit - end if - end do - - if (errflg == 0) then - ! Run finished without error, check answers - if (compare_data()) then - write(6, *) 'Answers are correct!' - errflg = 0 - else - write(6, *) 'Answers are not correct!' - errflg = -1 - end if - end if - - retval = errflg == 0 - - end subroutine test_host - -end module test_prog diff --git a/test/nested_suite_test/test_host.meta b/test/nested_suite_test/test_host.meta deleted file mode 100644 index da71b182..00000000 --- a/test/nested_suite_test/test_host.meta +++ /dev/null @@ -1,38 +0,0 @@ -[ccpp-table-properties] - name = suite_info - type = ddt -[ccpp-arg-table] - name = suite_info - type = ddt - -[ccpp-table-properties] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True -[ col_end ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - protected = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = None - dimensions = () - type = character - kind = len=512 -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer diff --git a/test/nested_suite_test/test_host_data.F90 b/test/nested_suite_test/test_host_data.F90 deleted file mode 100644 index ece60034..00000000 --- a/test/nested_suite_test/test_host_data.F90 +++ /dev/null @@ -1,103 +0,0 @@ -module test_host_data - - use ccpp_kinds, only: kind_phys - use mod_rad_ddt, only: ty_rad_lw, & - ty_rad_sw - - implicit none - private - - !> \section arg_table_physics_state Argument Table - !! \htmlinclude arg_table_physics_state.html - type physics_state - real(kind=kind_phys), dimension(:, :), allocatable :: & - effrr, & ! effective radius of cloud rain - effrl, & ! effective radius of cloud liquid water - effri, & ! effective radius of cloud ice - effrg, & ! effective radius of cloud graupel - ncg, & ! number concentration of cloud graupel - nci ! number concentration of cloud ice - real(kind=kind_phys) :: scalar_var - type(ty_rad_lw), dimension(:), allocatable :: & - fluxlw ! Longwave radiation fluxes - type(ty_rad_sw) :: & - fluxsw ! Shortwave radiation fluxes - real(kind=kind_phys) :: scalar_vara - real(kind=kind_phys) :: scalar_varb - real(kind=kind_phys) :: tke, tke2 - integer :: scalar_varc - integer :: scheme_order - integer :: num_subcycles - end type physics_state - - public :: physics_state - public :: allocate_physics_state - -contains - - subroutine allocate_physics_state(cols, levels, state, has_graupel, has_ice) - integer, intent(in) :: cols - integer, intent(in) :: levels - type(physics_state), intent(out) :: state - logical, intent(in) :: has_graupel - logical, intent(in) :: has_ice - - if (allocated(state%effrr)) then - deallocate(state%effrr) - end if - allocate(state%effrr(cols, levels)) - - if (allocated(state%effrl)) then - deallocate(state%effrl) - end if - allocate(state%effrl(cols, levels)) - - if (has_ice) then - if (allocated(state%effri)) then - deallocate(state%effri) - end if - allocate(state%effri(cols, levels)) - end if - - if (has_graupel) then - if (allocated(state%effrg)) then - deallocate(state%effrg) - end if - allocate(state%effrg(cols, levels)) - - if (allocated(state%ncg)) then - deallocate(state%ncg) - end if - allocate(state%ncg(cols, levels)) - end if - - if (has_ice) then - if (allocated(state%nci)) then - deallocate(state%nci) - end if - allocate(state%nci(cols, levels)) - end if - - if (allocated(state%fluxlw)) then - deallocate(state%fluxlw) - end if - allocate(state%fluxlw(cols)) - - if (associated(state%fluxsw%sfc_up_sw)) then - nullify(state%fluxsw%sfc_up_sw) - end if - allocate(state%fluxsw%sfc_up_sw(cols)) - - if (associated(state%fluxsw%sfc_down_sw)) then - nullify(state%fluxsw%sfc_down_sw) - end if - allocate(state%fluxsw%sfc_down_sw(cols)) - - ! Initialize scheme counter. - state%scheme_order = 1 - ! Initialize subcycle counter. - state%num_subcycles = 3 - - end subroutine allocate_physics_state - -end module test_host_data diff --git a/test/nested_suite_test/test_host_data.meta b/test/nested_suite_test/test_host_data.meta deleted file mode 100644 index 59a0fb4d..00000000 --- a/test/nested_suite_test/test_host_data.meta +++ /dev/null @@ -1,128 +0,0 @@ -[ccpp-table-properties] - name = physics_state - type = ddt - dependencies = module_rad_ddt.F90 -[ccpp-arg-table] - name = physics_state - type = ddt -[effrr] - standard_name = effective_radius_of_stratiform_cloud_rain_particle - long_name = effective radius of cloud rain particle in meter - units = m - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys -[effrl] - standard_name = effective_radius_of_stratiform_cloud_liquid_water_particle - long_name = effective radius of cloud liquid water particle in meter - units = m - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys -[effri] - standard_name = effective_radius_of_stratiform_cloud_ice_particle - long_name = effective radius of cloud ice water particle in meter - units = m - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - active = (flag_indicating_cloud_microphysics_has_ice) -[effrg] - standard_name = effective_radius_of_stratiform_cloud_graupel - long_name = effective radius of cloud graupel in meter - units = m - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - active = (flag_indicating_cloud_microphysics_has_graupel) -[ncg] - standard_name = cloud_graupel_number_concentration - long_name = number concentration of cloud graupel - units = kg-1 - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - active = (flag_indicating_cloud_microphysics_has_graupel) -[nci] - standard_name = cloud_ice_number_concentration - long_name = number concentration of cloud ice - units = kg-1 - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - active = (flag_indicating_cloud_microphysics_has_ice) -[scalar_var] - standard_name = scalar_variable_for_testing - long_name = unused scalar variable - units = m - dimensions = () - type = real - kind = kind_phys -[ tke ] - standard_name = turbulent_kinetic_energy - long_name = turbulent_kinetic_energy - units = J kg-1 - dimensions = () - type = real - kind = kind_phys -[ tke2 ] - standard_name = turbulent_kinetic_energy2 - long_name = turbulent_kinetic_energy2 - units = m2 s-2 - dimensions = () - type = real - kind = kind_phys -[fluxSW] - standard_name = shortwave_radiation_fluxes - long_name = shortwave radiation fluxes - units = W m-2 - dimensions = () - type = ty_rad_sw -[fluxLW] - standard_name = longwave_radiation_fluxes - long_name = longwave radiation fluxes - units = W m-2 - dimensions = (horizontal_dimension) - type = ty_rad_lw -[scalar_varA] - standard_name = scalar_variable_for_testing_a - long_name = unused scalar variable A - units = m - dimensions = () - type = real - kind = kind_phys -[scalar_varB] - standard_name = scalar_variable_for_testing_b - long_name = unused scalar variable B - units = m - dimensions = () - type = real - kind = kind_phys -[scalar_varC] - standard_name = scalar_variable_for_testing_c - long_name = unused scalar variable C - units = m - dimensions = () - type = integer -[scheme_order] - standard_name = scheme_order_in_suite - long_name = scheme order in suite definition file - units = None - dimensions = () - type = integer -[num_subcycles] - standard_name = num_subcycles_for_effr - long_name = Number of times to subcycle the effr calculation - units = None - dimensions = () - type = integer - -[ccpp-table-properties] - name = test_host_data - type = module - dependencies = module_rad_ddt.F90 -[ccpp-arg-table] - name = test_host_data - type = module diff --git a/test/nested_suite_test/test_host_mod.F90 b/test/nested_suite_test/test_host_mod.F90 deleted file mode 100644 index d3bde866..00000000 --- a/test/nested_suite_test/test_host_mod.F90 +++ /dev/null @@ -1,132 +0,0 @@ -module test_host_mod - - use ccpp_kinds, only: kind_phys - use test_host_data, only: physics_state, & - allocate_physics_state - - implicit none - public - - !> \section arg_table_test_host_mod Argument Table - !! \htmlinclude arg_table_test_host_host.html - !! - integer, parameter :: ncols = 12 - integer, parameter :: pver = 4 - type(physics_state) :: phys_state - real(kind=kind_phys) :: effrs(ncols, pver) - logical, parameter :: has_ice = .true. - logical, parameter :: has_graupel = .true. - - public :: init_data - public :: compare_data - -contains - - subroutine init_data() - - ! Allocate and initialize state - call allocate_physics_state(ncols, pver, phys_state, has_graupel, has_ice) - phys_state%effrr = 1.0E-3 ! 1000 microns, in meter - phys_state%effrl = 1.0E-4 ! 100 microns, in meter - phys_state%scalar_var = 1.0 ! in m - phys_state%scalar_vara = 273.15 ! in K - phys_state%scalar_varb = 1013.0 ! in mb - phys_state%scalar_varc = 380 ! in ppmv - effrs = 5.0E-4 ! 500 microns, in meter - if (has_graupel) then - phys_state%effrg = 2.5E-4 ! 250 microns, in meter - phys_state%ncg = 40 - end if - if (has_ice) then - phys_state%effri = 5.0E-5 ! 50 microns, in meter - phys_state%nci = 80 - end if - phys_state%tke = 10.0 !J kg-1 - phys_state%tke2 = 42.0 !J kg-1 - - end subroutine init_data - - logical function compare_data() - - real(kind=kind_phys), parameter :: effrr_expected = 1.0E-3 ! 1000 microns, in meter - real(kind=kind_phys), parameter :: effrl_expected = 5.0E-5 ! 50 microns, in meter - real(kind=kind_phys), parameter :: effri_expected = 7.5E-5 ! 75 microns, in meter - real(kind=kind_phys), parameter :: effrs_expected = 5.3E-4 ! 530 microns, in meter - real(kind=kind_phys), parameter :: scalar_expected = 2.0E3 ! 2 km, in meter - real(kind=kind_phys), parameter :: tke_expected = 10.0 ! 10 J kg-1 - real(kind=kind_phys), parameter :: tolerance = 1.0E-6 ! used as scaling factor for expected value - real(kind=kind_phys), parameter :: sfc_up_sw_expected = 100. ! W/m2 - real(kind=kind_phys), parameter :: sfc_down_sw_expected = 400. ! W/m2 - real(kind=kind_phys), parameter :: sfc_up_lw_expected = 300. ! W/m2 - real(kind=kind_phys), parameter :: sfc_down_lw_expected = 50. ! W/m2 - - compare_data = .true. - - if (maxval(abs(phys_state%effrr - effrr_expected)) > tolerance * effrr_expected) then - write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effrr from expected value exceeds tolerance: ', & - maxval(abs(phys_state%effrr - effrr_expected)), ' > ', tolerance * effrr_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%effrl - effrl_expected)) > tolerance * effrl_expected) then - write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effrl from expected value exceeds tolerance: ', & - maxval(abs(phys_state%effrl - effrl_expected)), ' > ', tolerance * effrl_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%effri - effri_expected)) > tolerance * effri_expected) then - write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effri from expected value exceeds tolerance: ', & - maxval(abs(phys_state%effri - effri_expected)), ' > ', tolerance * effri_expected - compare_data = .false. - end if - - if (maxval(abs(effrs - effrs_expected)) > tolerance * effrs_expected) then - write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of effrs from expected value exceeds tolerance: ', & - maxval(abs(effrs - effrs_expected)), ' > ', tolerance * effrs_expected - compare_data = .false. - end if - - if (abs(phys_state%scalar_var - scalar_expected) > tolerance * scalar_expected) then - write(6, '(a,e16.7,a,e16.7)') & - 'Error: max diff of scalar_var from expected value exceeds tolerance: ', & - abs(phys_state%scalar_var - scalar_expected), ' > ', tolerance * scalar_expected - compare_data = .false. - end if - - if (abs(phys_state%tke - tke_expected) > tolerance * tke_expected) then - write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of tke from expected value exceeds tolerance: ', & - abs(phys_state%tke - tke_expected), ' > ', tolerance * tke_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%fluxsw%sfc_up_sw - sfc_up_sw_expected)) > tolerance * sfc_up_sw_expected) then - write(6, '(a,e16.7,a,e16.7)') & - 'Error: max diff of sfc_up_sw from expected value exceeds tolerance: ', & - abs(phys_state%fluxsw%sfc_up_sw - sfc_up_sw_expected), ' > ', tolerance * sfc_up_sw_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%fluxsw%sfc_down_sw - sfc_down_sw_expected)) > tolerance * sfc_down_sw_expected) then - write(6, '(a,e16.7,a,e16.7)') & - 'Error: max diff of sfc_down_sw from expected value exceeds tolerance: ', & - abs(phys_state%fluxsw%sfc_down_sw - sfc_down_sw_expected), ' > ', tolerance * sfc_down_sw_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%fluxlw%sfc_up_lw - sfc_up_lw_expected)) > tolerance * sfc_up_lw_expected) then - write(6, '(a,e16.7,a,e16.7)') & - 'Error: max diff of sfc_up_lw from expected value exceeds tolerance: ', & - abs(phys_state%fluxlw%sfc_up_lw - sfc_up_lw_expected), ' > ', tolerance * sfc_up_lw_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%fluxlw%sfc_down_lw - sfc_down_lw_expected)) > tolerance * sfc_down_lw_expected) then - write(6, '(a,e16.7,a,e16.7)') & - 'Error: max diff of sfc_down_lw from expected value exceeds tolerance: ', & - abs(phys_state%fluxlw%sfc_down_lw - sfc_down_lw_expected), ' > ', tolerance * sfc_down_lw_expected - compare_data = .false. - end if - - end function compare_data - -end module test_host_mod diff --git a/test/nested_suite_test/test_host_mod.meta b/test/nested_suite_test/test_host_mod.meta deleted file mode 100644 index 51a2f5c3..00000000 --- a/test/nested_suite_test/test_host_mod.meta +++ /dev/null @@ -1,42 +0,0 @@ -[ccpp-table-properties] - name = test_host_mod - type = module -[ccpp-arg-table] - name = test_host_mod - type = module -[ ncols] - standard_name = horizontal_dimension - units = count - type = integer - protected = True - dimensions = () -[ pver ] - standard_name = vertical_layer_dimension - units = count - type = integer - protected = True - dimensions = () -[ phys_state ] - standard_name = physics_state_derived_type - long_name = Physics State DDT - type = physics_state - dimensions = () -[effrs] - standard_name = effective_radius_of_stratiform_cloud_snow_particle - long_name = effective radius of cloud snow particle in meter - units = m - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys -[has_ice] - standard_name = flag_indicating_cloud_microphysics_has_ice - long_name = flag indicating that the cloud microphysics produces ice - units = flag - dimensions = () - type = logical -[has_graupel] - standard_name = flag_indicating_cloud_microphysics_has_graupel - long_name = flag indicating that the cloud microphysics produces graupel - units = flag - dimensions = () - type = logical diff --git a/test/nested_suite_test/test_nested_suite_integration.F90 b/test/nested_suite_test/test_nested_suite_integration.F90 deleted file mode 100644 index 5e9c3009..00000000 --- a/test/nested_suite_test/test_nested_suite_integration.F90 +++ /dev/null @@ -1,91 +0,0 @@ -program test_nested_suite_integration - use test_prog, only: test_host, & - suite_info, & - cm, & - cs - - implicit none - - character(len=cs), target :: test_parts1(3) = (/ & - 'radiation1 ', & - 'rad_lw_group ', & - 'rad_sw_group '/) - - character(len=cm), target :: test_invars1(18) = (/ & - 'effective_radius_of_stratiform_cloud_rain_particle ', & - 'effective_radius_of_stratiform_cloud_liquid_water_particle', & - 'effective_radius_of_stratiform_cloud_snow_particle ', & - 'effective_radius_of_stratiform_cloud_graupel ', & - 'cloud_graupel_number_concentration ', & - 'scalar_variable_for_testing ', & - 'turbulent_kinetic_energy ', & - 'turbulent_kinetic_energy2 ', & - 'scalar_variable_for_testing_a ', & - 'scalar_variable_for_testing_b ', & - 'scalar_variable_for_testing_c ', & - 'scheme_order_in_suite ', & - 'num_subcycles_for_effr ', & - 'flag_indicating_cloud_microphysics_has_graupel ', & - 'flag_indicating_cloud_microphysics_has_ice ', & - 'surface_downwelling_shortwave_radiation_flux ', & - 'surface_upwelling_shortwave_radiation_flux ', & - 'longwave_radiation_fluxes '/) - - character(len=cm), target :: test_outvars1(14) = (/ & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'effective_radius_of_stratiform_cloud_ice_particle ', & - 'effective_radius_of_stratiform_cloud_liquid_water_particle', & - 'effective_radius_of_stratiform_cloud_rain_particle ', & - 'effective_radius_of_stratiform_cloud_snow_particle ', & - 'cloud_ice_number_concentration ', & - 'scalar_variable_for_testing ', & - 'scheme_order_in_suite ', & - 'surface_downwelling_shortwave_radiation_flux ', & - 'surface_upwelling_shortwave_radiation_flux ', & - 'turbulent_kinetic_energy ', & - 'turbulent_kinetic_energy2 ', & - 'longwave_radiation_fluxes '/) - - character(len=cm), target :: test_reqvars1(22) = (/ & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'effective_radius_of_stratiform_cloud_rain_particle ', & - 'effective_radius_of_stratiform_cloud_ice_particle ', & - 'effective_radius_of_stratiform_cloud_liquid_water_particle', & - 'effective_radius_of_stratiform_cloud_snow_particle ', & - 'effective_radius_of_stratiform_cloud_graupel ', & - 'cloud_graupel_number_concentration ', & - 'cloud_ice_number_concentration ', & - 'scalar_variable_for_testing ', & - 'turbulent_kinetic_energy ', & - 'turbulent_kinetic_energy2 ', & - 'scalar_variable_for_testing_a ', & - 'scalar_variable_for_testing_b ', & - 'scalar_variable_for_testing_c ', & - 'scheme_order_in_suite ', & - 'num_subcycles_for_effr ', & - 'flag_indicating_cloud_microphysics_has_graupel ', & - 'flag_indicating_cloud_microphysics_has_ice ', & - 'surface_downwelling_shortwave_radiation_flux ', & - 'surface_upwelling_shortwave_radiation_flux ', & - 'longwave_radiation_fluxes '/) - - type(suite_info) :: test_suites(1) - logical :: run_okay - - ! Setup expected test suite info - test_suites(1)%suite_name = 'main_suite' - test_suites(1)%suite_parts => test_parts1 - test_suites(1)%suite_input_vars => test_invars1 - test_suites(1)%suite_output_vars => test_outvars1 - test_suites(1)%suite_required_vars => test_reqvars1 - - call test_host(run_okay, test_suites) - - if (run_okay) then - stop 0 - else - stop -1 - end if -end program test_nested_suite_integration diff --git a/test/pylint_test.sh b/test/pylint_test.sh deleted file mode 100755 index a8bf3f90..00000000 --- a/test/pylint_test.sh +++ /dev/null @@ -1,28 +0,0 @@ -#! /bin/bash - -# Script to run pylint tests on CCPP Framework python scripts - -# Add CCPP Framework paths to PYTHONPATH so pylint can find them -SCRIPTDIR="$( cd $( dirname ${0} ); pwd -P )" -SPINROOT="$( dirname ${SCRIPTDIR} )" -CCPPDIR="${SPINROOT}/scripts" -export PYTHONPATH="${CCPPDIR}:$PYTHONPATH" - -pylintcmd="pylint --rcfile=${SCRIPTDIR}/.pylintrc" - -# Test top-level scripts -scripts="${CCPPDIR}/ccpp_capgen.py" -scripts="${scripts} ${CCPPDIR}/ccpp_suite.py" -scripts="${scripts} ${CCPPDIR}/ddt_library.py" -scripts="${scripts} ${CCPPDIR}/host_cap.py" -scripts="${scripts} ${CCPPDIR}/host_model.py" -scripts="${scripts} ${CCPPDIR}/metadata_table.py" -scripts="${scripts} ${CCPPDIR}/metavar.py" -scripts="${scripts} ${CCPPDIR}/state_machine.py" -${pylintcmd} ${scripts} -# Test the fortran_tools module -${pylintcmd} ${CCPPDIR}/fortran_tools -# Test the parse_tools module -${pylintcmd} ${CCPPDIR}/parse_tools -# Test the fortran to metadata converter tool -${pylintcmd} ${CCPPDIR}/ccpp_fortran_to_metadata.py diff --git a/test/test_fortran_to_metadata.sh b/test/test_fortran_to_metadata.sh deleted file mode 100755 index adedaac6..00000000 --- a/test/test_fortran_to_metadata.sh +++ /dev/null @@ -1,37 +0,0 @@ -#! /bin/bash - -## Relevant directories and file paths -test_dir="$(cd $(dirname ${0}); pwd -P)" -script_dir="$(dirname ${test_dir})/scripts" -sample_files_dir="${test_dir}/unit_tests/sample_files" -f2m_script="${script_dir}/ccpp_fortran_to_metadata.py" -filename="test_fortran_to_metadata" -test_input="${sample_files_dir}/${filename}.F90" -tmp_dir="${test_dir}/unit_tests/tmp" -sample_meta="${sample_files_dir}/check_fortran_to_metadata.meta" - -# Run the script -opts="--ddt-names serling_t" -${f2m_script} --output-root "${tmp_dir}" ${opts} "${test_input}" -res=$? - -retval=0 -if [ ${res} -ne 0 ]; then - echo "FAIL: ccpp_fortran_to_metadata.py exited with error ${res}" - retval=${res} -elif [ ! -f "${tmp_dir}/${filename}.meta" ]; then - echo "FAIL: metadata file, '${tmp_dir}/${filename}.meta', not created" - retval=1 -else - cmp --quiet "${sample_meta}" "${tmp_dir}/${filename}.meta" - res=$? - if [ ${res} -ne 0 ]; then - echo "FAIL: Comparison with correct metadata file failed" - retval=${res} - else - echo "PASS" - # Cleanup - rm "${tmp_dir}/${filename}.meta" - fi -fi -exit ${retval} diff --git a/test/test_offline_metadata_checker.sh b/test/test_offline_metadata_checker.sh deleted file mode 100755 index 90befbec..00000000 --- a/test/test_offline_metadata_checker.sh +++ /dev/null @@ -1,35 +0,0 @@ -#! /bin/bash - -## Relevant directories and file paths -root_dir="$(cd $(dirname ${0}); pwd -P)" -script_dir="$(dirname ${root_dir})/scripts/fortran_tools" -test_dir="$(dirname ${root_dir})/test/advection_test" -offline_script="${script_dir}/offline_check_fortran_vs_metadata.py" -relative_path="capgen_test" - -# Run the script -${offline_script} --directory ${test_dir} -res=$? - -retval=0 -if [ ${res} -ne 0 ]; then - echo "FAIL: offline_check_fortran_vs_metadata.py exited with error ${res} while checking ${test_dir}" - retval=${res} - exit ${retval} -else - echo "PASS" -fi - -# Run the script again with a relative path -cd ${root_dir} -${offline_script} --directory ${relative_path} -res=$? - -retval=0 -if [ ${res} -ne 0 ]; then - echo "FAIL: offline_check_fortran_vs_metadata.py exited with error ${res} while checking ${relative_path}" - retval=${res} -else - echo "PASS" -fi -exit ${retval} diff --git a/test/test_stub.py b/test/test_stub.py deleted file mode 100644 index 664f3277..00000000 --- a/test/test_stub.py +++ /dev/null @@ -1,162 +0,0 @@ -from ccpp_datafile import datatable_report, DatatableReport -import subprocess - - -class BaseTests: - - - class TestHostDataTables: - _SEP = "," - - def test_host_files(self): - test_str = datatable_report(self.database, DatatableReport("host_files"), self._SEP) - self.assertSetEqual(set(self.host_files), set(test_str.split(self._SEP))) - - def test_suite_files(self): - test_str = datatable_report(self.database, DatatableReport("suite_files"), self._SEP) - self.assertSetEqual(set(self.suite_files), set(test_str.split(self._SEP))) - - def test_utility_files(self): - test_str = datatable_report(self.database, DatatableReport("utility_files"), self._SEP) - self.assertSetEqual(set(self.utility_files), set(test_str.split(self._SEP))) - - def test_ccpp_files(self): - test_str = datatable_report(self.database, DatatableReport("ccpp_files"), self._SEP) - self.assertSetEqual(set(self.ccpp_files), set(test_str.split(self._SEP))) - - def test_process_list(self): - test_str = datatable_report(self.database, DatatableReport("process_list"), self._SEP) - self.assertSetEqual(set(self.process_list), set(test_str.split(self._SEP))) - - def test_module_list(self): - test_str = datatable_report(self.database, DatatableReport("module_list"), self._SEP) - self.assertSetEqual(set(self.module_list), set(test_str.split(self._SEP))) - - def test_dependencies_list(self): - test_str = datatable_report(self.database, DatatableReport("dependencies"), self._SEP) - self.assertSetEqual(set(self.dependencies), set(test_str.split(self._SEP))) - - def test_suite_list(self): - test_str = datatable_report(self.database, DatatableReport("suite_list"), self._SEP) - self.assertSetEqual(set(self.suite_list), set(test_str.split(self._SEP))) - - - class TestHostCommandLineDataFiles: - _SEP = "," - - def test_host_files(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--host-files"], - capture_output=True, - text=True) - self.assertEqual(self._SEP.join(self.host_files), completedProcess.stdout.strip()) - - def test_suite_files(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--suite-files"], - capture_output=True, - text=True) - self.assertEqual(self._SEP.join(self.suite_files), completedProcess.stdout.strip()) - - def test_utility_files(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--utility-files"], - capture_output=True, - text=True) - self.assertEqual(self._SEP.join(self.utility_files), completedProcess.stdout.strip()) - - def test_ccpp_files(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--ccpp-files"], - capture_output=True, - text=True) - self.assertEqual(self._SEP.join(self.ccpp_files), completedProcess.stdout.strip()) - - def test_process_list(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--process-list"], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.process_list), actualOutput) - - def test_module_list(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--module-list"], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.module_list), actualOutput) - - def test_dependencies(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--dependencies"], - capture_output=True, - text=True) - self.assertEqual(set(self.dependencies), set(completedProcess.stdout.strip().split(self._SEP))) - - def test_suite_list(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--suite-list"], - capture_output=True, - text=True) - self.assertEqual(self._SEP.join(self.suite_list), completedProcess.stdout.strip()) - - - class TestSuite: - _SEP = "," - - def test_required_variables(self): - test_str = datatable_report(self.database, DatatableReport("required_variables", value=self.suite_name), self._SEP) - self.assertSetEqual(set(self.required_vars), set(test_str.split(self._SEP))) - - def test_input_variables(self): - test_str = datatable_report(self.database, DatatableReport("input_variables", value=self.suite_name), self._SEP) - self.assertSetEqual(set(self.input_vars), set(test_str.split(self._SEP))) - - def test_output_variables(self): - test_str = datatable_report(self.database, DatatableReport("output_variables", value=self.suite_name), self._SEP) - self.assertSetEqual(set(self.output_vars), set(test_str.split(self._SEP))) - - - class TestSuiteCommandLine: - _SEP = "," - - def test_required_variables(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--required-variables", self.suite_name], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.required_vars), actualOutput) - - def test_input_variables(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--input-variables", self.suite_name], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.input_vars), actualOutput) - - def test_output_variables(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--output-variables", self.suite_name], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.output_vars), actualOutput) - - - class TestSuiteExcludeProtected(TestSuite): - def test_required_variables_excluding_protected(self): - test_str = datatable_report(self.database, DatatableReport("required_variables", value="temp_suite"), self._SEP, exclude_protected=True) - self.assertSetEqual(set(self.required_vars_excluding_protected), set(test_str.split(self._SEP))) - - def test_input_variables_excluding_protected(self): - test_str = datatable_report(self.database, DatatableReport("input_variables", value="temp_suite"), self._SEP, exclude_protected=True) - self.assertSetEqual(set(self.input_vars_excluding_protected), set(test_str.split(self._SEP))) - - - class TestSuiteExcludeProtectedCommandLine(TestSuiteCommandLine): - def test_required_variables_excluding_protected(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--exclude-protected", "--required-variables", self.suite_name], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.required_vars_excluding_protected), actualOutput) - - def test_input_variables_excluding_protected(self): - completedProcess = subprocess.run([self.datafile_script, self.database, "--exclude-protected", "--input-variables", self.suite_name], - capture_output=True, - text=True) - actualOutput = {s.strip() for s in completedProcess.stdout.split(self._SEP)} - self.assertSetEqual(set(self.input_vars_excluding_protected), actualOutput) diff --git a/test/unit_tests/README.md b/test/unit_tests/README.md deleted file mode 100644 index 5a2353ee..00000000 --- a/test/unit_tests/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# How to run the python unit-tests - -## Quick start: -To run all unit-tests: -```bash -$ export PYTHONPATH=/scripts:/scripts/parse_tools -$ pytest -v test/ -``` - -To run a specific unit tests: -```bash -$ cd /test/unit_tests -$ python test_metadata_table.py -``` -For more verbose output: -```bash -$ python test_metadata_table.py -v -``` -If you have `coverage` installed, to get test coverage: -```bash -$ coverage run test_metadata_table.py -$ coverage report -m -``` -To check source code quality with pylint: -```bash -$ cd -$ env PYTHONPATH=scripts:${PYTHONPATH} pylint --rcfile ./test/.pylintrc ./test/unit_tests/test_metadata_table.py -$ env PYTHONPATH=scripts:${PYTHONPATH} pylint --rcfile ./test/.pylintrc ./test/unit_tests/test_metadata_scheme_file.py -``` diff --git a/test/unit_tests/sample_files/bad_kind_spec_table_properties.meta b/test/unit_tests/sample_files/bad_kind_spec_table_properties.meta deleted file mode 100644 index d51a30d4..00000000 --- a/test/unit_tests/sample_files/bad_kind_spec_table_properties.meta +++ /dev/null @@ -1,5 +0,0 @@ -[ccpp-table-properties] - name = bad_scheme - type = scheme - dependencies = - kind_spec = temp_r8 diff --git a/test/unit_tests/sample_files/check_fortran_to_metadata.meta b/test/unit_tests/sample_files/check_fortran_to_metadata.meta deleted file mode 100644 index 7d84546b..00000000 --- a/test/unit_tests/sample_files/check_fortran_to_metadata.meta +++ /dev/null @@ -1,31 +0,0 @@ -[ccpp-table-properties] - name = do_stuff - type = scheme - -[ccpp-arg-table] - name = do_stuff_run - type = scheme -[ const_props ] - standard_name = enter_standard_name_1 - units = enter_units - type = ccpp_constituent_prop_ptr_t - dimensions = (enter_standard_name_5:enter_standard_name_6) - intent = in -[ twilight_zone ] - standard_name = enter_standard_name_2 - units = enter_units - type = serling_t - dimensions = () - intent = inout -[ errmsg ] - standard_name = enter_standard_name_3 - units = enter_units - type = character | kind = len=512 - dimensions = () - intent = out -[ errflg ] - standard_name = enter_standard_name_4 - units = enter_units - type = integer - dimensions = () - intent = out diff --git a/test/unit_tests/sample_files/double_header.meta b/test/unit_tests/sample_files/double_header.meta deleted file mode 100644 index 27051c17..00000000 --- a/test/unit_tests/sample_files/double_header.meta +++ /dev/null @@ -1,12 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/double_table_properties.meta b/test/unit_tests/sample_files/double_table_properties.meta deleted file mode 100644 index 28637b36..00000000 --- a/test/unit_tests/sample_files/double_table_properties.meta +++ /dev/null @@ -1,13 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies = -[ccpp-table-properties] - name = test_host - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/duplicate_kind_spec_table_properties.meta b/test/unit_tests/sample_files/duplicate_kind_spec_table_properties.meta deleted file mode 100644 index 12c1cfc5..00000000 --- a/test/unit_tests/sample_files/duplicate_kind_spec_table_properties.meta +++ /dev/null @@ -1,6 +0,0 @@ -[ccpp-table-properties] - name = scheme_scheme - type = scheme - dependencies = fmodule.F90 - kind_spec = fmodule:kind_temp=>temp_r8 - kind_spec = fmodule:kind_temp diff --git a/test/unit_tests/sample_files/fortran_files/array_parsing_test.F90 b/test/unit_tests/sample_files/fortran_files/array_parsing_test.F90 deleted file mode 100644 index 493ab43d..00000000 --- a/test/unit_tests/sample_files/fortran_files/array_parsing_test.F90 +++ /dev/null @@ -1,33 +0,0 @@ -!Test array specifications -! - -MODULE array_spec_test - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: array_spec_test_run - -CONTAINS - - !> \section arg_table_array_spec_test_run Argument Table - !! \htmlinclude arg_table_array_spec_test_run.html - !! - SUBROUTINE array_spec_test_run(ncol, lev, good_arr1, good_arr2, good_arr3, & - good_arr4, good_arr5, bad_arr1, bad_arr2, bad_arr3) - - integer, intent(in) :: ncol, lev - real(kind_phys), intent(in) :: good_arr1(ncol,lev) - real(kind_phys), intent(in) :: good_arr2(:,:) - real(kind_phys), intent(in), dimension(ncol,lev) :: good_arr3 - real(kind_phys), intent(in), dimension(:,:) :: good_arr4 - real(kind_phys), intent(in) :: good_arr5(ncol,:) - real(kind_phys), intent(in) :: bad_arr1(:,;) - real(kind_phys), intent(in), dimension(;,:) :: bad_arr2 - real(kind_phys), intent(in), dimension(:,;) :: bad_arr3 - - END SUBROUTINE array_spec_test_run - -END MODULE array_spec_test diff --git a/test/unit_tests/sample_files/fortran_files/comments_test.F90 b/test/unit_tests/sample_files/fortran_files/comments_test.F90 deleted file mode 100644 index e74410e7..00000000 --- a/test/unit_tests/sample_files/fortran_files/comments_test.F90 +++ /dev/null @@ -1,34 +0,0 @@ -! -! This work (Common Community Physics Package Framework), identified by -! NOAA, NCAR, CU/CIRES, is free of known copyright restrictions and is -! placed in the public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -!> -!! @brief Auto-generated Test of comment writing for FortranWriter -!! -! -module comments_test - -! codee format off -! We can write comments in the module header -! codee format on - ! We can write indented comments in the header - integer :: foo ! Comment at end of line works - integer :: bar ! - ! xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - ! - integer :: baz ! - ! yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy - ! yyyyyyyyyyyyyy - -contains - ! We can write comments in the module body - -end module comments_test diff --git a/test/unit_tests/sample_files/fortran_files/linebreak_test.F90 b/test/unit_tests/sample_files/fortran_files/linebreak_test.F90 deleted file mode 100644 index 2b8f0b74..00000000 --- a/test/unit_tests/sample_files/fortran_files/linebreak_test.F90 +++ /dev/null @@ -1,52 +0,0 @@ -! -! This work (Common Community Physics Package Framework), identified by -! NOAA, NCAR, CU/CIRES, is free of known copyright restrictions and is -! placed in the public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -!> -!! @brief Auto-generated Test of line breaking for FortranWriter -!! -! -module linebreak_test - - character(len=7) :: data(100) = (/ 'name000', 'name001', 'name002', 'name003', 'name004', & - 'name005', 'name006', 'name007', 'name008', 'name009', 'name010', 'name011', 'name012', & - 'name013', 'name014', 'name015', 'name016', 'name017', 'name018', 'name019', 'name020', & - 'name021', 'name022', 'name023', 'name024', 'name025', 'name026', 'name027', 'name028', & - 'name029', 'name030', 'name031', 'name032', 'name033', 'name034', 'name035', 'name036', & - 'name037', 'name038', 'name039', 'name040', 'name041', 'name042', 'name043', 'name044', & - 'name045', 'name046', 'name047', 'name048', 'name049', 'name050', 'name051', 'name052', & - 'name053', 'name054', 'name055', 'name056', 'name057', 'name058', 'name059', 'name060', & - 'name061', 'name062', 'name063', 'name064', 'name065', 'name066', 'name067', 'name068', & - 'name069', 'name070', 'name071', 'name072', 'name073', 'name074', 'name075', 'name076', & - 'name077', 'name078', 'name079', 'name080', 'name081', 'name082', 'name083', 'name084', & - 'name085', 'name086', 'name087', 'name088', 'name089', 'name090', 'name091', 'name092', & - 'name093', 'name094', 'name095', 'name096', 'name097', 'name098', 'name099' /) - -contains - - subroutine foo(ozone_constituents, aerosol_constituents, volcaero_constituents, & - other_constituents) - integer, intent(in) :: ozone_constituents(:) - integer, intent(in) :: aerosol_constituents(:) - integer, intent(in) :: volcaero_constituents(:) - integer, intent(in) :: other_constituents(:) - real, allocatable :: tracer_data_test_dynamic_constituents(:) -! codee format off - allocate(tracer_data_test_dynamic_constituents(0+size(ozone_constituents)+size( & - aerosol_constituents)+size(volcaero_constituents)+size(other_constituents))) - - write(6, '(a)') & - 'Cannot read columns_on_task from file'// & - ', columns_on_task has no horizontal dimension; columns_on_task is a protected variable' -! codee format on - end subroutine foo - -end module linebreak_test diff --git a/test/unit_tests/sample_files/fortran_files/long_string_test.F90 b/test/unit_tests/sample_files/fortran_files/long_string_test.F90 deleted file mode 100644 index 17fd17fd..00000000 --- a/test/unit_tests/sample_files/fortran_files/long_string_test.F90 +++ /dev/null @@ -1,97 +0,0 @@ -! -! This work (Common Community Physics Package Framework), identified by -! NOAA, NCAR, CU/CIRES, is free of known copyright restrictions and is -! placed in the public domain. -! -! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -! THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -! IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -! CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -!> -!! @brief Auto-generated Test of long string breaking for FortranWriter -!! -! -module long_string_test - - character(len=100) :: foo100 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789' - character(len=101) :: foo101 = & - '01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890' - character(len=102) :: foo102 = & - '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901' - character(len=103) :: foo103 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012' - character(len=104) :: foo104 = & - '01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123' - character(len=105) :: foo105 = & - '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234' - character(len=106) :: foo106 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345' - character(len=107) :: foo107 = & - '01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456' - character(len=108) :: foo108 = & - '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567' - character(len=109) :: foo109 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678' - character(len=110) :: foo110 = & - '01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789' - character(len=111) :: foo111 = & - '012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890' - character(len=112) :: foo112 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901' - character(len=113) :: foo113 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2' - character(len=114) :: foo114 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23' - character(len=115) :: foo115 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &234' - character(len=116) :: foo116 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2345' - character(len=117) :: foo117 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23456' - character(len=118) :: foo118 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &234567' - character(len=119) :: foo119 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2345678' - character(len=120) :: foo120 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23456789' - character(len=121) :: foo121 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &234567890' - character(len=122) :: foo122 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2345678901' - character(len=123) :: foo123 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23456789012' - character(len=124) :: foo124 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &234567890123' - character(len=125) :: foo125 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2345678901234' - character(len=126) :: foo126 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23456789012345' - character(len=127) :: foo127 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &234567890123456' - character(len=128) :: foo128 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &2345678901234567' - character(len=129) :: foo129 = & - '0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901& - &23456789012345678' - -end module long_string_test diff --git a/test/unit_tests/sample_files/good_kind_spec_table_properties.meta b/test/unit_tests/sample_files/good_kind_spec_table_properties.meta deleted file mode 100644 index 67aeef86..00000000 --- a/test/unit_tests/sample_files/good_kind_spec_table_properties.meta +++ /dev/null @@ -1,6 +0,0 @@ -[ccpp-table-properties] - name = good_scheme - type = scheme - dependencies = fmodule.F90 - kind_spec = fmodule:kind_temp=>temp_r8 - kind_spec = fmodule:temp_i8 diff --git a/test/unit_tests/sample_files/missing_table_properties.meta b/test/unit_tests/sample_files/missing_table_properties.meta deleted file mode 100644 index 0bef09dc..00000000 --- a/test/unit_tests/sample_files/missing_table_properties.meta +++ /dev/null @@ -1,3 +0,0 @@ -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/test_bad_1st_arg_table_header.meta b/test/unit_tests/sample_files/test_bad_1st_arg_table_header.meta deleted file mode 100644 index ac6468ac..00000000 --- a/test/unit_tests/sample_files/test_bad_1st_arg_table_header.meta +++ /dev/null @@ -1,23 +0,0 @@ -[ccpp-table-properties] - name = vmr_type - type = ddt - dependencies = - -######################################################################## -[ccpp-farg-table] - name = vmr_type - type = ddt -[ nvmr ] - standard_name = number_of_chemical_species - units = count - dimensions = () - type = integer -[ccpp-arg-table] - name = make_ddt_run - type = scheme -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in diff --git a/test/unit_tests/sample_files/test_bad_2nd_arg_table_header.meta b/test/unit_tests/sample_files/test_bad_2nd_arg_table_header.meta deleted file mode 100644 index 48381744..00000000 --- a/test/unit_tests/sample_files/test_bad_2nd_arg_table_header.meta +++ /dev/null @@ -1,23 +0,0 @@ -[ccpp-table-properties] - name = vmr_type - type = ddt - dependencies = - -######################################################################## -[ccpp-arg-table] - name = vmr_type - type = ddt -[ nvmr ] - standard_name = number_of_chemical_species - units = count - dimensions = () - type = integer -[ccpp-farg-table] - name = make_ddt_run - type = scheme -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in diff --git a/test/unit_tests/sample_files/test_bad_dimension.meta b/test/unit_tests/sample_files/test_bad_dimension.meta deleted file mode 100644 index 9a20b1f8..00000000 --- a/test/unit_tests/sample_files/test_bad_dimension.meta +++ /dev/null @@ -1,15 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = banana - protected = True diff --git a/test/unit_tests/sample_files/test_bad_line_split.meta b/test/unit_tests/sample_files/test_bad_line_split.meta deleted file mode 100644 index 3ace2ccb..00000000 --- a/test/unit_tests/sample_files/test_bad_line_split.meta +++ /dev/null @@ -1,16 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme -[ temp_calc ] - standard_name = potential_temperature_at_previous_timestep - units = K | - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = out diff --git a/test/unit_tests/sample_files/test_bad_table_key.meta b/test/unit_tests/sample_files/test_bad_table_key.meta deleted file mode 100644 index 8d0bbc7f..00000000 --- a/test/unit_tests/sample_files/test_bad_table_key.meta +++ /dev/null @@ -1,10 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host - banana = something diff --git a/test/unit_tests/sample_files/test_bad_table_type.meta b/test/unit_tests/sample_files/test_bad_table_type.meta deleted file mode 100644 index b6b9449d..00000000 --- a/test/unit_tests/sample_files/test_bad_table_type.meta +++ /dev/null @@ -1,15 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True diff --git a/test/unit_tests/sample_files/test_bad_type_name.meta b/test/unit_tests/sample_files/test_bad_type_name.meta deleted file mode 100644 index de11315d..00000000 --- a/test/unit_tests/sample_files/test_bad_type_name.meta +++ /dev/null @@ -1,9 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = banana diff --git a/test/unit_tests/sample_files/test_bad_var_property_name.meta b/test/unit_tests/sample_files/test_bad_var_property_name.meta deleted file mode 100644 index a2009233..00000000 --- a/test/unit_tests/sample_files/test_bad_var_property_name.meta +++ /dev/null @@ -1,35 +0,0 @@ -[ccpp-table-properties] - name = vmr_type - type = ddt - dependencies = - -######################################################################## -[ccpp-arg-table] - name = vmr_type - type = ddt -[ nvmr ] - standard_name = number_of_chemical_species - units = count - dimensions = () - type = integer -[ vmr_array ] - standard_name = array_of_volume_mixing_ratios - units = ppmv - dimensions = (horizontal_dimension, number_of_chemical_species) - type = real - kind = kind_phys - -[ccpp-table-properties] - name = make_ddt - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = make_ddt_run - type = scheme -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - None = None - intent = inout diff --git a/test/unit_tests/sample_files/test_dependencies_path.meta b/test/unit_tests/sample_files/test_dependencies_path.meta deleted file mode 100644 index 391ad93d..00000000 --- a/test/unit_tests/sample_files/test_dependencies_path.meta +++ /dev/null @@ -1,11 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies_path = ../../ccpp/physics/physics - dependencies = machine.F,physcons.F90,, - dependencies = GFDL_parse_tracers.F90,,rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90 - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/test_duplicate_variable.meta b/test/unit_tests/sample_files/test_duplicate_variable.meta deleted file mode 100644 index efc66b86..00000000 --- a/test/unit_tests/sample_files/test_duplicate_variable.meta +++ /dev/null @@ -1,21 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme -[ temp ] - standard_name = index_of_water_vapor_specific_humidity - units = index - type = integer - intent = in - dimensions = () -[ temp ] - standard_name = index_of_water_vapor_specific_humidity - units = index - type = integer - intent = in - dimensions = () diff --git a/test/unit_tests/sample_files/test_fortran_to_metadata.F90 b/test/unit_tests/sample_files/test_fortran_to_metadata.F90 deleted file mode 100644 index ff4542c4..00000000 --- a/test/unit_tests/sample_files/test_fortran_to_metadata.F90 +++ /dev/null @@ -1,28 +0,0 @@ -module dme_adjust - - use ccpp_kinds, only: kind_phys - - implicit none - -contains -!=============================================================================== -!> \section arg_table_do_stuff_run Argument Table -!! \htmlinclude do_stuff_run.html -!! - subroutine do_stuff_run(const_props, twilight_zone, errmsg, errflg) - ! - ! Arguments - ! - type(ccpp_constituent_prop_ptr_t), intent(in) :: const_props(:) - type(serling_t), intent(inout) :: twilight_zone - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = ' ' - errflg = 0 - twilight_zone('adjust_set') - - end subroutine dme_adjust_run - -end module dme_adjust diff --git a/test/unit_tests/sample_files/test_host.meta b/test/unit_tests/sample_files/test_host.meta deleted file mode 100644 index f618f871..00000000 --- a/test/unit_tests/sample_files/test_host.meta +++ /dev/null @@ -1,34 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True -[ col_end ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - protected = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer diff --git a/test/unit_tests/sample_files/test_invalid_intent.meta b/test/unit_tests/sample_files/test_invalid_intent.meta deleted file mode 100644 index 6f11169e..00000000 --- a/test/unit_tests/sample_files/test_invalid_intent.meta +++ /dev/null @@ -1,23 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = banana diff --git a/test/unit_tests/sample_files/test_invalid_table_properties_type.meta b/test/unit_tests/sample_files/test_invalid_table_properties_type.meta deleted file mode 100644 index 45f43e83..00000000 --- a/test/unit_tests/sample_files/test_invalid_table_properties_type.meta +++ /dev/null @@ -1,9 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = banana - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/test_mismatch_section_table_title.meta b/test/unit_tests/sample_files/test_mismatch_section_table_title.meta deleted file mode 100644 index 240b93c2..00000000 --- a/test/unit_tests/sample_files/test_mismatch_section_table_title.meta +++ /dev/null @@ -1,9 +0,0 @@ -[ccpp-table-properties] - name = banana - type = host - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host - type = host diff --git a/test/unit_tests/sample_files/test_missing_intent.meta b/test/unit_tests/sample_files/test_missing_intent.meta deleted file mode 100644 index 57be3ad6..00000000 --- a/test/unit_tests/sample_files/test_missing_intent.meta +++ /dev/null @@ -1,22 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys diff --git a/test/unit_tests/sample_files/test_missing_table_name.meta b/test/unit_tests/sample_files/test_missing_table_name.meta deleted file mode 100644 index b8deb22c..00000000 --- a/test/unit_tests/sample_files/test_missing_table_name.meta +++ /dev/null @@ -1,14 +0,0 @@ -[ccpp-table-properties] - name = test_missing_table_name - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True diff --git a/test/unit_tests/sample_files/test_missing_table_type.meta b/test/unit_tests/sample_files/test_missing_table_type.meta deleted file mode 100644 index 98b5bd4f..00000000 --- a/test/unit_tests/sample_files/test_missing_table_type.meta +++ /dev/null @@ -1,14 +0,0 @@ -[ccpp-table-properties] - name = test_host - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = test_host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True diff --git a/test/unit_tests/sample_files/test_missing_units.meta b/test/unit_tests/sample_files/test_missing_units.meta deleted file mode 100644 index 1b9546f2..00000000 --- a/test/unit_tests/sample_files/test_missing_units.meta +++ /dev/null @@ -1,16 +0,0 @@ -[ccpp-table-properties] - name = temp_calc_adjust - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = temp_calc_adjust_run - type = scheme -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - dimensions = () - type = real - kind = kind_phys - intent = in diff --git a/test/unit_tests/sample_files/test_multi_ccpp_arg_tables.meta b/test/unit_tests/sample_files/test_multi_ccpp_arg_tables.meta deleted file mode 100644 index 0be34179..00000000 --- a/test/unit_tests/sample_files/test_multi_ccpp_arg_tables.meta +++ /dev/null @@ -1,115 +0,0 @@ -[ccpp-table-properties] - name = vmr_type - type = ddt - dependencies = - -[ccpp-arg-table] - name = vmr_type - type = ddt -[ nvmr ] - standard_name = number_of_chemical_species - units = count - dimensions = () - type = integer -[ vmr_array ] - standard_name = array_of_volume_mixing_ratios - units = ppmv - dimensions = (horizontal_dimension, number_of_chemical_species) - type = real - kind = kind_phys - -######################################################################## -[ccpp-table-properties] - name = make_ddt - type = scheme - dependencies = - -[ccpp-arg-table] - name = make_ddt_run - type = scheme -[ nbox ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ O3 ] - standard_name = ozone - units = ppmv - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ HNO3 ] - standard_name = nitric_acid - units = ppmv - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = vmr_type - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none | dimensions = () | type = character | kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out - -[ccpp-arg-table] - name = make_ddt_init - type = scheme -[ nbox ] - standard_name = horizontal_dimension - type = integer - units = count - dimensions = () - intent = in -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = vmr_type - intent = out -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out - -[ccpp-arg-table] - name = make_ddt_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_files/test_unknown_ddt_type.meta b/test/unit_tests/sample_files/test_unknown_ddt_type.meta deleted file mode 100644 index e41eeefb..00000000 --- a/test/unit_tests/sample_files/test_unknown_ddt_type.meta +++ /dev/null @@ -1,14 +0,0 @@ -[ccpp-table-properties] - name = make_ddt - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = make_ddt_run - type = scheme -[ vmr ] - standard_name = volume_mixing_ratio_ddt - dimensions = () - type = banana - intent = inout diff --git a/test/unit_tests/sample_host_files/data1_mod.F90 b/test/unit_tests/sample_host_files/data1_mod.F90 deleted file mode 100644 index b85db315..00000000 --- a/test/unit_tests/sample_host_files/data1_mod.F90 +++ /dev/null @@ -1,11 +0,0 @@ -module data1_mod - - use ccpp_kinds, only: kind_phys - - !> \section arg_table_data1_mod Argument Table - !! \htmlinclude arg_table_data1_mod.html - real(kind_phys) :: ps1 - real(kind_phys), allocatable :: xbox(:,:) - real(kind_phys), allocatable :: switch(:,:) - -end module data1_mod diff --git a/test/unit_tests/sample_host_files/data1_mod.meta b/test/unit_tests/sample_host_files/data1_mod.meta deleted file mode 100644 index 37e2de96..00000000 --- a/test/unit_tests/sample_host_files/data1_mod.meta +++ /dev/null @@ -1,25 +0,0 @@ -[ccpp-table-properties] - name = data1_mod - type = module -[ccpp-arg-table] - name = data1_mod - type = module -[ ps1 ] - standard_name = play_station - state_variable = true - type = real | kind = kind_phys - units = Pa - dimensions = () -[ xbox ] - standard_name = xbox - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) -[ switch ] - standard_name = nintendo_switch - long_name = Incompatible junk - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) diff --git a/test/unit_tests/sample_host_files/ddt1.F90 b/test/unit_tests/sample_host_files/ddt1.F90 deleted file mode 100644 index 71b22b4f..00000000 --- a/test/unit_tests/sample_host_files/ddt1.F90 +++ /dev/null @@ -1,17 +0,0 @@ -module ddt1 - - use ccpp_kinds, only: kind_phys - - private - implicit none - - !! \section arg_table_ddt1_t - !! \htmlinclude ddt1_t.html - !! - type, public :: ddt1_t - integer, public :: num_vars = 0 - real(kind_phys), allocatable :: vars(:,:,:) - - end type ddt1_t - -end module ddt1 diff --git a/test/unit_tests/sample_host_files/ddt1.meta b/test/unit_tests/sample_host_files/ddt1.meta deleted file mode 100644 index e1a0f1ac..00000000 --- a/test/unit_tests/sample_host_files/ddt1.meta +++ /dev/null @@ -1,20 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ddt1_t - type = ddt - -[ccpp-arg-table] - name = ddt1_t - type = ddt -[ num_vars ] - standard_name = ddt_var_array_dimension - long_name = Number of vars managed by ddt1 - units = count - dimensions = () - type = integer -[ vars ] - standard_name = vars_array - long_name = Array of vars managed by ddt1 - units = none - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys diff --git a/test/unit_tests/sample_host_files/ddt1_plus.F90 b/test/unit_tests/sample_host_files/ddt1_plus.F90 deleted file mode 100644 index d1806932..00000000 --- a/test/unit_tests/sample_host_files/ddt1_plus.F90 +++ /dev/null @@ -1,33 +0,0 @@ -module ddt1_plus - - use ccpp_kinds, only: kind_phys - - private - implicit none - - !! - type, public :: ddt1_t - real, pointer :: undocumented_array(:) => NULL() - contains - procedure :: this_is_a_documented_object - end type ddt1_t - - !! \section arg_table_ddt2_t - !! \htmlinclude ddt2_t.html - !! - type, public :: ddt2_t - integer, public :: num_vars = 0 - real(kind_phys), allocatable :: vars(:,:,:) - - end type ddt2_t - -CONTAINS - - logical function this_is_a_documented_object(this) - class(ddt1_t) :: intent(in) :: this - - this_is_a_documented_object = .false. - - end function this_is_a_documented_object - -end module ddt1_plus diff --git a/test/unit_tests/sample_host_files/ddt1_plus.meta b/test/unit_tests/sample_host_files/ddt1_plus.meta deleted file mode 100644 index ca3a92ab..00000000 --- a/test/unit_tests/sample_host_files/ddt1_plus.meta +++ /dev/null @@ -1,20 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ddt2_t - type = ddt - -[ccpp-arg-table] - name = ddt2_t - type = ddt -[ num_vars ] - standard_name = ddt_var_array_dimension - long_name = Number of vars managed by ddt2 - units = count - dimensions = () - type = integer -[ vars ] - standard_name = vars_array - long_name = Array of vars managed by ddt2 - units = none - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys diff --git a/test/unit_tests/sample_host_files/ddt2.F90 b/test/unit_tests/sample_host_files/ddt2.F90 deleted file mode 100644 index 22d5af0e..00000000 --- a/test/unit_tests/sample_host_files/ddt2.F90 +++ /dev/null @@ -1,24 +0,0 @@ -module ddt2 - - use ccpp_kinds, only: kind_phys - - private - implicit none - - !! \section arg_table_ddt1_t - !! \htmlinclude ddt1_t.html - !! - type, public :: ddt1_t - real, pointer :: undocumented_array(:) => NULL() - end type ddt1_t - - !! \section arg_table_ddt2_t - !! \htmlinclude ddt2_t.html - !! - type, public :: ddt2_t - integer, public :: num_vars = 0 - real(kind_phys), allocatable :: vars(:,:,:) - - end type ddt2_t - -end module ddt2 diff --git a/test/unit_tests/sample_host_files/ddt2.meta b/test/unit_tests/sample_host_files/ddt2.meta deleted file mode 100644 index 159f08b0..00000000 --- a/test/unit_tests/sample_host_files/ddt2.meta +++ /dev/null @@ -1,29 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ddt1_t - type = ddt - -[ccpp-arg-table] - name = ddt1_t - type = ddt - -######################################################################## -[ccpp-table-properties] - name = ddt2_t - type = ddt - -[ccpp-arg-table] - name = ddt2_t - type = ddt -[ num_vars ] - standard_name = ddt_var_array_dimension - long_name = Number of vars managed by ddt2 - units = count - dimensions = () - type = integer -[ vars ] - standard_name = vars_array - long_name = Array of vars managed by ddt2 - units = none - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys diff --git a/test/unit_tests/sample_host_files/ddt2_extra_var.F90 b/test/unit_tests/sample_host_files/ddt2_extra_var.F90 deleted file mode 100644 index 00b4c170..00000000 --- a/test/unit_tests/sample_host_files/ddt2_extra_var.F90 +++ /dev/null @@ -1,34 +0,0 @@ -module ddt2_extra_var - - use ccpp_kinds, only: kind_phys - - private - implicit none - - !! \section arg_table_ddt1_t - !! \htmlinclude ddt1_t.html - !! - type, public :: ddt1_t - real, pointer :: undocumented_array(:) => NULL() - end type ddt1_t - - !! \section arg_table_ddt2_t - !! \htmlinclude ddt2_t.html - !! - type, public :: ddt2_t - integer, public :: num_vars = 0 - real(kind_phys), allocatable :: vars(:,:,:) - contains - procedure :: get_num_vars - end type ddt2_t - -CONTAINS - - integer function get_num_vars(this) - class(ddt2_t), intent(in) :: this - - get_num_vars = this%num_vars - - end function get_num_vars - -end module ddt2_extra_var diff --git a/test/unit_tests/sample_host_files/ddt2_extra_var.meta b/test/unit_tests/sample_host_files/ddt2_extra_var.meta deleted file mode 100644 index 867720e5..00000000 --- a/test/unit_tests/sample_host_files/ddt2_extra_var.meta +++ /dev/null @@ -1,34 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ddt1_t - type = ddt - -[ccpp-arg-table] - name = ddt1_t - type = ddt - -######################################################################## -[ccpp-table-properties] - name = ddt2_t - type = ddt - -[ccpp-arg-table] - name = ddt2_t - type = ddt -[ num_vars ] - standard_name = ddt_var_array_dimension - long_name = Number of vars managed by ddt2 - units = count - dimensions = () - type = integer -[ vars ] - standard_name = vars_array - long_name = Array of vars managed by ddt2 - units = none - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys -[ bogus ] - standard_name = misplaced_variable - units = count - dimensions = () - type = integer diff --git a/test/unit_tests/sample_host_files/ddt_data1_mod.F90 b/test/unit_tests/sample_host_files/ddt_data1_mod.F90 deleted file mode 100644 index 5efe0845..00000000 --- a/test/unit_tests/sample_host_files/ddt_data1_mod.F90 +++ /dev/null @@ -1,30 +0,0 @@ -module ddt_data1_mod - - use ccpp_kinds, only: kind_phys - - private - implicit none - - !! \section arg_table_ddt1_t - !! \htmlinclude ddt1_t.html - !! - type, public :: ddt1_t - real, pointer :: undocumented_array(:) => NULL() - end type ddt1_t - - !! \section arg_table_ddt2_t - !! \htmlinclude ddt2_t.html - !! - type, public :: ddt2_t - integer, public :: num_vars = 0 - real(kind_phys), allocatable :: vars(:,:,:) - - end type ddt2_t - - !> \section arg_table_ddt_data1_mod Argument Table - !! \htmlinclude arg_table_ddt_data1_mod.html - real(kind_phys) :: ps1 - real(kind_phys), allocatable :: xbox(:,:) - real(kind_phys), allocatable :: switch(:,:) - -end module ddt_data1_mod diff --git a/test/unit_tests/sample_host_files/ddt_data1_mod.meta b/test/unit_tests/sample_host_files/ddt_data1_mod.meta deleted file mode 100644 index e149c07b..00000000 --- a/test/unit_tests/sample_host_files/ddt_data1_mod.meta +++ /dev/null @@ -1,56 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = ddt1_t - type = ddt - -[ccpp-arg-table] - name = ddt1_t - type = ddt - -######################################################################## -[ccpp-table-properties] - name = ddt2_t - type = ddt - -[ccpp-arg-table] - name = ddt2_t - type = ddt -[ num_vars ] - standard_name = ddt_var_array_dimension - long_name = Number of vars managed by ddt2 - units = count - dimensions = () - type = integer -[ vars ] - standard_name = vars_array - long_name = Array of vars managed by ddt2 - units = none - dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) - type = real | kind = kind_phys - -######################################################################## -[ccpp-table-properties] - name = ddt_data1_mod - type = module -[ccpp-arg-table] - name = ddt_data1_mod - type = module -[ ps1 ] - standard_name = play_station - state_variable = true - type = real | kind = kind_phys - units = Pa - dimensions = () -[ xbox ] - standard_name = xbox - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) -[ switch ] - standard_name = nintendo_switch - long_name = Incompatible junk - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) diff --git a/test/unit_tests/sample_host_files/mismatch_hdim_mod.F90 b/test/unit_tests/sample_host_files/mismatch_hdim_mod.F90 deleted file mode 100644 index b3ebe52b..00000000 --- a/test/unit_tests/sample_host_files/mismatch_hdim_mod.F90 +++ /dev/null @@ -1,11 +0,0 @@ -module mismatch_hdim_mod - - use ccpp_kinds, only: kind_phys - - !> \section arg_table_mismatch_hdim_mod Argument Table - !! \htmlinclude arg_table_mismatch_hdim_mod.html - real(kind_phys) :: ps1 - real(kind_phys), allocatable :: xbox(:,:) - real(kind_phys), allocatable :: switch(:,:) - -end module mismatch_hdim_mod diff --git a/test/unit_tests/sample_host_files/mismatch_hdim_mod.meta b/test/unit_tests/sample_host_files/mismatch_hdim_mod.meta deleted file mode 100644 index 24f6ba77..00000000 --- a/test/unit_tests/sample_host_files/mismatch_hdim_mod.meta +++ /dev/null @@ -1,25 +0,0 @@ -[ccpp-table-properties] - name = mismatch_hdim_mod - type = module -[ccpp-arg-table] - name = mismatch_hdim_mod - type = module -[ ps1 ] - standard_name = play_station - state_variable = true - type = real | kind = kind_phys - units = Pa - dimensions = () -[ xbox ] - standard_name = xbox - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_loop_extent, vertical_layer_dimension) -[ switch ] - standard_name = nintendo_switch - long_name = Incompatible junk - state_variable = true - type = real | kind = kind_phys - units = m s-1 - dimensions = (horizontal_loop_being:horizontal_loop_end, vertical_layer_dimension) diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.F90 b/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.F90 deleted file mode 100644 index 20446f64..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.F90 +++ /dev/null @@ -1,38 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE CCPPeq1_var_in_fort_meta - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: CCPPeq1_var_in_fort_meta_run - -CONTAINS - - !> \section arg_table_CCPPeq1_var_in_fort_meta_run Argument Table - !! \htmlinclude arg_table_CCPPeq1_var_in_fort_meta_run.html - !! - subroutine CCPPeq1_var_in_fort_meta_run (foo, & -#ifdef CCPP - bar, & -#endif - errmsg, errflg) - - integer, intent(in) :: foo -#ifdef CCPP - real(kind_phys), intent(in) :: bar -#endif - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine CCPPeq1_var_in_fort_meta_run - -END MODULE CCPPeq1_var_in_fort_meta diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.meta b/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.meta deleted file mode 100644 index 9a05a3ac..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_in_fort_meta.meta +++ /dev/null @@ -1,37 +0,0 @@ -[ccpp-table-properties] - name = CCPPeq1_var_in_fort_meta - type = scheme - -######################################################################## -[ccpp-arg-table] - name = CCPPeq1_var_in_fort_meta_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ bar ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.F90 b/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.F90 deleted file mode 100644 index 85b9b370..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.F90 +++ /dev/null @@ -1,38 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE CCPPeq1_var_missing_in_fort - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: CCPPeq1_var_missing_in_fort_run - -CONTAINS - - !> \section arg_table_CCPPeq1_var_missing_in_fort_run Argument Table - !! \htmlinclude arg_table_CCPPeq1_var_missing_in_fort_run.html - !! - subroutine CCPPeq1_var_missing_in_fort_run (foo, & -#ifndef CCPP - bar, & -#endif - errmsg, errflg) - - integer, intent(in) :: foo -#ifndef CCPP - real(kind_phys), intent(in) :: bar -#endif - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine CCPPeq1_var_missing_in_fort_run - -END MODULE CCPPeq1_var_missing_in_fort diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.meta b/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.meta deleted file mode 100644 index 283c697d..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_fort.meta +++ /dev/null @@ -1,37 +0,0 @@ -[ccpp-table-properties] - name = CCPPeq1_var_missing_in_fort - type = scheme - -######################################################################## -[ccpp-arg-table] - name = CCPPeq1_var_missing_in_fort_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ bar ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.F90 b/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.F90 deleted file mode 100644 index 155db942..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.F90 +++ /dev/null @@ -1,38 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE CCPPeq1_var_missing_in_meta - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: CCPPeq1_var_missing_in_meta_finalize - -CONTAINS - - !> \section arg_table_CCPPeq1_var_missing_in_meta_finalize Argument Table - !! \htmlinclude arg_table_CCPPeq1_var_missing_in_meta_finalize.html - !! - subroutine CCPPeq1_var_missing_in_meta_finalize (foo, & -#ifdef CCPP - bar, & -#endif - errmsg, errflg) - - integer, intent(in) :: foo -#ifdef CCPP - real(kind_phys), intent(in) :: bar -#endif - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine CCPPeq1_var_missing_in_meta_finalize - -END MODULE CCPPeq1_var_missing_in_meta diff --git a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.meta b/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.meta deleted file mode 100644 index 92f20ff1..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPeq1_var_missing_in_meta.meta +++ /dev/null @@ -1,29 +0,0 @@ -[ccpp-table-properties] - name = CCPPeq1_var_missing_in_meta - type = scheme - -######################################################################## -[ccpp-arg-table] - name = CCPPeq1_var_missing_in_meta_finalize - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.F90 b/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.F90 deleted file mode 100644 index fed23ff0..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.F90 +++ /dev/null @@ -1,38 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE CCPPgt1_var_in_fort_meta - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: CCPPgt1_var_in_fort_meta_init - -CONTAINS - - !> \section arg_table_CCPPgt1_var_in_fort_meta_init Argument Table - !! \htmlinclude arg_table_CCPPgt1_var_in_fort_meta_init.html - !! - subroutine CCPPgt1_var_in_fort_meta_init (foo, & -#if CCPP > 1 - bar, & -#endif - errmsg, errflg) - - integer, intent(in) :: foo -#if CCPP > 1 - real(kind_phys), intent(in) :: bar -#endif - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine CCPPgt1_var_in_fort_meta_init - -END MODULE CCPPgt1_var_in_fort_meta diff --git a/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.meta b/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.meta deleted file mode 100644 index 00924ba6..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPgt1_var_in_fort_meta.meta +++ /dev/null @@ -1,37 +0,0 @@ -[ccpp-table-properties] - name = CCPPgt1_var_in_fort_meta - type = scheme - -######################################################################## -[ccpp-arg-table] - name = CCPPgt1_var_in_fort_meta_init - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ bar ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.F90 b/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.F90 deleted file mode 100644 index 14a49168..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.F90 +++ /dev/null @@ -1,38 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE CCPPnotset_var_missing_in_meta - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: CCPPnotset_var_missing_in_meta_run - -CONTAINS - - !> \section arg_table_CCPPnotset_var_missing_in_meta_run Argument Table - !! \htmlinclude arg_table_CCPPnotset_var_missing_in_meta_run.html - !! - subroutine CCPPnotset_var_missing_in_meta_run (foo, & -#ifndef CCPP - bar, & -#endif - errmsg, errflg) - - integer, intent(in) :: foo -#ifndef CCPP - real(kind_phys), intent(in) :: bar -#endif - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine CCPPnotset_var_missing_in_meta_run - -END MODULE CCPPnotset_var_missing_in_meta diff --git a/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.meta b/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.meta deleted file mode 100644 index c6435aab..00000000 --- a/test/unit_tests/sample_scheme_files/CCPPnotset_var_missing_in_meta.meta +++ /dev/null @@ -1,29 +0,0 @@ -[ccpp-table-properties] - name = CCPPnotset_var_missing_in_meta - type = scheme - -######################################################################## -[ccpp-arg-table] - name = CCPPnotset_var_missing_in_meta_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/invalid_dummy_arg.F90 b/test/unit_tests/sample_scheme_files/invalid_dummy_arg.F90 deleted file mode 100644 index 16f93864..00000000 --- a/test/unit_tests/sample_scheme_files/invalid_dummy_arg.F90 +++ /dev/null @@ -1,43 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE invalid_dummy_arg - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: invalid_dummy_arg_run - -CONTAINS - - !> \section arg_table_invalid_dummy_arg_run Argument Table - !! \htmlinclude arg_table_invalid_dummy_arg_run.html - !! - subroutine invalid_dummy_arg_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: woohoo(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE invalid_dummy_arg_run - -END MODULE invalid_dummy_arg diff --git a/test/unit_tests/sample_scheme_files/invalid_dummy_arg.meta b/test/unit_tests/sample_scheme_files/invalid_dummy_arg.meta deleted file mode 100644 index 4dd8e9e6..00000000 --- a/test/unit_tests/sample_scheme_files/invalid_dummy_arg.meta +++ /dev/null @@ -1,66 +0,0 @@ -[ccpp-table-properties] - name = invalid_dummy_arg - type = scheme - -######################################################################## -[ccpp-arg-table] - name = invalid_dummy_arg_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/invalid_subr_stmnt.F90 b/test/unit_tests/sample_scheme_files/invalid_subr_stmnt.F90 deleted file mode 100644 index 98100553..00000000 --- a/test/unit_tests/sample_scheme_files/invalid_subr_stmnt.F90 +++ /dev/null @@ -1,30 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE invalid_subr_stmnt - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: invalid_subr_stmnt_init - -CONTAINS - - !> \section arg_table_invalid_subr_stmnt_init Argument Table - !! \htmlinclude arg_table_invalid_subr_stmnt_init.html - !! - subroutine invalid_subr_stmnt_init (woohoo, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine invalid_subr_stmnt_init - -END MODULE invalid_subr_stmnt diff --git a/test/unit_tests/sample_scheme_files/invalid_subr_stmnt.meta b/test/unit_tests/sample_scheme_files/invalid_subr_stmnt.meta deleted file mode 100644 index 30bdc1e9..00000000 --- a/test/unit_tests/sample_scheme_files/invalid_subr_stmnt.meta +++ /dev/null @@ -1,23 +0,0 @@ -[ccpp-table-properties] - name = invalid_subr_stmnt - type = scheme - -######################################################################## -[ccpp-arg-table] - name = invalid_subr_stmnt_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/mismatch_hdim.F90 b/test/unit_tests/sample_scheme_files/mismatch_hdim.F90 deleted file mode 100644 index 67680917..00000000 --- a/test/unit_tests/sample_scheme_files/mismatch_hdim.F90 +++ /dev/null @@ -1,48 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE mismatch_hdim - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: mismatch_hdim_init - PUBLIC :: mismatch_hdim_run - -CONTAINS - - !> \section arg_table_mismatch_hdim_run Argument Table - !! \htmlinclude arg_table_mismatch_hdim_run.html - !! - subroutine mismatch_hdim_run(tsfc, errmsg, errflg) - - real(kind_phys), intent(inout) :: tsfc(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - tsfc = tsfc-1.0_kind_phys - - END SUBROUTINE mismatch_hdim_run - - !> \section arg_table_mismatch_hdim_init Argument Table - !! \htmlinclude arg_table_mismatch_hdim_init.html - !! - subroutine mismatch_hdim_init (tsfc, errmsg, errflg) - - real(kind_phys), intent(inout) :: tsfc(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - tsfc = tsfc+1.0_kind_phys - - errmsg = '' - errflg = 0 - - end subroutine mismatch_hdim_init - -END MODULE mismatch_hdim diff --git a/test/unit_tests/sample_scheme_files/mismatch_hdim.meta b/test/unit_tests/sample_scheme_files/mismatch_hdim.meta deleted file mode 100644 index 55d87fc3..00000000 --- a/test/unit_tests/sample_scheme_files/mismatch_hdim.meta +++ /dev/null @@ -1,55 +0,0 @@ -[ccpp-table-properties] - name = mismatch_hdim - type = scheme - -######################################################################## -[ccpp-arg-table] - name = mismatch_hdim_run - type = scheme -[ tsfc ] - standard_name = temperature_at_surface - units = K - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = mismatch_hdim_init - type = scheme -[ tsfc ] - standard_name = temperature_at_surface - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/mismatch_intent.F90 b/test/unit_tests/sample_scheme_files/mismatch_intent.F90 deleted file mode 100644 index abcf7bc0..00000000 --- a/test/unit_tests/sample_scheme_files/mismatch_intent.F90 +++ /dev/null @@ -1,75 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE mismatch_intent - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: mismatch_intent_init - PUBLIC :: mismatch_intent_run - PUBLIC :: mismatch_intent_finalize - -CONTAINS - - !> \section arg_table_mismatch_intent_run Argument Table - !! \htmlinclude arg_table_mismatch_intent_run.html - !! - subroutine mismatch_intent_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: temp_prev(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE mismatch_intent_run - - !> \section arg_table_mismatch_intent_init Argument Table - !! \htmlinclude arg_table_mismatch_intent_init.html - !! - subroutine mismatch_intent_init (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine mismatch_intent_init - - !> \section arg_table_mismatch_intent_finalize Argument Table - !! \htmlinclude arg_table_mismatch_intent_finalize.html - !! - subroutine mismatch_intent_finalize (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine mismatch_intent_finalize - -END MODULE mismatch_intent diff --git a/test/unit_tests/sample_scheme_files/mismatch_intent.meta b/test/unit_tests/sample_scheme_files/mismatch_intent.meta deleted file mode 100644 index d838db03..00000000 --- a/test/unit_tests/sample_scheme_files/mismatch_intent.meta +++ /dev/null @@ -1,102 +0,0 @@ -[ccpp-table-properties] - name = mismatch_intent - type = scheme - -######################################################################## -[ccpp-arg-table] - name = mismatch_intent_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = () - type = real - kind = kind_phys - intent = in -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_fizz - intent = inout -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = integer - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = mismatch_intent_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = mismatch_intent_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/missing_arg_table.F90 b/test/unit_tests/sample_scheme_files/missing_arg_table.F90 deleted file mode 100644 index 9d0a02af..00000000 --- a/test/unit_tests/sample_scheme_files/missing_arg_table.F90 +++ /dev/null @@ -1,75 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE missing_arg_table - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: missing_arg_table_init - PUBLIC :: missing_arg_table_run - PUBLIC :: missing_arg_table_finalize - -CONTAINS - - !> \section arg_table_missing_arg_table_run Argument Table - !! \htmlinclude arg_table_missing_arg_table_run.html - !! - subroutine missing_arg_table_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: temp_prev(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE missing_arg_table_run - - !> \section arg_table_missing_arg_table_init Argument Table - !! \htmlinclude arg_table_missing_arg_table_init.html - !! - subroutine missing_arg_table_init (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine missing_arg_table_init - - !> \section arg_table_missing_arg_table_finalize Argument Table - !! \htmlinclude arg_table_missing_arg_table_finalize.html - !! - subroutine missing_arg_table_finalize (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine missing_arg_table_finalize - -END MODULE missing_arg_table diff --git a/test/unit_tests/sample_scheme_files/missing_arg_table.meta b/test/unit_tests/sample_scheme_files/missing_arg_table.meta deleted file mode 100644 index f4b920b2..00000000 --- a/test/unit_tests/sample_scheme_files/missing_arg_table.meta +++ /dev/null @@ -1,41 +0,0 @@ -[ccpp-table-properties] - name = missing_arg_table - type = scheme - -######################################################################## -[ccpp-arg-table] - name = missing_arg_table_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = missing_arg_table_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/missing_fort_header.F90 b/test/unit_tests/sample_scheme_files/missing_fort_header.F90 deleted file mode 100644 index 92981eb5..00000000 --- a/test/unit_tests/sample_scheme_files/missing_fort_header.F90 +++ /dev/null @@ -1,73 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE missing_fort_header - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: missing_fort_header_init - PUBLIC :: missing_fort_header_run - PUBLIC :: missing_fort_header_finalize - -CONTAINS - - !> \section fort_header_missing_arg_table_run Argument Table - !! \htmlinclude fort_header_missing_arg_table_run.html - !! - subroutine missing_fort_header_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: temp_prev(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE missing_fort_header_run - - !> \section fort_header_missing_arg_table_init Argument Table - !! \htmlinclude fort_header_missing_arg_table_init.html - !! - subroutine missing_fort_header_init (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine missing_fort_header_init - - !! - subroutine missing_fort_header_finalize (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine missing_fort_header_finalize - -END MODULE missing_fort_header diff --git a/test/unit_tests/sample_scheme_files/missing_fort_header.meta b/test/unit_tests/sample_scheme_files/missing_fort_header.meta deleted file mode 100644 index d4478ffc..00000000 --- a/test/unit_tests/sample_scheme_files/missing_fort_header.meta +++ /dev/null @@ -1,102 +0,0 @@ -[ccpp-table-properties] - name = missing_fort_header - type = scheme - -######################################################################## -[ccpp-arg-table] - name = missing_fort_header_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = missing_fort_header_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = missing_fort_header_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/reorder.F90 b/test/unit_tests/sample_scheme_files/reorder.F90 deleted file mode 100644 index 690aebe0..00000000 --- a/test/unit_tests/sample_scheme_files/reorder.F90 +++ /dev/null @@ -1,73 +0,0 @@ -MODULE reorder - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: reorder_init - PUBLIC :: reorder_run - PUBLIC :: reorder_finalize - -CONTAINS - - !> \section arg_table_reorder_run Argument Table - !! \htmlinclude arg_table_reorder_run.html - !! - subroutine reorder_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: temp_prev(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE reorder_run - - !> \section arg_table_reorder_init Argument Table - !! \htmlinclude arg_table_reorder_init.html - !! - subroutine reorder_init (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - end subroutine reorder_init - - !> \section arg_table_reorder_finalize Argument Table - !! \htmlinclude arg_table_reorder_finalize.html - !! - subroutine reorder_finalize (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - errmsg = '' - errflg = 0 - - end subroutine reorder_finalize - -END MODULE reorder - - ! add some stuff here to check if codee really ignores this - - -! BLA \ No newline at end of file diff --git a/test/unit_tests/sample_scheme_files/reorder.meta b/test/unit_tests/sample_scheme_files/reorder.meta deleted file mode 100644 index e69f6f8d..00000000 --- a/test/unit_tests/sample_scheme_files/reorder.meta +++ /dev/null @@ -1,102 +0,0 @@ -[ccpp-table-properties] - name = reorder - type = scheme - -######################################################################## -[ccpp-arg-table] - name = reorder_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = reorder_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = reorder_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_scheme_files/temp_adjust.F90 b/test/unit_tests/sample_scheme_files/temp_adjust.F90 deleted file mode 100644 index 70613ba1..00000000 --- a/test/unit_tests/sample_scheme_files/temp_adjust.F90 +++ /dev/null @@ -1,96 +0,0 @@ -! Test parameterization with no vertical level -! - -MODULE temp_adjust - - USE ccpp_kinds, ONLY: kind_phys - - IMPLICIT NONE - PRIVATE - - PUBLIC :: temp_adjust_init - PUBLIC :: temp_adjust_run - PUBLIC :: temp_adjust_finalize - -CONTAINS - - !> \section arg_table_temp_adjust_register Argument Table - !! \htmlinclude arg_table_temp_adjust_register.html - !! - subroutine temp_adjust_register(config_var, dyn_const, errflg, errmsg) - logical, intent(in) :: config_var - type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - if (.not. config_var) then - return - end if - - allocate(dyn_const(1)) - call dyn_const(1)%instantiate(std_name="dyn_const", long_name='dyn const', & - diag_name='DYNCONST', units='kg kg-1', default_value=1._kind_phys, & - vertical_dim='vertical_layer_dimension', advected=.true., & - errcode=errflg, errmsg=errmsg) - - end subroutine temp_adjust_register - - !> \section arg_table_temp_adjust_run Argument Table - !! \htmlinclude arg_table_temp_adjust_run.html - !! - subroutine temp_adjust_run(foo, timestep, temp_prev, temp_layer, qv, ps, & - errmsg, errflg) - - integer, intent(in) :: foo - real(kind_phys), intent(in) :: timestep - real(kind_phys), intent(inout) :: qv(:) - real(kind_phys), intent(inout) :: ps(:) - REAL(kind_phys), intent(in) :: temp_prev(:) - REAL(kind_phys), intent(inout) :: temp_layer(foo) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - - integer :: col_index - - errmsg = '' - errflg = 0 - - do col_index = 1, foo - temp_layer(col_index) = temp_layer(col_index) + temp_prev(col_index) - qv(col_index) = qv(col_index) + 1.0_kind_phys - end do - - END SUBROUTINE temp_adjust_run - - !> \section arg_table_temp_adjust_init Argument Table - !! \htmlinclude arg_table_temp_adjust_init.html - !! - subroutine temp_adjust_init (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_adjust_init - - !> \section arg_table_temp_adjust_finalize Argument Table - !! \htmlinclude arg_table_temp_adjust_finalize.html - !! - subroutine temp_adjust_finalize (errmsg, errflg) - - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! This routine currently does nothing - - errmsg = '' - errflg = 0 - - end subroutine temp_adjust_finalize - -END MODULE temp_adjust diff --git a/test/unit_tests/sample_scheme_files/temp_adjust.meta b/test/unit_tests/sample_scheme_files/temp_adjust.meta deleted file mode 100644 index 4b96e316..00000000 --- a/test/unit_tests/sample_scheme_files/temp_adjust.meta +++ /dev/null @@ -1,134 +0,0 @@ -[ccpp-table-properties] - name = temp_adjust - type = scheme - -######################################################################## -[ccpp-arg-table] - name = temp_adjust_register - type = scheme -[ config_var ] - standard_name = configuration_variable - type = logical - units = none - dimensions = () - intent = in -[ dyn_const ] - standard_name = dynamic_constituents_for_temp_adjust - type = ccpp_constituent_properties_t - units = none - dimensions = () - intent = out - allocatable = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -######################################################################## -[ccpp-arg-table] - name = temp_adjust_run - type = scheme -[ foo ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ timestep ] - standard_name = time_step_for_physics - long_name = time step - units = s - dimensions = () - type = real - kind = kind_phys - intent = in -[ temp_prev ] - standard_name = potential_temperature_at_previous_timestep - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[ temp_layer ] - standard_name = potential_temperature - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ qv ] - standard_name = water_vapor_specific_humidity - units = kg kg-1 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ ps ] - standard_name = surface_air_pressure - state_variable = true - type = real - kind = kind_phys - units = Pa - dimensions = (horizontal_loop_extent) - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_adjust_init - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[ccpp-arg-table] - name = temp_adjust_finalize - type = scheme -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/unit_tests/sample_suite_files/another_suite.xml b/test/unit_tests/sample_suite_files/another_suite.xml deleted file mode 100644 index 72346933..00000000 --- a/test/unit_tests/sample_suite_files/another_suite.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - another_scheme - - more_scheme - - diff --git a/test/unit_tests/sample_suite_files/another_suite2.xml b/test/unit_tests/sample_suite_files/another_suite2.xml deleted file mode 100644 index def97177..00000000 --- a/test/unit_tests/sample_suite_files/another_suite2.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - another_scheme - - more_scheme - - - - another_scheme - - more_scheme - - diff --git a/test/unit_tests/sample_suite_files/nested_full_suite.xml b/test/unit_tests/sample_suite_files/nested_full_suite.xml deleted file mode 100644 index 2979f3ff..00000000 --- a/test/unit_tests/sample_suite_files/nested_full_suite.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - g1_scheme1 - - - - - diff --git a/test/unit_tests/sample_suite_files/subsuite1.xml b/test/unit_tests/sample_suite_files/subsuite1.xml deleted file mode 100644 index c58ed752..00000000 --- a/test/unit_tests/sample_suite_files/subsuite1.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - scheme_subsuite1 - - diff --git a/test/unit_tests/sample_suite_files/subsuite_inline.xml b/test/unit_tests/sample_suite_files/subsuite_inline.xml deleted file mode 100644 index 706ea801..00000000 --- a/test/unit_tests/sample_suite_files/subsuite_inline.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - scheme1i - scheme2i - scheme1i - - diff --git a/test/unit_tests/sample_suite_files/suite_bad_v2_duplicate_group.xml b/test/unit_tests/sample_suite_files/suite_bad_v2_duplicate_group.xml deleted file mode 100644 index 8ea72077..00000000 --- a/test/unit_tests/sample_suite_files/suite_bad_v2_duplicate_group.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - effr_pre - - - scheme9 - - - scheme3 - - - - diff --git a/test/unit_tests/sample_suite_files/suite_bad_v2_suite_tag.xml b/test/unit_tests/sample_suite_files/suite_bad_v2_suite_tag.xml deleted file mode 100644 index 6bc4f424..00000000 --- a/test/unit_tests/sample_suite_files/suite_bad_v2_suite_tag.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - subsuite_inline - - diff --git a/test/unit_tests/sample_suite_files/suite_bad_version01.xml b/test/unit_tests/sample_suite_files/suite_bad_version01.xml deleted file mode 100644 index ecee0c63..00000000 --- a/test/unit_tests/sample_suite_files/suite_bad_version01.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - scheme1 - scheme2 - - diff --git a/test/unit_tests/sample_suite_files/suite_bad_version02.xml b/test/unit_tests/sample_suite_files/suite_bad_version02.xml deleted file mode 100644 index 55aff67a..00000000 --- a/test/unit_tests/sample_suite_files/suite_bad_version02.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - scheme1 - scheme2 - - diff --git a/test/unit_tests/sample_suite_files/suite_bad_version03.xml b/test/unit_tests/sample_suite_files/suite_bad_version03.xml deleted file mode 100644 index 794bfe7b..00000000 --- a/test/unit_tests/sample_suite_files/suite_bad_version03.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - scheme1 - scheme2 - - diff --git a/test/unit_tests/sample_suite_files/suite_bad_version04.xml b/test/unit_tests/sample_suite_files/suite_bad_version04.xml deleted file mode 100644 index aaaab154..00000000 --- a/test/unit_tests/sample_suite_files/suite_bad_version04.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - scheme1 - scheme2 - - diff --git a/test/unit_tests/sample_suite_files/suite_good_v1_test01.xml b/test/unit_tests/sample_suite_files/suite_good_v1_test01.xml deleted file mode 100644 index 0eee366b..00000000 --- a/test/unit_tests/sample_suite_files/suite_good_v1_test01.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - scheme1 - scheme2 - - diff --git a/test/unit_tests/sample_suite_files/suite_good_v1_test02.xml b/test/unit_tests/sample_suite_files/suite_good_v1_test02.xml deleted file mode 100644 index 3d355cc5..00000000 --- a/test/unit_tests/sample_suite_files/suite_good_v1_test02.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - scheme1 - scheme2 - scheme3 - scheme2 - scheme1 - - diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test01.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test01.xml deleted file mode 100644 index 73c30732..00000000 --- a/test/unit_tests/sample_suite_files/suite_good_v2_test01.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - scheme5 - - scheme9 - - diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test01_exp.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test01_exp.xml deleted file mode 100644 index dcecf218..00000000 --- a/test/unit_tests/sample_suite_files/suite_good_v2_test01_exp.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - scheme5 - scheme1i - scheme2i - scheme1i - scheme9 - - diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test02.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test02.xml deleted file mode 100644 index c7b5b8c9..00000000 --- a/test/unit_tests/sample_suite_files/suite_good_v2_test02.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - scheme6 - - - - diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test02_exp.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test02_exp.xml deleted file mode 100644 index 6b100283..00000000 --- a/test/unit_tests/sample_suite_files/suite_good_v2_test02_exp.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - scheme6 - - - another_scheme - - more_scheme - - diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test03.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test03.xml deleted file mode 100644 index eff15298..00000000 --- a/test/unit_tests/sample_suite_files/suite_good_v2_test03.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - scheme13 - - effr_pre - - - main_calc - - - main_post - - - - - - diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test03_exp.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test03_exp.xml deleted file mode 100644 index 5f9a9987..00000000 --- a/test/unit_tests/sample_suite_files/suite_good_v2_test03_exp.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - scheme13 - - effr_pre - - - main_calc - - - main_post - - - another_scheme - - more_scheme - - another_scheme - - more_scheme - - - g1_scheme1 - - - scheme_subsuite1 - - diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test04.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test04.xml deleted file mode 100644 index abb87008..00000000 --- a/test/unit_tests/sample_suite_files/suite_good_v2_test04.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - effr_pre - - - main_calc - - - main_post - - - - - - diff --git a/test/unit_tests/sample_suite_files/suite_good_v2_test04_exp.xml b/test/unit_tests/sample_suite_files/suite_good_v2_test04_exp.xml deleted file mode 100644 index af103e7d..00000000 --- a/test/unit_tests/sample_suite_files/suite_good_v2_test04_exp.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - effr_pre - - - main_calc - - - main_post - - - another_scheme - - more_scheme - - another_scheme - - more_scheme - - - scheme_subsuite1 - - diff --git a/test/unit_tests/sample_suite_files/suite_invalid_group_fortran_id.xml b/test/unit_tests/sample_suite_files/suite_invalid_group_fortran_id.xml deleted file mode 100644 index 0fbb40dc..00000000 --- a/test/unit_tests/sample_suite_files/suite_invalid_group_fortran_id.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - scheme1 - scheme2 - - diff --git a/test/unit_tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml b/test/unit_tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml deleted file mode 100644 index e236cbd0..00000000 --- a/test/unit_tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - scheme-1 - scheme2 - - diff --git a/test/unit_tests/sample_suite_files/suite_invalid_suite_fortran_id.xml b/test/unit_tests/sample_suite_files/suite_invalid_suite_fortran_id.xml deleted file mode 100644 index ae8a28a4..00000000 --- a/test/unit_tests/sample_suite_files/suite_invalid_suite_fortran_id.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - scheme1 - scheme2 - - diff --git a/test/unit_tests/sample_suite_files/suite_missing_file.xml b/test/unit_tests/sample_suite_files/suite_missing_file.xml deleted file mode 100644 index 8c7b85e6..00000000 --- a/test/unit_tests/sample_suite_files/suite_missing_file.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - diff --git a/test/unit_tests/sample_suite_files/suite_missing_group.xml b/test/unit_tests/sample_suite_files/suite_missing_group.xml deleted file mode 100644 index a33078e6..00000000 --- a/test/unit_tests/sample_suite_files/suite_missing_group.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/test/unit_tests/sample_suite_files/suite_missing_loaded_suite.xml b/test/unit_tests/sample_suite_files/suite_missing_loaded_suite.xml deleted file mode 100644 index cf21a590..00000000 --- a/test/unit_tests/sample_suite_files/suite_missing_loaded_suite.xml +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - scheme23 - - - scheme9 - - - scheme3 - - - - diff --git a/test/unit_tests/sample_suite_files/suite_missing_version.xml b/test/unit_tests/sample_suite_files/suite_missing_version.xml deleted file mode 100644 index 463c56d9..00000000 --- a/test/unit_tests/sample_suite_files/suite_missing_version.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - scheme1 - scheme2 - - diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level2.xml b/test/unit_tests/sample_suite_files/suite_recurse_level2.xml deleted file mode 100644 index 35fed8b2..00000000 --- a/test/unit_tests/sample_suite_files/suite_recurse_level2.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - scheme13 - scheme3 - - scheme43 - - diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level2a.xml b/test/unit_tests/sample_suite_files/suite_recurse_level2a.xml deleted file mode 100644 index 2e018e39..00000000 --- a/test/unit_tests/sample_suite_files/suite_recurse_level2a.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - scheme13 - scheme3 - scheme43 - - - diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level3.xml b/test/unit_tests/sample_suite_files/suite_recurse_level3.xml deleted file mode 100644 index f38a9d8f..00000000 --- a/test/unit_tests/sample_suite_files/suite_recurse_level3.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - scheme13 - scheme3 - - scheme43 - - diff --git a/test/unit_tests/sample_suite_files/suite_recurse_level3a.xml b/test/unit_tests/sample_suite_files/suite_recurse_level3a.xml deleted file mode 100644 index a11182dc..00000000 --- a/test/unit_tests/sample_suite_files/suite_recurse_level3a.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - scheme13 - scheme3 - scheme43 - - - diff --git a/test/unit_tests/sample_suite_files/suite_recurse_top1.xml b/test/unit_tests/sample_suite_files/suite_recurse_top1.xml deleted file mode 100644 index 8e8c4f1a..00000000 --- a/test/unit_tests/sample_suite_files/suite_recurse_top1.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - scheme13 - - scheme23 - - - scheme9 - - - scheme3 - - - scheme43 - - diff --git a/test/unit_tests/sample_suite_files/suite_recurse_top2.xml b/test/unit_tests/sample_suite_files/suite_recurse_top2.xml deleted file mode 100644 index 87ce7057..00000000 --- a/test/unit_tests/sample_suite_files/suite_recurse_top2.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - scheme13 - - scheme23 - - - scheme9 - - - scheme3 - - scheme43 - - - diff --git a/test/unit_tests/test_common.py b/test/unit_tests/test_common.py deleted file mode 100644 index aa81ee8d..00000000 --- a/test/unit_tests/test_common.py +++ /dev/null @@ -1,92 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for functions in common.py - - Assumptions: - - Command line arguments: none - - Usage: python test_common.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import logging -import unittest - -TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_FILE = os.path.abspath(os.path.abspath(__file__)) -SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, os.pardir, "scripts")) -SAMPLE_FILES_DIR = os.path.join(TEST_DIR, "sample_files") - -if not os.path.exists(SCRIPTS_DIR): - raise ImportError("Cannot find scripts directory") - -sys.path.append(SCRIPTS_DIR) -logging.disable(level=logging.CRITICAL) - -# pylint: disable=wrong-import-position -import common -# pylint: enable=wrong-import-position - -class CommonTestCase(unittest.TestCase): - - """Tests functionality of functions in common.py""" - - def test_split_var_name_and_array_reference(self): - """Test split_var_name_and_array_reference() function""" - - self.assertEqual(common.split_var_name_and_array_reference("foo(:,a,1:ddt%ngas)"), - ("foo","(:,a,1:ddt%ngas)")) - - def test_encode_container(self): - """Test encode_container() function""" - - modulename = "ABCD1234" - typename = "COMPLEX" - schemename = "testscheme" - subroutinename = "testsubroutine" - self.assertEqual(common.encode_container(modulename),f"MODULE_{modulename.lower()}") - self.assertEqual(common.encode_container(modulename,typename),f"MODULE_{modulename.lower()} TYPE_{typename.lower()}") - self.assertEqual(common.encode_container(modulename,schemename,subroutinename), - f"MODULE_{modulename.lower()} SCHEME_{schemename.lower()} SUBROUTINE_{subroutinename.lower()}") - self.assertRaises(Exception,common.encode_container,modulename,typename,schemename,subroutinename) - self.assertRaises(Exception,common.encode_container) - - def test_decode_container(self): - """Test decode_container() function""" - - modulename = "abcd1234" - typename = "complex" - schemename = "testscheme" - subroutinename = "testsubroutine" - self.assertEqual(common.decode_container(f"MODULE_{modulename}"),f"MODULE {modulename}") - self.assertEqual(common.decode_container(f"MODULE_{modulename} TYPE_{typename}"), - f"MODULE {modulename} TYPE {typename}") - self.assertEqual(common.decode_container(f"MODULE_{modulename} SCHEME_{schemename} SUBROUTINE_{subroutinename}"), - f"MODULE {modulename} SCHEME {schemename} SUBROUTINE {subroutinename}") - self.assertRaises(Exception,common.decode_container, - f"MODULE_{modulename} TYPE_{typename} SCHEME_{schemename} SUBROUTINE_{subroutinename}") - self.assertRaises(Exception,common.decode_container,"That dog won't hunt, Monsignor") - self.assertRaises(Exception,common.decode_container) - - def test_string_to_python_identifier(self): - """Test string_to_python_identifier() function""" - - # Test various successful combinations - self.assertEqual(common.string_to_python_identifier("Test 1"),"Test_1") - self.assertEqual(common.string_to_python_identifier("Test.2"),"Test_p_2") - self.assertEqual(common.string_to_python_identifier("Test-3"),"Test_minus_3") - self.assertEqual(common.string_to_python_identifier("Test+4"),"Test_plus_4") - self.assertEqual(common.string_to_python_identifier("1"),"one") - self.assertEqual(common.string_to_python_identifier(" Test all--even +."), - "_Test_all_minus__minus_even__plus__p_") - # Test expected failures - self.assertRaises(Exception,common.string_to_python_identifier,"else") - self.assertRaises(Exception,common.string_to_python_identifier,"1 ") - self.assertRaises(Exception,common.string_to_python_identifier,"0") - self.assertRaises(Exception,common.string_to_python_identifier,"Disallowed character!") - -if __name__ == '__main__': - unittest.main() diff --git a/test/unit_tests/test_fortran_parse.py b/test/unit_tests/test_fortran_parse.py deleted file mode 100644 index 52696cd5..00000000 --- a/test/unit_tests/test_fortran_parse.py +++ /dev/null @@ -1,101 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for parsing Fortran - in scripts files scripts/fortran_tools/parse_fortran_file.py and - scripts/fortran_tools/parse_fortran.py - - Assumptions: - - Command line arguments: none - - Usage: python3 test_fortran_parse.py # run the unit tests ------------------------------------------------------------------------ -""" - -import os -import sys -import logging -import unittest - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_SCRIPTS_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, - os.pardir, "scripts")) -_SAMPLE_FILES_DIR = os.path.join(_TEST_DIR, "sample_files", "fortran_files") -_PRE_TMP_DIR = os.path.join(_TEST_DIR, "tmp") -_TMP_DIR = os.path.join(_PRE_TMP_DIR, "fortran_files") - -if not os.path.exists(_SCRIPTS_DIR): - raise ImportError(f"Cannot find scripts directory, {_SCRIPTS_DIR}") - -sys.path.append(_SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from fortran_tools import parse_fortran_file -from framework_env import CCPPFrameworkEnv -# pylint: enable=wrong-import-position - -############################################################################### -def remove_files(file_list): -############################################################################### - """Remove files in if they exist""" - if isinstance(file_list, str): - file_list = [file_list] - # end if - for fpath in file_list: - if os.path.exists(fpath): - os.remove(fpath) - # End if - # End for - -class FortranParseTestCase(unittest.TestCase): - - """Tests for `parse_fortran_file`.""" - - _run_env = None - - @classmethod - def setUpClass(cls): - """Clean output directory (tmp) before running tests""" - # Does "tmp" directory exist? If not then create it: - if not os.path.exists(_PRE_TMP_DIR): - os.makedirs(_PRE_TMP_DIR) - # end if - - # We need a run environment - logger = logging.getLogger(cls.__name__) - cls._run_env = CCPPFrameworkEnv(logger, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - - # Run inherited setup method: - super().setUpClass() - - def test_array_parsing(self): - """Test that the Fortran parser outputs an informative - error message for a badly formatted array specification. - Also, test that allowed specification strings are allowed. - """ - # Setup - testname = "array_parsing_test" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.F90") - # Exercise - header = "Test of parsing of Fortran array specification" - - with self.assertRaises(Exception) as context: - # Parse the file - _ = parse_fortran_file(source, self._run_env) - # end if - - # Check exception for expected error messages - self.assertTrue("bad_arr1: ';' is not a valid Fortran identifier" - in str(context.exception)) - self.assertTrue("bad_arr2: ';' is not a valid Fortran identifier" - in str(context.exception)) - self.assertTrue("bad_arr3: ';' is not a valid Fortran identifier" - in str(context.exception)) - self.assertTrue("Missing local_variables, ['bad_arr1', 'bad_arr2', 'bad_arr3'] in array_spec_test_run" - in str(context.exception)) - -if __name__ == "__main__": - unittest.main() diff --git a/test/unit_tests/test_fortran_write.py b/test/unit_tests/test_fortran_write.py deleted file mode 100644 index 49f551f7..00000000 --- a/test/unit_tests/test_fortran_write.py +++ /dev/null @@ -1,172 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for FortranWriter - in scripts file fortran/fortran_write.py - - Assumptions: - - Command line arguments: none - - Usage: python3 test_fortran_write.py # run the unit tests ------------------------------------------------------------------------ -""" - -import filecmp -import glob -import os -import sys -import unittest - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_SCRIPTS_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, - os.pardir, "scripts")) -_SAMPLE_FILES_DIR = os.path.join(_TEST_DIR, "sample_files", "fortran_files") -_PRE_TMP_DIR = os.path.join(_TEST_DIR, "tmp") -_TMP_DIR = os.path.join(_PRE_TMP_DIR, "fortran_files") - -if not os.path.exists(_SCRIPTS_DIR): - raise ImportError(f"Cannot find scripts directory, {_SCRIPTS_DIR}") - -sys.path.append(_SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from fortran_tools import FortranWriter -# pylint: enable=wrong-import-position - -############################################################################### -def remove_files(file_list): -############################################################################### - """Remove files in if they exist""" - if isinstance(file_list, str): - file_list = [file_list] - # end if - for fpath in file_list: - if os.path.exists(fpath): - os.remove(fpath) - # End if - # End for - -class FortranWriterTestCase(unittest.TestCase): - - """Tests for `FortranWriter`.""" - - @classmethod - def setUpClass(cls): - """Clean output directory (tmp) before running tests""" - #Does "tmp" directory exist? If not then create it: - if not os.path.exists(_PRE_TMP_DIR): - os.mkdir(_PRE_TMP_DIR) - # Ensure the "sample_files/fortran_files" directory exists and is empty - if os.path.exists(_TMP_DIR): - # Clear out all files: - remove_files(glob.iglob(os.path.join(_TMP_DIR, '*.*'))) - else: - os.makedirs(_TMP_DIR) - # end if - - #Run inherited setup method: - super().setUpClass() - - def test_line_breaking(self): - """Test that FortranWriter correctly breaks long lines""" - # Setup - testname = "linebreak_test" - compare = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.F90") - generate = os.path.join(_TMP_DIR, f"{testname}.F90") - # Exercise - header = "Test of line breaking for FortranWriter" - with FortranWriter(generate, 'w', header, f"{testname}") as gen: - # Test long declaration - qchr = "'" - nditems = 100 - data_items = ', '.join([f"{qchr}name{x:03}{qchr}" for x in range(nditems)]) - gen.write(f"character(len=7) :: data({nditems}) = (/ {data_items} /)", 1) - gen.end_module_header() - # Test long code lines - gen.blank_line() - gen.write("subroutine foo(ozone_constituents, aerosol_constituents, " - "volcaero_constituents, other_constituents)", 1) - gen.write("integer, intent(in) :: ozone_constituents(:)", 2) - gen.write("integer, intent(in) :: aerosol_constituents(:)", 2) - gen.write("integer, intent(in) :: volcaero_constituents(:)", 2) - gen.write("integer, intent(in) :: other_constituents(:)", 2) - gen.write("real, allocatable :: tracer_data_test_dynamic_constituents(:)", 2) - gen.comment("codee format off", 0) - gen.write("allocate(tracer_data_test_dynamic_constituents(0+" - "size(ozone_constituents)+size(aerosol_constituents)" - "+size(volcaero_constituents)+size(other_constituents)))", 2) - gen.blank_line() - line_items = ["write(6, '(a)') 'Cannot read columns_on_task from ", - "file'//', columns_on_task has no horizontal ", - "dimension; columns_on_task is a ", - "protected variable'"] - gen.write(f"{''.join(line_items)}", 2) - gen.comment("codee format on", 0) - gen.write("end subroutine foo", 1) - # end with - - # Check that file was generated - amsg = f"{generate} does not exist" - self.assertTrue(os.path.exists(generate), msg=amsg) - amsg = f"{generate} does not match {compare}" - self.assertTrue(filecmp.cmp(generate, compare, shallow=False), msg=amsg) - - def test_good_comments(self): - """Test that comments are written and broken correctly.""" - # Setup - testname = "comments_test" - compare = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.F90") - generate = os.path.join(_TMP_DIR, f"{testname}.F90") - # Exercise - header = "Test of comment writing for FortranWriter" - with FortranWriter(generate, 'w', header, f"{testname}") as gen: - gen.comment("codee format off", 0) - gen.comment("We can write comments in the module header", 0) - gen.comment("codee format on", 0) - gen.comment("We can write indented comments in the header", 1) - gen.write("integer :: foo ! Comment at end of line works", 1) - # Test long comments at end of line - gen.write(f"integer :: bar ! {'x'*100}", 1) - gen.write(f"integer :: baz ! {'y'*130}", 1) - gen.end_module_header() - # Test comment line in body - gen.comment("We can write comments in the module body", 1) - - # end with - - # Check that file was generated - amsg = f"{generate} does not exist" - self.assertTrue(os.path.exists(generate), msg=amsg) - amsg = f"{generate} does not match {compare}" - self.assertTrue(filecmp.cmp(generate, compare, shallow=False), msg=amsg) - - def test_long_strings(self): - """Test breaking of long strings""" - # Setup - testname = "long_string_test" - compare = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.F90") - generate = os.path.join(_TMP_DIR, f"{testname}.F90") - # Exercise - header = "Test of long string breaking for FortranWriter" - foostr = '0123456789'*10 - nxtchr = ord('0') - with FortranWriter(generate, 'w', header, f"{testname}") as gen: - while len(foostr) < 130: - gen.write(f"character(len={len(foostr)}) :: foo{len(foostr)} = '{foostr.strip()}'", 1) - foostr += chr(nxtchr) - nxtchr += 1 - if nxtchr > ord('9'): - nxtchr = ord('0') - # end if - # end while - # end with - - # Check that file was generated - amsg = f"{generate} does not exist" - self.assertTrue(os.path.exists(generate), msg=amsg) - amsg = f"{generate} does not match {compare}" - self.assertTrue(filecmp.cmp(generate, compare, shallow=False), msg=amsg) - -if __name__ == "__main__": - unittest.main() diff --git a/test/unit_tests/test_metadata_host_file.py b/test/unit_tests/test_metadata_host_file.py deleted file mode 100644 index 53e0a9fe..00000000 --- a/test/unit_tests/test_metadata_host_file.py +++ /dev/null @@ -1,280 +0,0 @@ -#! /usr/bin/env python3 - -""" ------------------------------------------------------------------------ - Description: capgen needs to compare a metadata header against the - associated CCPP Fortran interface routine. This set of - tests is testing the parse_host_model_files function in - ccpp_capgen.py which performs the operations in the first - bullet below. Each test calls this function. - - * This script contains unit tests that do the following: - 1) Read one or more metadata files (to collect - the metadata headers) - 2) Read the associated CCPP Fortran host file(s) (to - collect Fortran interfaces) - 3) Create a CCPP host model object - 3) Test the properties of the CCPP host model object - - * Tests include: - - Correctly parse and match a simple module file with - data fields (a data block) - - Correctly parse and match a simple module file with - a DDT definition - - Correctly parse and match a simple module file with - two DDT definitions - - Correctly parse and match a simple module file with - two DDT definitions and a data block - - "Test correct use of loop variables (horizontal - dimensions) in host metadata - - Assumptions: - - Command line arguments: none - - Usage: python3 test_metadata_host_file.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import logging -import unittest - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_SCRIPTS_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, - os.pardir, "scripts")) -if not os.path.exists(_SCRIPTS_DIR): - raise ImportError("Cannot find scripts directory") - -sys.path.append(_SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from ccpp_capgen import parse_host_model_files -from framework_env import CCPPFrameworkEnv -from parse_tools import CCPPError -# pylint: enable=wrong-import-position - -class MetadataHeaderTestCase(unittest.TestCase): - """Unit tests for parse_host_model_files""" - - def setUp(self): - """Setup important directories and logging""" - self._sample_files_dir = os.path.join(_TEST_DIR, "sample_host_files") - logger = logging.getLogger(self.__class__.__name__) - self._run_env = CCPPFrameworkEnv(logger, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - - def test_module_with_data(self): - """Test that a module containing a data block is parsed and matched - correctly.""" - # Setup - module_files = [os.path.join(self._sample_files_dir, "data1_mod.meta")] - # Exercise - hname = 'host_name_data1' - host_model = parse_host_model_files(module_files, hname, self._run_env) - # Verify the name of the host model - self.assertEqual(host_model.name, hname) - module_headers = host_model.metadata_tables() - self.assertEqual(len(module_headers), 1) - # Verify header titles - self.assertTrue('data1_mod' in module_headers) - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 3) - std_names = [x.get_prop_value('standard_name') for x in vlist] - self.assertTrue('play_station' in std_names) - self.assertTrue('xbox' in std_names) - self.assertTrue('nintendo_switch' in std_names) - - def test_module_with_one_ddt(self): - """Test that a module containing a DDT definition is parsed and matched - correctly.""" - # Setup - ddt_name = 'ddt1_t' - module_files = [os.path.join(self._sample_files_dir, "ddt1.meta")] - # Exercise - hname = 'host_name_ddt1' - host_model = parse_host_model_files(module_files, hname, self._run_env) - # Verify the name of the host model - self.assertEqual(host_model.name, hname) - module_headers = host_model.metadata_tables() - self.assertEqual(len(module_headers), 1) - # Verify header titles - self.assertTrue(ddt_name in module_headers) - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 0) - # Verify that the DDT was found and parsed - ddt_lib = host_model.ddt_lib - self.assertEqual(ddt_lib.name, f"{hname}_ddts_ddt_lib") - # Check DDT variables - ddt_mod = ddt_lib[ddt_name] - self.assertEqual(ddt_mod.name, ddt_name) - vlist = ddt_mod.variable_list() - self.assertEqual(len(vlist), 2) - std_names = [x.get_prop_value('standard_name') for x in vlist] - self.assertTrue('ddt_var_array_dimension' in std_names) - self.assertTrue('vars_array' in std_names) - - def test_module_with_two_ddts(self): - """Test that a module containing two DDT definitions is parsed and - matched correctly.""" - # Setup - ddt_names = ['ddt1_t', 'ddt2_t'] - ddt_vars = [(), ('ddt_var_array_dimension', 'vars_array')] - - module_files = [os.path.join(self._sample_files_dir, "ddt2.meta")] - # Exercise - hname = 'host_name_ddt2' - host_model = parse_host_model_files(module_files, hname, self._run_env) - # Verify the name of the host model - self.assertEqual(host_model.name, hname) - module_headers = host_model.metadata_tables() - self.assertEqual(len(module_headers), len(ddt_names)) - # Verify header titles - for ddt_name in ddt_names: - self.assertTrue(ddt_name in module_headers) - # end for - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 0) - # Verify that each DDT was found and parsed - ddt_lib = host_model.ddt_lib - self.assertEqual(ddt_lib.name, f"{hname}_ddts_ddt_lib") - # Check DDT variables - for index, ddt_name in enumerate(ddt_names): - ddt_mod = ddt_lib[ddt_name] - self.assertEqual(ddt_mod.name, ddt_name) - vlist = ddt_mod.variable_list() - self.assertEqual(len(vlist), len(ddt_vars[index])) - std_names = [x.get_prop_value('standard_name') for x in vlist] - for sname in ddt_vars[index]: - self.assertTrue(sname in std_names) - # end for - # end for - - def test_module_with_two_ddts_and_data(self): - """Test that a module containing two DDT definitions and a block of - module data is parsed and matched correctly.""" - # Setup - ddt_names = ['ddt1_t', 'ddt2_t'] - ddt_vars = [(), ('ddt_var_array_dimension', 'vars_array')] - - module_files = [os.path.join(self._sample_files_dir, - "ddt_data1_mod.meta")] - # Exercise - hname = 'host_name_ddt_data' - host_model = parse_host_model_files(module_files, hname, self._run_env) - # Verify the name of the host model - self.assertEqual(host_model.name, hname) - module_headers = host_model.metadata_tables() - self.assertEqual(len(module_headers), len(ddt_names) + 1) - # Verify header titles - for ddt_name in ddt_names: - self.assertTrue(ddt_name in module_headers) - # end for - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 3) - # Verify that each DDT was found and parsed - ddt_lib = host_model.ddt_lib - self.assertEqual(ddt_lib.name, f"{hname}_ddts_ddt_lib") - # Check DDT variables - for index, ddt_name in enumerate(ddt_names): - ddt_mod = ddt_lib[ddt_name] - self.assertEqual(ddt_mod.name, ddt_name) - vlist = ddt_mod.variable_list() - self.assertEqual(len(vlist), len(ddt_vars[index])) - std_names = [x.get_prop_value('standard_name') for x in vlist] - for sname in ddt_vars[index]: - self.assertTrue(sname in std_names) - # end for - # end for - # Verify header titles - self.assertTrue('ddt_data1_mod' in module_headers) - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 3) - std_names = [x.get_prop_value('standard_name') for x in vlist] - self.assertTrue('play_station' in std_names) - self.assertTrue('xbox' in std_names) - self.assertTrue('nintendo_switch' in std_names) - - def test_module_with_one_ddt_plus_undoc(self): - """Test that a module containing a one documented DDT definition - (i.e., with metadata) and one DDT without (i.e., no metadata) - is parsed and matched correctly.""" - # Setup - ddt_name = 'ddt2_t' - module_files = [os.path.join(self._sample_files_dir, "ddt1_plus.meta")] - # Exercise - hname = 'host_name_ddt1_plus' - host_model = parse_host_model_files(module_files, hname, self._run_env) - # Verify the name of the host model - self.assertEqual(host_model.name, hname) - module_headers = host_model.metadata_tables() - self.assertEqual(len(module_headers), 1) - # Verify header titles - self.assertTrue(ddt_name in module_headers) - # Verify host model variable list - vlist = host_model.variable_list() - self.assertEqual(len(vlist), 0) - # Verify that the DDT was found and parsed - ddt_lib = host_model.ddt_lib - self.assertEqual(ddt_lib.name, f"{hname}_ddts_ddt_lib") - # Check DDT variables - ddt_mod = ddt_lib[ddt_name] - self.assertEqual(ddt_mod.name, ddt_name) - vlist = ddt_mod.variable_list() - self.assertEqual(len(vlist), 2) - std_names = [x.get_prop_value('standard_name') for x in vlist] - self.assertTrue('ddt_var_array_dimension' in std_names) - self.assertTrue('vars_array' in std_names) - - def test_module_with_two_ddts_and_extra_var(self): - """Test that a module containing two DDT definitions is parsed and - a useful error message is produced if the DDT metadata has an - extra variable.""" - # Setup - ddt_names = ['ddt1_t', 'ddt2_t'] - ddt_vars = [(), ('ddt_var_array_dimension', 'vars_array')] - - module_files = [os.path.join(self._sample_files_dir, - "ddt2_extra_var.meta")] - # Exercise - hname = 'host_name_ddt_extra_var' - with self.assertRaises(CCPPError) as context: - host_model = parse_host_model_files(module_files, hname, - self._run_env) - # end with - # Check error messages - except_str = str(context.exception) - emsgs = ["Variable mismatch in ddt2_t, variables missing from Fortran ddt.", - "No Fortran variable for bogus in ddt2_t", - "2 errors found comparing"] - for emsg in emsgs: - self.assertTrue(emsg in except_str) - # end for - - def test_mismatch_hdim(self): - """Test correct use of loop variables (horizontal dimensions) - in host metadata.""" - # Setup - module_files = [os.path.join(self._sample_files_dir, "mismatch_hdim_mod.meta")] - # Exercise - hname = 'host_name_mismatch_hdim' - with self.assertRaises(CCPPError) as context: - _ = parse_host_model_files(module_files, hname, self._run_env) - # end with - # Check error messages - except_str = str(context.exception) - emsgs = ["Invalid horizontal dimension, 'horizontal_loop_extent'", - "Invalid horizontal dimension, 'horizontal_loop_end'"] - for emsg in emsgs: - self.assertTrue(emsg in except_str) - # end for - -if __name__ == "__main__": - unittest.main() - diff --git a/test/unit_tests/test_metadata_scheme_file.py b/test/unit_tests/test_metadata_scheme_file.py deleted file mode 100644 index ef24742c..00000000 --- a/test/unit_tests/test_metadata_scheme_file.py +++ /dev/null @@ -1,357 +0,0 @@ -#! /usr/bin/env python3 - -""" ------------------------------------------------------------------------ - Description: capgen needs to compare a metadata header against the - associated CCPP Fortran interface routine. This set of - tests is testing the parse_scheme_files function in - ccpp_capgen.py which performs the operations in the first - bullet below. Each test calls this function. - - * This script contains unit tests that do the following: - 1) Read a metadata file (to collect the metadata headers) - 2) Read the associated CCPP Fortran scheme file (to - collect Fortran interfaces) - 3) Compare the metadata header against the Fortran - - * Tests include: - - Correctly identify when the metadata file matches the - Fortran, even if the routines are not in the same order - - Correctly detect a missing metadata header - - Correctly detect a missing Fortran interface - - Correctly detect a mismatch between the metadata and the - Fortran - - Correctly detect invalid Fortran subroutine statements, - invalid dummy argument statements, and invalid Fortran - between the subroutine statement and the end of the - variable declaration block. - - Correctly interpret Fortran with preprocessor logic - which affects the subroutine statement and/or the dummy - argument statements - - Correctly interpret Fortran with preprocessor logic - which affects the subroutine statement and/or the dummy - argument statements resulting in a mismatch between the - metadata header and the Fortran - - Correctly interpret Fortran with preprocessor logic - which affects the subroutine statement and/or the dummy - argument statements resulting in incorrect Fortran - - Test correct use of loop variables (horizontal dimensions) - in scheme metadata. The allowed values depend on the phase - (run phase or not) - - Assumptions: - - Command line arguments: none - - Usage: python3 test_metadata_scheme_file.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import logging -import unittest - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_SCRIPTS_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, - os.pardir, "scripts")) -if not os.path.exists(_SCRIPTS_DIR): - raise ImportError("Cannot find scripts directory") - -sys.path.append(_SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from ccpp_capgen import parse_scheme_files -from framework_env import CCPPFrameworkEnv -from parse_tools import CCPPError -# pylint: enable=wrong-import-position - -class MetadataHeaderTestCase(unittest.TestCase): - """Unit tests for parse_scheme_files""" - - def setUp(self): - """Setup important directories and logging""" - self._sample_files_dir = os.path.join(_TEST_DIR, "sample_scheme_files") - self._host_files_dir = os.path.join(_TEST_DIR, "sample_host_files") - logger = logging.getLogger(self.__class__.__name__) - self._run_env = CCPPFrameworkEnv(logger, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - self._run_env_ccpp = CCPPFrameworkEnv(logger, - ndict={'host_files':'', - 'scheme_files':'', - 'suites':'', - 'preproc_directives': - 'CCPP=1'}) - self._run_env_ccpp2 = CCPPFrameworkEnv(logger, - ndict={'host_files':'', - 'scheme_files':'', - 'suites':'', - 'preproc_directives': - 'CCPP=2'}) - - def test_good_scheme_file(self): - """Test that good metadata file matches the Fortran, - with routines in the same order """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "temp_adjust.meta")] - # Exercise - scheme_headers, table_dict = parse_scheme_files(scheme_files, - self._run_env, - skip_ddt_check=True) - # Verify size of returned list equals number of scheme headers - # in the test file and that header (subroutine) names are - # 'temp_adjust_[register,init,run,finalize]' - self.assertEqual(len(scheme_headers), 4) - # Verify header titles - titles = [elem.title for elem in scheme_headers] - self.assertTrue('temp_adjust_register' in titles) - self.assertTrue('temp_adjust_init' in titles) - self.assertTrue('temp_adjust_run' in titles) - self.assertTrue('temp_adjust_finalize' in titles) - # Verify size and name of table_dict matches scheme name - self.assertEqual(len(table_dict), 1) - self.assertTrue('temp_adjust' in table_dict) - - def test_reordered_scheme_file(self): - """Test that metadata file matches the Fortran when the - routines are not in the same order """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, "reorder.meta")] - # Exercise - scheme_headers, table_dict = parse_scheme_files(scheme_files, - self._run_env) - # Verify size of returned list equals number of scheme headers - # in the test file and that header (subroutine) names are - # 'reorder_[init,run,finalize]' - self.assertEqual(len(scheme_headers), 3) - # Verify header titles - titles = [elem.title for elem in scheme_headers] - self.assertTrue('reorder_init' in titles) - self.assertTrue('reorder_run' in titles) - self.assertTrue('reorder_finalize' in titles) - # Verify size and name of table_dict matches scheme name - self.assertEqual(len(table_dict), 1) - self.assertTrue('reorder' in table_dict) - - def test_missing_metadata_header(self): - """Test that a missing metadata header (aka arg table) is - corretly detected """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "missing_arg_table.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify correct error message returned - emsg = "No matching metadata header found for missing_arg_table_run in" - self.assertTrue(emsg in str(context.exception)) - - def test_missing_fortran_header(self): - """Test that a missing fortran header is corretly detected """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "missing_fort_header.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify correct error message returned - emsg = "No matching Fortran routine found for missing_fort_header_run in" - self.assertTrue(emsg in str(context.exception)) - - def test_mismatch_intent(self): - """Test that differing intent, kind, rank, and type between - metadata and fortran is corretly detected """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "mismatch_intent.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify 4 correct error messages returned - emsg = "intent mismatch (in != inout) in mismatch_intent_run, at" - self.assertTrue(emsg in str(context.exception)) - emsg = "kind mismatch (kind_fizz != kind_phys) in mismatch_intent_run, at" - self.assertTrue(emsg in str(context.exception)) - emsg = "rank mismatch in mismatch_intent_run/potential_temperature (0 != 1), at" - self.assertTrue(emsg in str(context.exception)) - emsg = "type mismatch (integer != real) in mismatch_intent_run, at" - self.assertTrue(emsg in str(context.exception)) - self.assertTrue("4 errors found comparing" in str(context.exception)) - - def test_invalid_subr_stmnt(self): - """Test that invalid Fortran subroutine statements are correctly - detected """ - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "invalid_subr_stmnt.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify correct error message returned - self.assertTrue("Invalid dummy argument, 'errmsg', at" - in str(context.exception)) - - def test_invalid_dummy_arg(self): - """Test that invalid dummy argument statements are correctly detected""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "invalid_dummy_arg.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify correct error message returned - emsg = "Invalid dummy argument, 'woohoo', at" - self.assertTrue(emsg in str(context.exception)) - - def test_ccpp_notset_var_missing_in_meta(self): - """Test for correct detection of a variable that REMAINS in the - subroutine argument list - (due to an undefined pre-processor directive: #ifndef CCPP), - BUT IS NOT PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPnotset_var_missing_in_meta.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env) - # Verify 2 correct error messages returned - emsg = "Variable mismatch in CCPPnotset_var_missing_in_meta_run, " + \ - "variables missing from metadata header." - self.assertTrue(emsg in str(context.exception)) - emsg = "Fortran variable, bar, not in metadata" - self.assertTrue(emsg in str(context.exception)) - self.assertTrue("2 errors found comparing" in str(context.exception)) - - def test_ccpp_eq1_var_missing_in_fort(self): - """Test for correct detection of a variable that IS REMOVED the - subroutine argument list - (due to a pre-processor directive: #ifndef CCPP), - but IS PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPeq1_var_missing_in_fort.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - parse_scheme_files(scheme_files, self._run_env_ccpp) - # Verify 2 correct error messages returned - emsg = "Variable mismatch in CCPPeq1_var_missing_in_fort_run, " + \ - "variables missing from Fortran scheme." - self.assertTrue(emsg in str(context.exception)) - emsg = "Variable mismatch in CCPPeq1_var_missing_in_fort_run, " + \ - "no Fortran variable bar." - self.assertTrue(emsg in str(context.exception)) - self.assertTrue("2 errors found comparing" in str(context.exception)) - - def test_ccpp_eq1_var_in_fort_meta(self): - """Test positive case of a variable that IS PRESENT the - subroutine argument list - (due to a pre-processor directive: #ifdef CCPP), - and IS PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPeq1_var_in_fort_meta.meta")] - # Exercise - scheme_headers, table_dict = parse_scheme_files(scheme_files, - self._run_env_ccpp) - # Verify size of returned list equals number of scheme headers in - # the test file (1) and that header (subroutine) name is - # 'CCPPeq1_var_in_fort_meta_run' - self.assertEqual(len(scheme_headers), 1) - # Verify header titles - titles = [elem.title for elem in scheme_headers] - self.assertTrue("CCPPeq1_var_in_fort_meta_run" in titles) - - # Verify size and name of table_dict matches scheme name - self.assertEqual(len(table_dict), 1) - self.assertTrue("CCPPeq1_var_in_fort_meta" in table_dict) - - def test_ccpp_gt1_var_in_fort_meta(self): - """Test positive case of a variable that IS PRESENT the - subroutine argument list - (due to a pre-processor directive: #if CCPP > 1), - and IS PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPgt1_var_in_fort_meta.meta")] - # Exercise - # Set CCPP directive to > 1 - scheme_headers, table_dict = parse_scheme_files(scheme_files, - self._run_env_ccpp2) - # Verify size of returned list equals number of scheme headers - # in the test file (1) and that header (subroutine) name is - # 'CCPPgt1_var_in_fort_meta_init' - self.assertEqual(len(scheme_headers), 1) - # Verify header titles - titles = [elem.title for elem in scheme_headers] - self.assertTrue("CCPPgt1_var_in_fort_meta_init" in titles) - - # Verify size and name of table_dict matches scheme name - self.assertEqual(len(table_dict), 1) - self.assertTrue("CCPPgt1_var_in_fort_meta" in table_dict) - - def test_ccpp_gt1_var_in_fort_meta2(self): - """Test correct detection of a variable that - IS NOT PRESENT the subroutine argument list - (due to a pre-processor directive: #if CCPP > 1), - but IS PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPgt1_var_in_fort_meta.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - _, _ = parse_scheme_files(scheme_files, self._run_env_ccpp) - # Verify 2 correct error messages returned - emsg = "Variable mismatch in CCPPgt1_var_in_fort_meta_init, " + \ - "variables missing from Fortran scheme." - self.assertTrue(emsg in str(context.exception)) - emsg = "Variable mismatch in CCPPgt1_var_in_fort_meta_init, " + \ - "no Fortran variable bar." - self.assertTrue(emsg in str(context.exception)) - self.assertTrue("2 errors found comparing" in str(context.exception)) - - def test_ccpp_eq1_var_missing_in_meta(self): - """Test correct detection of a variable that - IS PRESENT the subroutine argument list - (due to a pre-processor directive: #ifdef CCPP), - and IS NOT PRESENT in meta file""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, - "CCPPeq1_var_missing_in_meta.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - _, _ = parse_scheme_files(scheme_files, self._run_env_ccpp) - # Verify 2 correct error messages returned - emsg = "Variable mismatch in CCPPeq1_var_missing_in_meta_finalize, "+ \ - "variables missing from metadata header." - self.assertTrue(emsg in str(context.exception)) - emsg = "Fortran variable, bar, not in metadata" - self.assertTrue(emsg in str(context.exception)) - self.assertTrue("2 errors found comparing" in str(context.exception)) - - def test_scheme_ddt_only(self): - """Test correct detection of a "scheme" file which contains only - DDT definitions""" - # Setup - scheme_files = [os.path.join(self._host_files_dir, "ddt2.meta")] - # Exercise - scheme_headers, table_dict = parse_scheme_files(scheme_files, - self._run_env_ccpp) - - def test_mismatch_hdim(self): - """Test correct use of loop variables (horizontal dimensions) - in scheme metadata. The allowed values depend on the phase - (run phase or not)""" - # Setup - scheme_files = [os.path.join(self._sample_files_dir, "mismatch_hdim.meta")] - # Exercise - with self.assertRaises(CCPPError) as context: - _, _ = parse_scheme_files(scheme_files, self._run_env_ccpp) - # Verify 2 correct error messages returned - emsg = "Invalid horizontal dimension, 'horizontal_dimension'" - self.assertTrue(emsg in str(context.exception)) - emsg = "Invalid horizontal dimension, 'horizontal_loop_extent'" - self.assertTrue(emsg in str(context.exception)) - -if __name__ == "__main__": - unittest.main() diff --git a/test/unit_tests/test_metadata_table.py b/test/unit_tests/test_metadata_table.py deleted file mode 100644 index 74f598d0..00000000 --- a/test/unit_tests/test_metadata_table.py +++ /dev/null @@ -1,445 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for parse_metadata_file - in scripts file metadata_table.py - - Assumptions: - - Command line arguments: none - - Usage: python3 test_metadata_table.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import unittest - -UNIT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DIR = os.path.abspath(os.path.join(UNIT_TEST_DIR, os.pardir)) -SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, "scripts")) -SAMPLE_FILES_DIR = os.path.join(UNIT_TEST_DIR, "sample_files") - -if not os.path.exists(SCRIPTS_DIR): - raise ImportError("Cannot find scripts directory") - -sys.path.append(SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from metadata_table import parse_metadata_file, MetadataTable -from framework_env import CCPPFrameworkEnv -# pylint: enable=wrong-import-position - -class MetadataTableTestCase(unittest.TestCase): - - """Tests for `parse_metadata_file`.""" - - _DUMMY_RUN_ENV = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - - def test_good_host_file(self): - """Test that good host file test_host.meta returns one header named test_host""" - #Setup - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_host.meta") - #Exercise - result = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - #Verify that: - # no dependencies is returned as '' - # rel_path is returned as None - # size of returned list equals number of headers in the test file - # ccpp-table-properties name is 'test_host' - dependencies = result[0].dependencies - rel_path = result[0].dependencies_path - self.assertFalse('' in dependencies) - self.assertEqual(len(dependencies), 0) - self.assertIsNone(rel_path) - self.assertEqual(len(result), 1) - titles = [elem.table_name for elem in result] - self.assertIn('test_host', titles, msg="Header name 'test_host' is expected but not found") - - def test_good_multi_ccpp_arg_table(self): - """Test that good file with 4 ccpp-arg-table returns 4 headers""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_multi_ccpp_arg_tables.meta") - #Exercise - result = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - #Verify that size of returned list equals number of ccpp-table-properties in the test file - # ccpp-arg-tables are returned in result[0].sections() and result[1].sections() - self.assertEqual(len(result), 2) - - titles = list() - for table in result: - titles.extend([x.title for x in table.sections()]) - - self.assertIn('vmr_type', titles, msg="Header name 'vmr_type' is expected but not found") - self.assertIn('make_ddt_run', titles, msg="Header name 'make_ddt_run' is expected but not found") - self.assertIn('make_ddt_init', titles, msg="Header name 'make_ddt_init' is expected but not found") - self.assertIn('make_ddt_finalize', titles, msg="Header name 'make_ddt_finalize' is expected but not found") - - def test_bad_type_name(self): - """Test that `type = banana` returns expected error""" - #Setup - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_type_name.meta") - - #Exercise - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #Verify - #print("The exception is", context.exception) - self.assertTrue("Section type, 'banana', does not match table type, 'scheme'" in str(context.exception)) - - def test_double_header(self): - """Test that a duplicate header returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "double_header.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - self.assertTrue('table already contains \'test_host\'' in str(context.exception)) - - def test_bad_dimension(self): - """Test that `dimension = banana` returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_dimension.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - self.assertTrue('Invalid \'dimensions\' property value, \'' in str(context.exception)) - - def test_duplicate_variable(self): - """Test that a duplicate variable returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_duplicate_variable.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - self.assertTrue('Invalid (duplicate) standard name in temp_calc_adjust_run, defined at ' in str(context.exception)) - - def test_invalid_intent(self): - """Test that an invalid intent returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_invalid_intent.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - self.assertTrue('Invalid \'intent\' property value, \'banana\', at ' in str(context.exception)) - - def test_missing_intent(self): - """Test that a missing intent returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_missing_intent.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Required property, 'intent', missing, at " - self.assertTrue(emsg in str(context.exception)) - - def test_missing_units(self): - """Test that a missing units attribute returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_missing_units.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Required property, 'units', missing, at" - self.assertTrue(emsg in str(context.exception)) - - def test_missing_table_type(self): - """Test that a missing table type returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_missing_table_type.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid section type, 'None'" - self.assertTrue(emsg in str(context.exception)) - - def test_bad_table_type(self): - """Test that a mismatched table type returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_table_type.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Section type, 'host', does not match table type, 'scheme'" - self.assertTrue(emsg in str(context.exception)) - - def test_missing_table_name(self): - """Test that a missing table name returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_missing_table_name.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Section name, 'None', does not match table title, 'test_missing_table_name'" - self.assertTrue(emsg in str(context.exception)) - - def test_bad_table_key(self): - """Test that a bad table key returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_table_key.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid metadata table start property, 'banana', at " - self.assertTrue(emsg in str(context.exception)) - - def test_bad_line_split(self): - """Test that a bad split line with | returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_line_split.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid variable property syntax, \'\', at " - self.assertTrue(emsg in str(context.exception)) - - def test_unknown_ddt_type(self): - """Test that a DDT type = banana returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_unknown_ddt_type.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Unknown DDT type, banana, at " - self.assertTrue(emsg in str(context.exception)) - - def test_bad_var_property_name(self): - """Test that a ddt_type = None returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, "test_bad_var_property_name.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid variable property name, 'none', at " - self.assertTrue(emsg in str(context.exception)) - - def test_no_input(self): - """Test that no input returns expected error""" - with self.assertRaises(Exception) as context: - MetadataTable(self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "MetadataTable requires a name" - self.assertTrue(emsg in str(context.exception)) - - def test_no_table_type(self): - """Test that __init__ with table_type_in=None returns expected error""" - with self.assertRaises(Exception) as context: - MetadataTable(self._DUMMY_RUN_ENV, table_name_in="something", - table_type_in=None, dependencies=None, - dependencies_path=None, known_ddts=None, var_dict=None, - module=None, parse_object=None) - - #print("The exception is", context.exception) - emsg = "MetadataTable requires a table type" - self.assertTrue(emsg in str(context.exception)) - - def test_bad_header_type(self): - """Test that __init__ with table_type_in=banana returns expected error""" - with self.assertRaises(Exception) as context: - MetadataTable(self._DUMMY_RUN_ENV, table_name_in="something", - table_type_in="banana", dependencies=None, - dependencies_path=None, known_ddts=None, var_dict=None, - module=None, parse_object=None) - - #print("The exception is", context.exception) - emsg = "Invalid metadata arg table type, 'banana'" - self.assertTrue(emsg in str(context.exception)) - - def test_no_module(self): - """Test that __init__ with module=None returns expected error""" - with self.assertRaises(Exception) as context: - MetadataTable(self._DUMMY_RUN_ENV, table_name_in=None, - table_type_in=None, dependencies=None, - dependencies_path=None, known_ddts=None, var_dict=None, - module=None, parse_object=None) - - #print("The exception is", context.exception) - emsg = "MetadataTable requires a name" - self.assertTrue(emsg in str(context.exception)) - - def test_bad_1st_ccpp_arg_table(self): - """Test that first arg table named ccpp-farg-table returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_bad_1st_arg_table_header.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid variable property syntax, '[ccpp-farg-table]', at " - self.assertTrue(emsg in str(context.exception)) - - def test_bad_2nd_ccpp_arg_table(self): - """Test that second arg table named ccpp-farg-table returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_bad_2nd_arg_table_header.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid variable property syntax, '[ccpp-farg-table]', at " - self.assertTrue(emsg in str(context.exception)) - - def test_mismatch_section_table_title(self): - """Test that mismatched section name and table title - returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_mismatch_section_table_title.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Section name, 'test_host', does not match table title, 'banana', at " - self.assertTrue(emsg in str(context.exception)) - - def test_double_table_properties(self): - """Test that duplicate ccpp-table-properties returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "double_table_properties.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Duplicate metadata table, test_host, at " - self.assertTrue(emsg in str(context.exception)) - - def test_missing_table_properties(self): - """Test that a missing ccpp-table-properties returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "missing_table_properties.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid CCPP metadata line, '[ccpp-arg-table]', at " - self.assertTrue(emsg in str(context.exception)) - - def test_dependencies_path(self): - """Test that dependencies_path and dependencies from ccpp-table-properties are read in correctly""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_dependencies_path.meta") - - result = parse_metadata_file(filename, known_ddts, - self._DUMMY_RUN_ENV) - - dependencies = result[0].dependencies - rel_path = result[0].dependencies_path - titles = [elem.table_name for elem in result] - - self.assertEqual(len(dependencies), 4) - phys_dir = os.path.join(TEST_DIR, "ccpp", "physics", "physics") - self.assertIn(os.path.join(phys_dir, 'machine.F'), dependencies, \ - msg="Dependency 'machine.F' is expected but not found") - self.assertIn(os.path.join(phys_dir, 'physcons.F90'), dependencies, \ - msg="Dependency 'physcons.F90' is expected but not found") - self.assertIn(os.path.join(phys_dir, 'GFDL_parse_tracers.F90'), dependencies, \ - msg="Dependency 'GFDL_parse_tracers.F90' is expected but not found") - self.assertIn(os.path.join(phys_dir, 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90'), dependencies, \ - msg="Header name 'rte-rrtmgp/rrtmgp/mo_gas_optics_rrtmgp.F90' is expected but not found") - - self.assertIn(rel_path, "../../ccpp/physics/physics") - self.assertEqual(len(result), 1) - self.assertIn('test_host', titles, msg="Table name 'test_host' is expected but not found") - - def test_invalid_table_properties_type(self): - """Test that an invalid ccpp-table-properties type returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "test_invalid_table_properties_type.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - #print("The exception is", context.exception) - emsg = "Invalid metadata table type, 'banana', at " - self.assertTrue(emsg in str(context.exception)) - - def test_added_kind_spec(self): - """Test that adding a kind_spec to a Metadata table works as expected""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "good_kind_spec_table_properties.meta") - - # Test that we can parse the table with no error - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - # Test that the expected names are in the table - kind_types = self._DUMMY_RUN_ENV.kind_types() - self.assertEqual(len(kind_types), 3) - self.assertIn("kind_temp", kind_types) - self.assertEqual(self._DUMMY_RUN_ENV.kind_module("kind_temp"), "fmodule") - self.assertEqual(self._DUMMY_RUN_ENV.kind_spec("kind_temp"), "temp_r8") - self.assertIn("temp_i8", kind_types) - self.assertIn("kind_phys", kind_types) - - def test_bad_kind_spec(self): - """Test that adding a bad kind_spec to a Metadata table returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "bad_kind_spec_table_properties.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - emsg = "A Fortran kind name is required for 'temp_r8'" - self.assertTrue(emsg in str(context.exception), msg=str(context.exception)) - - def test_duplicate_kind_spec(self): - """Test that adding a duplicate kind_spec to a Metadata table returns expected error""" - known_ddts = list() - filename = os.path.join(SAMPLE_FILES_DIR, - "duplicate_kind_spec_table_properties.meta") - - with self.assertRaises(Exception) as context: - _ = parse_metadata_file(filename, known_ddts, self._DUMMY_RUN_ENV) - - emsg = ("'kind_temp = [kind_temp, fmodule]'is an invalid duplicate. " - "kind_temp is already '['temp_r8', 'fmodule'],") - self.assertTrue(emsg in str(context.exception), msg=str(context.exception)) - -if __name__ == "__main__": - unittest.main() diff --git a/test/unit_tests/test_sdf.py b/test/unit_tests/test_sdf.py deleted file mode 100644 index fbb2c1db..00000000 --- a/test/unit_tests/test_sdf.py +++ /dev/null @@ -1,569 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for parsing Suite Definition Files (SDFs) - in scripts/parse_tools/xml_tools.py - - Assumptions: - - Command line arguments: none - - Usage: python3 test_sdf.py # run the unit tests ------------------------------------------------------------------------ -""" - -import filecmp -import glob -import logging -import os -import sys -import unittest -import xml.etree.ElementTree as ET - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_SCRIPTS_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, - os.pardir, "scripts")) -_SAMPLE_FILES_DIR = os.path.join(_TEST_DIR, "sample_suite_files") -_PRE_TMP_DIR = os.path.join(_TEST_DIR, "tmp") -_TMP_DIR = os.path.join(_PRE_TMP_DIR, "suite_files") - -if not os.path.exists(_SCRIPTS_DIR): - raise ImportError(f"Cannot find scripts directory, {_SCRIPTS_DIR}") - -sys.path.append(_SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from parse_tools import init_log -from parse_tools import read_xml_file, validate_xml_file, write_xml_file -from parse_tools import find_schema_version, expand_nested_suites -# pylint: enable=wrong-import-position - -class SDFParseTestCase(unittest.TestCase): - - """Tests for `expand_nested_suites` and related functions.""" - - logger = None - - @classmethod - def setUpClass(cls): - """Clean output directory (tmp) before running tests""" - # Does "tmp" directory exist? If not then create it: - if not os.path.exists(_PRE_TMP_DIR): - os.makedirs(_PRE_TMP_DIR) - # end if - - # We need a logger - cls.logger = init_log(cls.__name__, level=logging.WARNING) - - #Does "tmp" directory exist? If not then create it: - # Ensure the "tmp/suite_files" directory exists and is empty - if os.path.exists(_TMP_DIR): - # Clear out all files: - for fpath in glob.iglob(os.path.join(_TMP_DIR, '*.*')): - if os.path.exists(fpath): - os.remove(fpath) - # End if - # End for - else: - os.makedirs(_TMP_DIR) - # end if - - # Run inherited setup method: - super().setUpClass() - - @classmethod - def get_logger(cls): - return cls.logger - - @classmethod - def compare_text(cls, name, txt1, txt2, typ): - """Compare two XML text or tail items (which may be None). - Return None if items match, otherwise, return an error string""" - res = None - if txt1 and txt2: - if txt1.strip() != txt2.strip(): - res = f"{name} {typ}, '{txt1}', does not match {typ}, '{txt2}'" - # end if - elif txt1: - res = f"{name} {typ} is missing from string2" - elif txt2: - res = f"{name} {typ} is missing from string1" - else: - res = None - # end if - return res - - @classmethod - def xml_diff(cls, xt1, xt2): - """ - Compares two xml etrees, xt1 and xt2 - Return None if the trees match, otherwise, return a difference string - """ - - diffs = [] - # First, compare the XML tags - if xt1.tag != xt2.tag: - diffs.append(f"Tags do not match: {xt1.tag} != {xt2.tag}") - else: - # Compare the attributes - for name, value in xt1.attrib.items(): - if name not in xt2.attrib: - diffs.append(f"xt1 attribute, {name}, is missing in xt2") - else: - xt2v = xt2.attrib.get(name) - if xt2v != value: - diffs.append(f"Attributes for {name} do not match: {str(value)} != {str(xt2v)}") - # end if - # end if - # end for - for name in xt2.attrib.keys(): - if name not in xt1.attrib: - diffs.append(f"xt2 attribute, {name}, is missing in xt1") - # end if - # end for - # Compare the text bodies (if any) - tdiff = cls.compare_text(xt1.tag, xt1.text, xt2.text, "text") - if tdiff: - diffs.append(tdiff) - # end if - tdiff = cls.compare_text(xt1.tag, xt1.tail, xt2.tail, "tail") - if tdiff: - diffs.append(tdiff) - # end if - # Compare children - if len(xt1) != len(xt2): - diffs.append(f"Number of children length differs, {len(xt1)} != {len(xt2)}") - else: - for child1, child2 in zip(xt1, xt2): - kid_diffs = cls.xml_diff(child1, child2) - if kid_diffs: - diffs.extend(kid_diffs) - # end if - # end for - # end if - # end if - return diffs - - def test_xml_diff(self): - """Test that xml_diff catches xml differences""" - root1 = ET.fromstring("item") - root2 = ET.fromstring("item") - diffs = self.xml_diff(root1, root2) - self.assertTrue(diffs) - self.assertEqual(len(diffs), 1) - self.assertTrue("Tags do not match" in diffs[0], - msg="tag1 should not match taga") - root1 = ET.fromstring("item1") - root2 = ET.fromstring("item2") - diffs = self.xml_diff(root1, root2) - self.assertTrue(diffs) - self.assertEqual(len(diffs), 1) - self.assertTrue("does not match" in diffs[0], - msg="item1 should not match item2") - root1 = ET.fromstring('item1') - root2 = ET.fromstring('item1') - diffs = self.xml_diff(root1, root2) - self.assertTrue(diffs) - self.assertEqual(len(diffs), 3) - self.assertTrue("Attributes for" in diffs[0] and "do not match" in diffs[0], - msg="attrib1 values should not match") - self.assertTrue("xt1 attribute, attrib2, is missing in xt2" in diffs[1], - msg=f"attrib2 is missing in root2") - self.assertTrue("xt2 attribute, attrib3, is missing in xt1" in diffs[2], - msg=f"attrib3 is missing in root1") - root1 = ET.fromstring('') - root2 = ET.fromstring('') - diffs = self.xml_diff(root1, root2) - self.assertEqual(len(diffs), 1) - self.assertTrue("Attributes for" in diffs[0] and "do not match" in diffs[0], - msg=f"attrib2 values should not match") - - def test_good_v1_sdf(self): - """Test that the parser recognizes a V1 SDF and parses it correctly - """ - num_tests = 2 - header = "Test of parsing of good V1 SDF" - for test_num in range(num_tests): - # Setup - testname = f"suite_good_v1_test{test_num+1:{0}{2}}" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 1) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res) - write_xml_file(xml_root, compare, logger) - amsg = f"{compare} does not exist" - self.assertTrue(os.path.exists(compare), msg=amsg) - _, compare_root = read_xml_file(compare, logger) - diffs = self.xml_diff(xml_root, compare_root) - lsep = '\n' - amsg = f"{source} does not match {compare}\n{lsep.join(diffs)}" - self.assertFalse(diffs, msg=amsg) - # end for - - def test_good_v2_sdf_01(self): - """Test that the parser recognizes a V2 SDF and parses and - expands it correctly. - Test the expansion of one group of a simple nested suite at group level. - """ - header = "Test of parsing of good V2 SDF" - # Setup - testname = "suite_good_v2_test01" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res) - amsg = f"{compare} does not exist" - self.assertTrue(os.path.exists(compare), msg=amsg) - _, xml_root = read_xml_file(source_exp, logger) - _, compare_root = read_xml_file(compare, logger) - diffs = self.xml_diff(xml_root, compare_root) - lsep = '\n' - amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" - self.assertFalse(diffs, msg=amsg) - - def test_good_v2_sdf_02(self): - """Test that the parser recognizes a V2 SDF and parses and - expands it correctly - Test the expansion of one group of a multiple group nested suite at group level. - """ - header = "Test of parsing of good V2 SDF" - # Setup - testname = "suite_good_v2_test02" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res) - amsg = f"{compare} does not exist" - self.assertTrue(os.path.exists(compare), msg=amsg) - _, xml_root = read_xml_file(source_exp, logger) - _, compare_root = read_xml_file(compare, logger) - diffs = self.xml_diff(xml_root, compare_root) - lsep = '\n' - amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" - self.assertFalse(diffs, msg=amsg) - - def test_good_v2_sdf_03(self): - """Test that the parser recognizes a V2 SDF and parses and - expands it correctly - Test expansion of two nested suites at group level plus a full nested suite at - suite level. - """ - header = "Test of parsing of good V2 SDF" - # Setup - testname = "suite_good_v2_test03" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res) - amsg = f"{compare} does not exist" - self.assertTrue(os.path.exists(compare), msg=amsg) - _, xml_root = read_xml_file(source_exp, logger) - _, compare_root = read_xml_file(compare, logger) - diffs = self.xml_diff(xml_root, compare_root) - lsep = '\n' - amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" - self.assertFalse(diffs, msg=amsg) - - def test_good_v2_sdf_04(self): - """Test that the parser recognizes a V2 SDF and parses and - expands it correctly - Test expansion of two nested suites at group level plus one group from a - nested suite at suite level. - """ - header = "Test of parsing of good V2 SDF" - # Setup - testname = "suite_good_v2_test04" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - source_exp = os.path.join(_SAMPLE_FILES_DIR, f"{testname}_exp.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res) - amsg = f"{compare} does not exist" - self.assertTrue(os.path.exists(compare), msg=amsg) - _, xml_root = read_xml_file(source_exp, logger) - _, compare_root = read_xml_file(compare, logger) - diffs = self.xml_diff(xml_root, compare_root) - lsep = '\n' - amsg = f"{source_exp} does not match {compare}\n{lsep.join(diffs)}" - self.assertFalse(diffs, msg=amsg) - - def test_bad_v2_suite_tag_sdf(self): - """Test that verification system recognizes a misplaced suite tag""" - header = "Test trapping of version attribute on a v2 suite tag" - # Setup - testname = f"suite_bad_v2_suite_tag" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - # Some versions of xmllint return an exit code 0 even if the - # validation fails. "Good" versions return an exit code /= 0, - # which then raises a CCPPError internally. The following - # logic handles the correct behavior (validation fails ==> - # exit code /= 0 ==> CCPPError). - try: - res = validate_xml_file(source, 'suite', schema_version, logger) - except Exception as e: - emsg = "Schemas validity error : Element 'suite': This element is not expected." - msg = str(e) - self.assertTrue(emsg in msg) - - def test_bad_v2_suite_duplicate_group1(self): - """Test that verification system recognizes a duplicate group name""" - header = "Test trapping of expanded suite duplicate group name" - # Setup - testname = f"suite_bad_v2_duplicate_group" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res, msg="Initial suite file should be valid") - with self.assertRaises(Exception) as context: - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - _ = validate_xml_file(compare, 'suite', schema_version, logger) - # end with - emsg = "Schemas validity error : Element 'group', attribute 'name': " + \ - "'group1' is not a valid value of the atomic type 'fortran_id_type_unique'" - fmsg = str(context.exception) - self.assertTrue(emsg in fmsg, msg=fmsg) - if not emsg in fmsg: - raise context - - def test_bad_v2_suite_missing_group(self): - """Test that verification system recognizes a missing group name""" - header = "Test trapping of expanded suite missing group name" - # Setup - testname = f"suite_missing_group" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res, msg="Initial suite file should be valid") - with self.assertRaises(Exception) as context: - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - # end with - emsg = "Nested suite subsuite_1, group group2, not found" - fmsg = str(context.exception) - self.assertTrue(emsg in fmsg, msg=fmsg) - - def test_bad_v2_suite_missing_file(self): - """Test that verification system recognizes a missing file argument""" - header = "Test trapping of missing file for nested suite" - # Setup - testname = f"suite_missing_file" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - # See note about different behavior of xmllint versions - # in test test_bad_v2_suite_tag_sdf above. - try: - res = validate_xml_file(source, 'suite', schema_version, logger) - except Exception as e: - emsg = "Schemas validity error : Element 'nested_suite': " + \ - "The attribute 'file' is required but missing." - msg = str(e) - self.assertTrue(emsg in msg) - - def test_bad_v2_suite_missing_loaded_suite(self): - """Test that verification system recognizes a missing suite loaded - from another file""" - header = "Test trapping of expanded suite missing a subsuite in a different file" - # Setup - testname = f"suite_missing_loaded_suite" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res, msg="Initial suite file should be valid") - with self.assertRaises(Exception) as context: - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - # end with - emsg = "Nested suite v12_suite, group main_group, not found in file" - fmsg = str(context.exception) - self.assertTrue(emsg in fmsg, msg=fmsg) - - def test_bad_v2_suite_infinite_group_recursion(self): - """Test that verification system recognizes infinite recursion when - including at the group level""" - header = "Test trapping of expanded suite with infinite group recursion" - # Setup - testname = f"suite_recurse_top1" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res, msg="Initial suite file should be valid") - with self.assertRaises(Exception) as context: - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - # end with - emsg = ("Exceeded number of iterations while expanding nested suites") - fmsg = str(context.exception) - self.assertTrue(emsg in fmsg, msg=fmsg) - - def test_bad_v2_suite_infinite_suite_recursion(self): - """Test that verification system recognizes infinite recursion when - including at the imported suite level""" - header = "Test trapping of expanded suite with infinite suite recursion" - # Setup - testname = f"suite_recurse_top2" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - compare = os.path.join(_TMP_DIR, f"{testname}_out.xml") - logger = self.get_logger() - # Exercise - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - res = validate_xml_file(source, 'suite', schema_version, logger) - self.assertTrue(res, msg="Initial suite file should be valid") - with self.assertRaises(Exception) as context: - expand_nested_suites(xml_root, _SAMPLE_FILES_DIR, logger=logger) - write_xml_file(xml_root, compare, logger) - # end with - emsg = ("Exceeded number of iterations while expanding nested suites") - fmsg = str(context.exception) - self.assertTrue(emsg in fmsg, msg=fmsg) - - def test_bad_schema_version(self): - """Test that verification system recognizes a bad version entry""" - num_tests = 4 - header = "Test trapping of invalid SDF version" - exc_strings = ["Format must be .", - "Format must be .", - "Major version must be at least 1", - "Minor version must be non-negative"] - for test_num in range(num_tests): - # Setup - testname = f"suite_bad_version{test_num+1:{0}{2}}" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - logger = self.get_logger() - # Exercise - with self.assertRaises(Exception) as context: - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - # end with - # Check exception for expected error messages - exp_str = str(context.exception) - self.assertTrue(exc_strings[test_num] in exp_str, - msg=f"Bad exception in test {test_num + 1}, '{exp_str}'") - # end for - - def test_missing_schema_version(self): - """Test that verification system recognizes a missing version num""" - header = "Test trapping of missing SDF version" - # Setup - testname = f"suite_missing_version" - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - logger = self.get_logger() - # Exercise - with self.assertRaises(Exception) as context: - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - # end with - # Check exception for expected error messages - self.assertTrue("version attribute required" in str(context.exception), - msg=f"Bad exception for missing suite version") - - def test_invalid_fortran_id(self): - """Test that verification system recognizes a bad Fortran ID entry""" - num_tests = 3 - header = "Test trapping of invalid Fortran ID" - tests = [ - "suite_invalid_scheme_fortran_id", - "suite_invalid_group_fortran_id", - "suite_invalid_suite_fortran_id", - ] - exc_strings = [ - "The value 'scheme-1' is not accepted by the pattern '[A-Za-z][A-Za-z0-9_]{0,63}", - "The value 'group-1' is not accepted by the pattern '[A-Za-z][A-Za-z0-9_]{0,63}", - "The value 'ver-test-suite' is not accepted by the pattern '[A-Za-z][A-Za-z0-9_]{0,63}", - ] - for test_num in range(num_tests): - # Setup - testname = tests[test_num] - source = os.path.join(_SAMPLE_FILES_DIR, f"{testname}.xml") - logger = self.get_logger() - _, xml_root = read_xml_file(source, logger) - schema_version = find_schema_version(xml_root) - self.assertEqual(schema_version[0], 2) - self.assertEqual(schema_version[1], 0) - # Exercise - with self.assertRaises(Exception) as context: - res = validate_xml_file(source, 'suite', schema_version, logger) - # end with - # Check exception for expected error messages - exp_str = str(context.exception) - self.assertTrue(exc_strings[test_num] in exp_str, - msg=f"Bad exception in test {test_num + 1}, '{exp_str}'") - # end for diff --git a/test/unit_tests/test_var_transforms.py b/test/unit_tests/test_var_transforms.py deleted file mode 100644 index 3d5e3477..00000000 --- a/test/unit_tests/test_var_transforms.py +++ /dev/null @@ -1,475 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for variable transforms involving - a VarCompatObj object - - Assumptions: - - Command line arguments: none - - Usage: python test_var_transforms.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import unittest - -TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, os.pardir, "scripts")) -SAMPLE_FILES_DIR = os.path.join(TEST_DIR, "sample_files") - -if not os.path.exists(SCRIPTS_DIR): - raise ImportError("Cannot find scripts directory") - -sys.path.append(SCRIPTS_DIR) - -# pylint: disable=wrong-import-position -from framework_env import CCPPFrameworkEnv -from metavar import Var -from parse_tools import ParseContext, ParseSource, ParseSyntaxError -from var_props import VarCompatObj -# pylint: enable=wrong-import-position - -class VarCompatTestCase(unittest.TestCase): - - """Tests for variable transforms.""" - - def _new_var(self, standard_name, units, dimensions, vtype, vkind=''): - """Create and return a new Var object with the requested properties""" - context = ParseContext(linenum=self.__linenum, filename="foo.meta") - source = ParseSource("foo", "host", context) - prop_dict = {'local_name' : f"foo{self.__linenum}", - 'standard_name' : standard_name, - 'units' : units, - 'dimensions' : f"({', '.join(dimensions)})", - 'type' : vtype, 'kind' : vkind} - self.__linenum += 5 - return Var(prop_dict, source, self.__run_env) - - def setUp(self): - """Setup variables for testing""" - self.__run_env = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'foo.meta', - 'suites':''}, - kind_types=["kind_phys=REAL64", - "kind_dyn=REAL32", - "kind_host=REAL64"]) - # For making variables unique - self.__linenum = 2 - # For assert messages - self.__inst_emsg = "Var.compatible returned a '{}', not a VarCompatObj" - - def test_equiv_vars(self): - """Test that equivalent variables are reported as equivalent""" - int_scalar1 = self._new_var('int_stdname1', 'm s-1', [], 'integer') - int_array1 = self._new_var('int_stdname2', 'm s-1', ['hdim'], - 'real', vkind='kind_phys') - int_array2 = self._new_var('int_stdname2', 'm s-1', ['hdim'], - 'real', vkind='kind_host') - int_array3 = self._new_var('int_stdname2', 'm s-1', ['hdim'], - 'real', vkind='REAL64') - compat = int_scalar1.compatible(int_scalar1, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertTrue(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - compat = int_array1.compatible(int_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertTrue(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - compat = int_array3.compatible(int_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertTrue(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - - def test_incompatible_vars(self): - """Test that incompatible variables are reported correctly""" - int_scalar1 = self._new_var('int_stdname1', 'm s-1', [], 'integer') - int_scalar2 = self._new_var('int_stdname2', 'm s-1', [], 'integer') - int_array1 = self._new_var('int_stdname1', 'm s-1', ['hdim'], - 'integer') - real_array1 = self._new_var('int_stdname1', 'm s-1', ['hdim'], - 'real', vkind='kind_phys') - # Array and scalar - compat = int_scalar1.compatible(int_array1, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertFalse(compat.compat) - self.assertEqual(compat.incompat_reason, 'dimensions') - # Variables with different standard names - compat = int_scalar1.compatible(int_scalar2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertFalse(compat.compat) - self.assertEqual(compat.incompat_reason, 'standard names') - # Variables with different types - compat = int_array1.compatible(real_array1, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertFalse(compat.compat) - self.assertEqual(compat.incompat_reason, 'types') - - def test_valid_unit_change(self): - """Test that valid unit changes are detected""" - real_scalar1 = self._new_var('real_stdname1', 'm', [], - 'real', vkind='kind_phys') - real_scalar2 = self._new_var('real_stdname1', 'mm', [], - 'real', vkind='kind_phys') - compat = real_scalar1.compatible(real_scalar2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat.equiv) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertTrue(compat.has_unit_transforms) - - real_array1 = self._new_var('real_stdname1', 'm s-1', ['hdim', 'vdim'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('real_stdname1', 'km h-1', ['hdim', 'vdim'], - 'real', vkind='kind_phys') - compat = real_scalar1.compatible(real_scalar2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat.equiv) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertTrue(compat.has_unit_transforms) - - def test_unsupported_unit_change(self): - """Test that unsupported unit changes are detected""" - real_scalar1 = self._new_var('real_stdname1', 'min', [], - 'real', vkind='kind_phys') - real_scalar2 = self._new_var('real_stdname1', 'd', [], - 'real', vkind='kind_phys') - char_nounit1 = self._new_var('char_stdname1', 'none', [], - 'character', vkind='len=256') - char_nounit2 = self._new_var('char_stdname1', '1', [], - 'character', vkind='len=256') - with self.assertRaises(ParseSyntaxError) as context: - compat = real_scalar1.compatible(real_scalar2, self.__run_env) - # end with - #Test bad conversion for real time variables - #Verify correct error message returned - emsg = "Unsupported unit conversion, 'min' to 'd' for 'real_stdname1'" - self.assertTrue(emsg in str(context.exception)) - #Test bad conversion for unitless variables - with self.assertRaises(ParseSyntaxError) as context: - compat = char_nounit1.compatible(char_nounit2, self.__run_env) - # end with - #Verify correct error message returned - emsg = "Unsupported unit conversion, 'none' to '1' for 'char_stdname1'" - self.assertTrue(emsg in str(context.exception)) - - def test_valid_kind_change(self): - """Test that valid kind changes are detected""" - real_scalar1 = self._new_var('real_stdname1', 'mm', [], - 'real', vkind='kind_phys') - real_scalar2 = self._new_var('real_stdname1', 'mm', [], - 'real', vkind='kind_dyn') - compat = real_scalar1.compatible(real_scalar2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertTrue(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - - real_scalar1 = self._new_var('real_stdname1', 'm', [], - 'real', vkind='kind_phys') - real_scalar2 = self._new_var('real_stdname1', 'mm', [], - 'real', vkind='REAL32') - compat = real_scalar1.compatible(real_scalar2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertTrue(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertTrue(compat.has_unit_transforms) - - def test_valid_dim_change(self): - """Test that valid dimension changes are detected""" - real_array1 = self._new_var('real_stdname1', 'C', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('real_stdname1', 'K', - ['ccpp_constant_one:horizontal_loop_extent', - 'vertical_layer_dimension'], - 'real', vkind='kind_dyn') - compat = real_array1.compatible(real_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertTrue(compat.has_kind_transforms) - self.assertTrue(compat.has_dim_transforms) - self.assertTrue(compat.has_unit_transforms) - - real_array1 = self._new_var('real_stdname1', 'C', - ['ccpp_constant_one:horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('real_stdname1', 'K', - ['vertical_layer_dimension', - 'horizontal_loop_extent'], - 'real', vkind='kind_dyn') - compat = real_array1.compatible(real_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertFalse(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertTrue(compat.has_kind_transforms) - self.assertTrue(compat.has_dim_transforms) - self.assertTrue(compat.has_unit_transforms) - - def test_valid_dim_transforms(self): - """Test that valid variable transform code is created""" - real_array1 = self._new_var('real_stdname1', 'C', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('real_stdname1', 'C', - ['ccpp_constant_one:horizontal_loop_extent', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array3 = self._new_var('real_stdname1', 'K', - ['ccpp_constant_one:horizontal_loop_extent', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array4 = self._new_var('real_stdname1', 'K', - ['ccpp_constant_one:horizontal_loop_extent', - 'vertical_layer_dimension'], - 'real', vkind='kind_dyn') - real_array5 = self._new_var('real_stdname1', 'K', - ['vertical_layer_dimension', - 'ccpp_constant_one:horizontal_dimension'], - 'real', vkind='kind_phys') - v1_lname = real_array1.get_prop_value('local_name') - v2_lname = real_array2.get_prop_value('local_name') - v3_lname = real_array3.get_prop_value('local_name') - v4_lname = real_array4.get_prop_value('local_name') - v5_lname = real_array5.get_prop_value('local_name') - # Comparison between equivalent variables - compat = real_array1.compatible(real_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = rindices - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) - ind_str = ','.join(rindices) - expected = f"{v2_lname}({ind_str}) = {v1_lname}({ind_str})" - self.assertEqual(fwd_stmt, expected) - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) - expected = f"{v1_lname}({ind_str}) = {v2_lname}({ind_str})" - self.assertEqual(rev_stmt, expected) - - # Comparison between equivalent variables with loop correction - compat = real_array1.compatible(real_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("hind-col_start+1", "vind") - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - expected = f"{v2_lname}({lind_str}) = {v1_lname}({rind_str})" - self.assertEqual(fwd_stmt, expected) - lindices = ("hind+col_start-1", "vind") - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) - lind_str = ','.join(lindices) - expected = f"{v1_lname}({lind_str}) = {v2_lname}({rind_str})" - self.assertEqual(rev_stmt, expected) - - # Comparison between equivalent variables with loop correction - # plus vertical flip - compat = real_array1.compatible(real_array2, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("hind-col_start+1", "pver-vind+1") - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - expected = f"{v2_lname}({lind_str}) = {v1_lname}({rind_str})" - self.assertEqual(fwd_stmt, expected) - lindices = ("hind+col_start-1", "pver-vind+1") - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) - lind_str = ','.join(lindices) - expected = f"{v1_lname}({lind_str}) = {v2_lname}({rind_str})" - self.assertEqual(rev_stmt, expected) - - # Comparison between variables with different units - compat = real_array1.compatible(real_array3, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("hind-col_start+1", "vind") - conv = f"273.15_{real_array1.get_prop_value('kind')}" - fwd_stmt = compat.forward_transform(v3_lname, v1_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - expected = f"{v3_lname}({lind_str}) = {v1_lname}({rind_str})+{conv}" - self.assertEqual(fwd_stmt, expected) - lindices = ("hind+col_start-1", "vind") - rev_stmt = compat.reverse_transform(v1_lname, v3_lname, rindices, lindices) - lind_str = ','.join(lindices) - conv = f"273.15_{real_array2.get_prop_value('kind')}" - expected = f"{v1_lname}({lind_str}) = {v3_lname}({rind_str})-{conv}" - self.assertEqual(rev_stmt, expected) - - # Comparison between variables with different kind - compat = real_array4.compatible(real_array3, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("hind", "vind") - fwd_stmt = compat.forward_transform(v4_lname, v3_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - rkind = real_array3.get_prop_value('kind') - expected = f"{v4_lname}({lind_str}) = real({v3_lname}({rind_str}), {rkind})" - self.assertEqual(fwd_stmt, expected) - lindices = ("hind", "vind") - rev_stmt = compat.reverse_transform(v3_lname, v4_lname, rindices, lindices) - lind_str = ','.join(lindices) - rkind = real_array4.get_prop_value('kind') - expected = f"{v3_lname}({lind_str}) = real({v4_lname}({rind_str}), {rkind})" - self.assertEqual(rev_stmt, expected) - - # Comparison between variables with different units and kind - compat = real_array1.compatible(real_array4, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("hind-col_start+1", "vind") - rkind = real_array4.get_prop_value('kind') - conv = f"273.15_{rkind}" - fwd_stmt = compat.forward_transform(v2_lname, v1_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - expected = f"{v2_lname}({lind_str}) = real({v1_lname}({rind_str}), {rkind})+{conv}" - self.assertEqual(fwd_stmt, expected) - lindices = ("hind+col_start-1", "vind") - rev_stmt = compat.reverse_transform(v1_lname, v2_lname, rindices, lindices) - lind_str = ','.join(lindices) - rkind = real_array1.get_prop_value('kind') - conv = f"273.15_{rkind}" - expected = f"{v1_lname}({lind_str}) = real({v2_lname}({rind_str}), {rkind})-{conv}" - self.assertEqual(rev_stmt, expected) - - # Comparison between variables with different dimension ordering - # and horizontal loop adjustment and vertical flip - compat = real_array5.compatible(real_array3, self.__run_env) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - rindices = ("hind", "vind") - lindices = ("pver-vind+1", "hind-col_start+1") - fwd_stmt = compat.forward_transform(v4_lname, v5_lname, rindices, lindices) - lind_str = ','.join(lindices) - rind_str = ','.join(rindices) - rkind = real_array3.get_prop_value('kind') - expected = f"{v4_lname}({lind_str}) = {v5_lname}({rind_str})" - self.assertEqual(fwd_stmt, expected) - rindices = ("vind", "hind") - rind_str = ','.join(rindices) - lindices = ("hind+col_start-1", "pver-vind+1") - rev_stmt = compat.reverse_transform(v5_lname, v4_lname, rindices, lindices) - lind_str = ','.join(lindices) - rkind = real_array4.get_prop_value('kind') - expected = f"{v5_lname}({lind_str}) = {v4_lname}({rind_str})" - self.assertEqual(rev_stmt, expected) - - def test_compatible_tendency_variable(self): - """Test that a given tendency variable is compatible with - its corresponding state variable""" - real_array1 = self._new_var('real_stdname1', 'C', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('tendency_of_real_stdname1', 'C s-1', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - compat = real_array2.compatible(real_array1, self.__run_env, is_tend=True) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertTrue(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - - def test_compatible_tendency_variable_equivalent_units(self): - """Test that a given tendency variable is compatible with - its corresponding state variable""" - real_array1 = self._new_var('real_stdname1', 'V A', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('tendency_of_real_stdname1', 'W s-1', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - compat = real_array2.compatible(real_array1, self.__run_env, is_tend=True) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - self.assertTrue(compat) - self.assertTrue(compat.compat) - self.assertEqual(compat.incompat_reason, '') - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - - def test_incompatible_tendency_variable(self): - """Test that the correct error is returned when a given tendency - variable has inconsistent units vs the state variable""" - real_array1 = self._new_var('real_stdname1', 'm', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - real_array2 = self._new_var('tendency_of_real_stdname1', 'cm s-1', - ['horizontal_dimension', - 'vertical_layer_dimension'], - 'real', vkind='kind_phys') - compat = real_array2.compatible(real_array1, self.__run_env, is_tend=True) - self.assertIsInstance(compat, VarCompatObj, - msg=self.__inst_emsg.format(type(compat))) - # Verify correct error message returned - emsg = "\nMismatch tendency variable units 'cm s-1' for variable 'tendency_of_real_stdname1'. No variable transforms supported for tendencies. Tendency units should be 'm s-1' to match state variable." - self.assertEqual(compat.incompat_reason, emsg) - self.assertFalse(compat.has_kind_transforms) - self.assertFalse(compat.has_dim_transforms) - self.assertFalse(compat.has_unit_transforms) - - -if __name__ == "__main__": - unittest.main() diff --git a/test/unit_tests/xmllint_wrapper/xmllint b/test/unit_tests/xmllint_wrapper/xmllint deleted file mode 100755 index 4f043c1c..00000000 --- a/test/unit_tests/xmllint_wrapper/xmllint +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env python3 - -# This is a wrapper around xmllint to emulate the "bad behavior" -# of some xmllint versions that return an exit code 0 even if the -# validation fails. It requires the full path to the "real" xmllint -# executable to be defined as environment variable XMLLINT_REAL - -import os -import shutil -import subprocess -import sys - -xmllint = os.getenv("XMLLINT_REAL") -if not xmllint: - raise Exception("xmllint not found") - -cmd = [xmllint] + sys.argv[1:] -cproc = subprocess.run(cmd, check=False, capture_output=True) -if cproc.stdout: - sys.stdout.write(cproc.stdout.decode('utf-8', errors='replace').strip()+"\n") -if cproc.stderr: - sys.stderr.write(cproc.stderr.decode('utf-8', errors='replace').strip()+"\n") -# Exit with an exit code of zero no matter what -sys.exit(0) diff --git a/test/utils/CMakeLists.txt b/test/utils/CMakeLists.txt deleted file mode 100644 index dee888ca..00000000 --- a/test/utils/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -add_library(test_utils STATIC test_utils.F90) diff --git a/test/utils/test_utils.F90 b/test/utils/test_utils.F90 deleted file mode 100644 index 3ae8d549..00000000 --- a/test/utils/test_utils.F90 +++ /dev/null @@ -1,88 +0,0 @@ -module test_utils - - public :: check_list - -contains - logical function check_list(test_list, chk_list, list_desc, suite_name) - ! Check a list () against its expected value () - - ! Dummy arguments - character(len=*), intent(in) :: test_list(:) - character(len=*), intent(in) :: chk_list(:) - character(len=*), intent(in) :: list_desc - character(len=*), optional, intent(in) :: suite_name - - ! Local variables - logical :: found - integer :: num_items - integer :: lindex, tindex - integer, allocatable :: check_unique(:) - character(len=2) :: sep - character(len=256) :: errmsg - - check_list = .true. - errmsg = '' - - ! Check the list size - num_items = size(chk_list) - if (size(test_list) /= num_items) then - write(errmsg, '(a,i0,2a)') 'ERROR: Found ', size(test_list), & - ' ', trim(list_desc) - if (present(suite_name)) then - write(errmsg(len_trim(errmsg) + 1:), '(2a)') ' for suite, ', & - trim(suite_name) - end if - write(errmsg(len_trim(errmsg) + 1:), '(a,i0)') ', should be ', num_items - write(6, *) trim(errmsg) - errmsg = '' - check_list = .false. - end if - - ! Now, check the list contents for 1-1 correspondence - if (check_list) then - allocate(check_unique(num_items)) - check_unique = -1 - do lindex = 1, num_items - found = .false. - do tindex = 1, num_items - if (trim(test_list(lindex)) == trim(chk_list(tindex))) then - check_unique(tindex) = lindex - found = .true. - exit - end if - end do - if (.not. found) then - check_list = .false. - write(errmsg, '(5a)') 'ERROR: ', trim(list_desc), ' item, ', & - trim(test_list(lindex)), ', was not found' - if (present(suite_name)) then - write(errmsg(len_trim(errmsg) + 1:), '(2a)') ' in suite, ', & - trim(suite_name) - end if - write(6, *) trim(errmsg) - errmsg = '' - end if - end do - if (check_list .and. any(check_unique < 0)) then - check_list = .false. - write(errmsg, '(3a)') 'ERROR: The following ', trim(list_desc), & - ' items were not found' - if (present(suite_name)) then - write(errmsg(len_trim(errmsg) + 1:), '(2a)') ' in suite, ', & - trim(suite_name) - end if - sep = '; ' - do lindex = 1, num_items - if (check_unique(lindex) < 0) then - write(errmsg(len_trim(errmsg) + 1:), '(2a)') sep, & - trim(chk_list(lindex)) - sep = ', ' - end if - end do - write(6, *) trim(errmsg) - errmsg = '' - end if - end if - - end function check_list -end module test_utils diff --git a/test/var_compatibility_test/CMakeLists.txt b/test/var_compatibility_test/CMakeLists.txt deleted file mode 100644 index 2938c2d0..00000000 --- a/test/var_compatibility_test/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ - -#------------------------------------------------------------------------------ -# -# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES -# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) -# -#------------------------------------------------------------------------------ -set(SCHEME_FILES "effr_calc" "effrs_calc" "effr_diag" "effr_pre" "effr_post" "rad_lw" "rad_sw") -set(HOST_FILES "module_rad_ddt" "test_host_data" "test_host_mod") -set(SUITE_FILES "var_compatibility_suite.xml") -# HOST is the name of the executable we will build. -# We assume there are files ${HOST}.meta and ${HOST}.F90 in CMAKE_SOURCE_DIR -set(HOST "test_host") - -# By default, generated caps go in ccpp subdir -set(CCPP_CAP_FILES "${CMAKE_CURRENT_BINARY_DIR}/ccpp") - -# Create lists for Fortran and meta data files from file names -list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) -list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_META_FILES) -list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE VAR_COMPATIBILITY_HOST_FORTRAN_FILES) -list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE VAR_COMPATIBILITY_HOST_METADATA_FILES) - -list(APPEND VAR_COMPATIBILITY_HOST_METADATA_FILES "${HOST}.meta") - -# Run ccpp_capgen -ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} - HOSTFILES ${VAR_COMPATIBILITY_HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_META_FILES} - SUITES ${SUITE_FILES} - HOST_NAME ${HOST} - OUTPUT_ROOT "${CCPP_CAP_FILES}") - -# Retrieve the list of Fortran files required for test host from datatable.xml and set to CCPP_CAPS_LIST -ccpp_datafile(DATATABLE "${CCPP_CAP_FILES}/datatable.xml" - REPORT_NAME "--ccpp-files") - -# Create test host library -add_library(VAR_COMPATIBILITY_TESTLIB OBJECT ${SCHEME_FORTRAN_FILES} - ${VAR_COMPATIBILITY_HOST_FORTRAN_FILES} - ${CCPP_CAPS_LIST}) - -# Setup test executable with needed dependencies -add_executable(var_compatibility_host_integration test_var_compatibility_integration.F90 ${HOST}.F90) -target_link_libraries(var_compatibility_host_integration PRIVATE VAR_COMPATIBILITY_TESTLIB test_utils) -target_include_directories(var_compatibility_host_integration PRIVATE "$") - -# Add executable to be called with ctest -add_test(NAME ctest_var_compatibility_host_integration COMMAND var_compatibility_host_integration) diff --git a/test/var_compatibility_test/README.md b/test/var_compatibility_test/README.md deleted file mode 100644 index d5573f3a..00000000 --- a/test/var_compatibility_test/README.md +++ /dev/null @@ -1,23 +0,0 @@ -# Variable Compatibility Test - -Tests the variable compatibility object (`VarCompatObj`): -- Unit conversions (forward & reverse) -- Vertical array flipping (`top_at_one=true`) -- Kind conversions (`kind_phys <-> 8`) -- And various combinations thereof of the above cases -- Also tests subcycles: - - Nested subcycles - - A subcycle with dynamic iteration length (defined by a standard name) and a subcycle with fixed/integer iteration length - - Multiple subcycles with same standard name defining the iteration length - - Nested subcycles with the same iteration length - -## Building/Running - -To explicitly build/run the variable compatibility test host, run: - -```bash -$ cmake --fresh -S -B -DCCPP_RUN_VAR_COMPATIBILITY_TEST=ON -$ cd -$ make -$ ctest -``` diff --git a/test/var_compatibility_test/effr_calc.F90 b/test/var_compatibility_test/effr_calc.F90 deleted file mode 100644 index b8fc43ed..00000000 --- a/test/var_compatibility_test/effr_calc.F90 +++ /dev/null @@ -1,84 +0,0 @@ -!Test unit conversions for intent in, inout, out variables -! - -module effr_calc - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: effr_calc_run, effr_calc_init - -contains - !> \section arg_table_effr_calc_init Argument Table - !! \htmlinclude arg_table_effr_calc_init.html - !! - subroutine effr_calc_init(scheme_order, errmsg, errflg) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(inout) :: scheme_order - - errmsg = '' - errflg = 0 - - if (scheme_order /= 2) then - errflg = 1 - errmsg = 'ERROR: effr_calc_init() needs to be called second' - return - else - scheme_order = scheme_order + 1 - end if - - end subroutine effr_calc_init - - !> \section arg_table_effr_calc_run Argument Table - !! \htmlinclude arg_table_effr_calc_run.html - !! - subroutine effr_calc_run(ncol, nlev, effrr_in, effrg_in, ncg_in, nci_out, & - effrl_inout, effri_out, effrs_inout, ncl_out, & - has_graupel, scalar_var, tke_inout, tke2_inout, & - errmsg, errflg) - - integer, intent(in) :: ncol - integer, intent(in) :: nlev - real(kind=kind_phys), intent(in) :: effrr_in(:, :) - real(kind=kind_phys), intent(in), optional :: effrg_in(:, :) - real(kind=kind_phys), intent(in), optional :: ncg_in(:, :) - real(kind=kind_phys), intent(out), optional :: nci_out(:, :) - real(kind=kind_phys), intent(inout) :: effrl_inout(:, :) - real(kind=kind_phys), intent(out), optional :: effri_out(:, :) - real(kind=8), intent(inout) :: effrs_inout(:, :) - logical, intent(in) :: has_graupel - real(kind=kind_phys), intent(inout) :: scalar_var - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - real(kind=kind_phys), intent(out), optional :: ncl_out(:, :) - real(kind=kind_phys), intent(inout) :: tke_inout - real(kind=kind_phys), intent(inout) :: tke2_inout - - !---------------------------------------------------------------- - - real(kind=kind_phys), parameter :: re_qc_min = 2.5 ! microns - real(kind=kind_phys), parameter :: re_qc_max = 50. ! microns - real(kind=kind_phys), parameter :: re_qi_avg = 75. ! microns - real(kind=kind_phys) :: effrr_local(ncol, nlev) - real(kind=kind_phys) :: effrg_local(ncol, nlev) - real(kind=kind_phys) :: ncg_in_local(ncol, nlev) - real(kind=kind_phys) :: nci_out_local(ncol, nlev) - - errmsg = '' - errflg = 0 - - effrr_local = effrr_in - if (present(effrg_in)) effrg_local = effrg_in - if (present(ncg_in)) ncg_in_local = ncg_in - if (present(nci_out)) nci_out_local = nci_out - effrl_inout = min(max(effrl_inout, re_qc_min), re_qc_max) - if (present(effri_out)) effri_out = re_qi_avg - effrs_inout = effrs_inout + (10.0 / 6.0) ! in micrometer - scalar_var = 2.0 ! in km - - end subroutine effr_calc_run - -end module effr_calc diff --git a/test/var_compatibility_test/effr_calc.meta b/test/var_compatibility_test/effr_calc.meta deleted file mode 100644 index c3733f13..00000000 --- a/test/var_compatibility_test/effr_calc.meta +++ /dev/null @@ -1,163 +0,0 @@ -[ccpp-table-properties] - name = effr_calc - type = scheme - dependencies = -######################################################################## -[ccpp-arg-table] - name = effr_calc_init - type = scheme -[ scheme_order ] - standard_name = scheme_order_in_suite - long_name = scheme order in suite definition file - units = None - dimensions = () - type = integer - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -######################################################################## -[ccpp-arg-table] - name = effr_calc_run - type = scheme -[ ncol ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ nlev ] - standard_name = vertical_layer_dimension - type = integer - units = count - dimensions = () - intent = in -[effrr_in] - standard_name = effective_radius_of_stratiform_cloud_rain_particle - long_name = effective radius of cloud rain particle in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - top_at_one = True -[effrg_in] - standard_name = effective_radius_of_stratiform_cloud_graupel - long_name = effective radius of cloud graupel in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - optional = True -[ncg_in] - standard_name = cloud_graupel_number_concentration - long_name = number concentration of cloud graupel - units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - optional = True -[nci_out] - standard_name = cloud_ice_number_concentration - long_name = number concentration of cloud ice - units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = out - optional = True -[effrl_inout] - standard_name = effective_radius_of_stratiform_cloud_liquid_water_particle - long_name = effective radius of cloud liquid water particle in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout -[effri_out] - standard_name = effective_radius_of_stratiform_cloud_ice_particle - long_name = effective radius of cloud ice water particle in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = out - optional = True -[effrs_inout] - standard_name = effective_radius_of_stratiform_cloud_snow_particle - long_name = effective radius of cloud snow particle in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = 8 - intent = inout - top_at_one = True -[ncl_out] - standard_name = cloud_liquid_number_concentration - long_name = number concentration of cloud liquid - units = kg-1 - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = out - optional = True -[has_graupel] - standard_name = flag_indicating_cloud_microphysics_has_graupel - long_name = flag indicating that the cloud microphysics produces graupel - units = flag - dimensions = () - type = logical - intent = in -[ scalar_var ] - standard_name = scalar_variable_for_testing - long_name = scalar variable for testing - units = km - dimensions = () - type = real - kind = kind_phys - intent = inout -[ tke_inout ] - standard_name = turbulent_kinetic_energy - long_name = turbulent_kinetic_energy - units = m2 s-2 - dimensions = () - type = real - kind = kind_phys - intent = inout -[ tke2_inout ] - standard_name = turbulent_kinetic_energy2 - long_name = turbulent_kinetic_energy2 - units = m+2 s-2 - dimensions = () - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/var_compatibility_test/effr_diag.F90 b/test/var_compatibility_test/effr_diag.F90 deleted file mode 100644 index 75da29c7..00000000 --- a/test/var_compatibility_test/effr_diag.F90 +++ /dev/null @@ -1,68 +0,0 @@ -!Test unit conversions for intent in, inout, out variables -! - -module effr_diag - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: effr_diag_run, effr_diag_init - -contains - - !> \section arg_table_effr_diag_init Argument Table - !! \htmlinclude arg_table_effr_diag_init.html - !! - subroutine effr_diag_init(scheme_order, errmsg, errflg) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(inout) :: scheme_order - - errmsg = '' - errflg = 0 - - if (scheme_order /= 4) then - errflg = 1 - errmsg = 'ERROR: effr_diag_init() needs to be called fourth' - return - else - scheme_order = scheme_order + 1 - end if - - end subroutine effr_diag_init - - !> \section arg_table_effr_diag_run Argument Table - !! \htmlinclude arg_table_effr_diag_run.html - !! - subroutine effr_diag_run(effrr_in, scalar_var, errmsg, errflg) - - real(kind=kind_phys), intent(in) :: effrr_in(:, :) - integer, intent(in) :: scalar_var - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - real(kind=kind_phys) :: effrr_min, effrr_max - - errmsg = '' - errflg = 0 - - call cmp_effr_diag(effrr_in, effrr_min, effrr_max) - - if (scalar_var /= 380) then - errmsg = 'ERROR: effr_diag_run(): scalar_var should be 380' - errflg = 1 - end if - end subroutine effr_diag_run - - subroutine cmp_effr_diag(effr, effr_min, effr_max) - real(kind=kind_phys), intent(in) :: effr(:, :) - real(kind=kind_phys), intent(out) :: effr_min, effr_max - - ! Do some diagnostic calcualtions... - effr_min = minval(effr) - effr_max = maxval(effr) - - end subroutine cmp_effr_diag -end module effr_diag diff --git a/test/var_compatibility_test/effr_diag.meta b/test/var_compatibility_test/effr_diag.meta deleted file mode 100644 index 9e0e4fc2..00000000 --- a/test/var_compatibility_test/effr_diag.meta +++ /dev/null @@ -1,65 +0,0 @@ -[ccpp-table-properties] - name = effr_diag - type = scheme - dependencies = -######################################################################## -[ccpp-arg-table] - name = effr_diag_init - type = scheme -[ scheme_order ] - standard_name = scheme_order_in_suite - long_name = scheme order in suite definition file - units = None - dimensions = () - type = integer - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -######################################################################## -[ccpp-arg-table] - name = effr_diag_run - type = scheme -[effrr_in] - standard_name = effective_radius_of_stratiform_cloud_rain_particle - long_name = effective radius of cloud rain particle in micrometer - units = um - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - top_at_one = True -[ scalar_var ] - standard_name = scalar_variable_for_testing_c - long_name = unused scalar variable C - units = m - dimensions = () - type = integer - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/var_compatibility_test/effr_post.F90 b/test/var_compatibility_test/effr_post.F90 deleted file mode 100644 index 01357350..00000000 --- a/test/var_compatibility_test/effr_post.F90 +++ /dev/null @@ -1,61 +0,0 @@ -!Test unit conversions for intent in, inout, out variables -! - -module effr_post - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: effr_post_run, effr_post_init - -contains - - !> \section arg_table_effr_post_init Argument Table - !! \htmlinclude arg_table_effr_post_init.html - !! - subroutine effr_post_init(scheme_order, errmsg, errflg) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(inout) :: scheme_order - - errmsg = '' - errflg = 0 - - if (scheme_order /= 3) then - errflg = 1 - errmsg = 'ERROR: effr_post_init() needs to be called third' - return - else - scheme_order = scheme_order + 1 - end if - - end subroutine effr_post_init - - !> \section arg_table_effr_post_run Argument Table - !! \htmlinclude arg_table_effr_post_run.html - !! - subroutine effr_post_run(effrr_inout, scalar_var, errmsg, errflg) - - real(kind=kind_phys), intent(inout) :: effrr_inout(:, :) - real(kind=kind_phys), intent(in) :: scalar_var - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - real(kind=kind_phys) :: effrr_min, effrr_max - - errmsg = '' - errflg = 0 - - ! Do some post-processing on effrr... - effrr_inout(:, :) = effrr_inout(:, :) * 1._kind_phys - - if (scalar_var /= 1013.0) then - errmsg = 'ERROR: effr_post_run(): scalar_var should be 1013.0' - errflg = 1 - end if - - end subroutine effr_post_run - -end module effr_post diff --git a/test/var_compatibility_test/effr_post.meta b/test/var_compatibility_test/effr_post.meta deleted file mode 100644 index 721582a6..00000000 --- a/test/var_compatibility_test/effr_post.meta +++ /dev/null @@ -1,65 +0,0 @@ -[ccpp-table-properties] - name = effr_post - type = scheme - dependencies = -######################################################################## -[ccpp-arg-table] - name = effr_post_init - type = scheme -[ scheme_order ] - standard_name = scheme_order_in_suite - long_name = scheme order in suite definition file - units = None - dimensions = () - type = integer - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -######################################################################## -[ccpp-arg-table] - name = effr_post_run - type = scheme -[effrr_inout] - standard_name = effective_radius_of_stratiform_cloud_rain_particle - long_name = effective radius of cloud rain particle in micrometer - units = m - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout -[ scalar_var ] - standard_name = scalar_variable_for_testing_b - long_name = unused scalar variable B - units = m - dimensions = () - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/var_compatibility_test/effr_pre.F90 b/test/var_compatibility_test/effr_pre.F90 deleted file mode 100644 index a2fe2f5c..00000000 --- a/test/var_compatibility_test/effr_pre.F90 +++ /dev/null @@ -1,60 +0,0 @@ -!Test unit conversions for intent in, inout, out variables -! - -module mod_effr_pre - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: effr_pre_run, effr_pre_init - -contains - !> \section arg_table_effr_pre_init Argument Table - !! \htmlinclude arg_table_effr_pre_init.html - !! - subroutine effr_pre_init(scheme_order, errmsg, errflg) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(inout) :: scheme_order - - errmsg = '' - errflg = 0 - - if (scheme_order /= 1) then - errflg = 1 - errmsg = 'ERROR: effr_pre_init() needs to be called first' - return - else - scheme_order = scheme_order + 1 - end if - - end subroutine effr_pre_init - - !> \section arg_table_effr_pre_run Argument Table - !! \htmlinclude arg_table_effr_pre_run.html - !! - subroutine effr_pre_run(effrr_inout, scalar_var, errmsg, errflg) - - real(kind=kind_phys), intent(inout) :: effrr_inout(:, :) - real(kind=kind_phys), intent(in) :: scalar_var - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - !---------------------------------------------------------------- - real(kind=kind_phys) :: effrr_min, effrr_max - - errmsg = '' - errflg = 0 - - ! Do some pre-processing on effrr... - effrr_inout(:, :) = effrr_inout(:, :) * 1._kind_phys - - if (scalar_var /= 273.15) then - errmsg = 'ERROR: effr_pre_run(): scalar_var should be 273.15' - errflg = 1 - end if - - end subroutine effr_pre_run - -end module mod_effr_pre diff --git a/test/var_compatibility_test/effr_pre.meta b/test/var_compatibility_test/effr_pre.meta deleted file mode 100644 index 251b4175..00000000 --- a/test/var_compatibility_test/effr_pre.meta +++ /dev/null @@ -1,66 +0,0 @@ -[ccpp-table-properties] - name = effr_pre - type = scheme - module_name = mod_effr_pre - dependencies = -######################################################################## -[ccpp-arg-table] - name = effr_pre_init - type = scheme -[ scheme_order ] - standard_name = scheme_order_in_suite - long_name = scheme order in suite definition file - units = None - dimensions = () - type = integer - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -######################################################################## -[ccpp-arg-table] - name = effr_pre_run - type = scheme -[effrr_inout] - standard_name = effective_radius_of_stratiform_cloud_rain_particle - long_name = effective radius of cloud rain particle in micrometer - units = m - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout -[ scalar_var ] - standard_name = scalar_variable_for_testing_a - long_name = unused scalar variable A - units = m - dimensions = () - type = real - kind = kind_phys - intent = in -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/var_compatibility_test/effrs_calc.F90 b/test/var_compatibility_test/effrs_calc.F90 deleted file mode 100644 index 3aa8d196..00000000 --- a/test/var_compatibility_test/effrs_calc.F90 +++ /dev/null @@ -1,32 +0,0 @@ -!Test unit conversions for intent in, inout, out variables -! - -module effrs_calc - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: effrs_calc_run - -contains - !> \section arg_table_effrs_calc_run Argument Table - !! \htmlinclude arg_table_effrs_calc_run.html - !! - subroutine effrs_calc_run(effrs_inout, errmsg, errflg) - - real(kind=kind_phys), intent(inout) :: effrs_inout(:, :) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - !---------------------------------------------------------------- - - errmsg = '' - errflg = 0 - - effrs_inout = effrs_inout + (10.E-6_kind_phys / 3._kind_phys) ! in meters - - end subroutine effrs_calc_run - -end module effrs_calc diff --git a/test/var_compatibility_test/effrs_calc.meta b/test/var_compatibility_test/effrs_calc.meta deleted file mode 100644 index 9ce7b88e..00000000 --- a/test/var_compatibility_test/effrs_calc.meta +++ /dev/null @@ -1,25 +0,0 @@ -[ccpp-table-properties] - name = effrs_calc - type = scheme - -[ccpp-arg-table] - name = effrs_calc_run - type = scheme -[ effrs_inout ] - standard_name = effective_radius_of_stratiform_cloud_snow_particle - units = m - type = real | kind = kind_phys - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - units = none - type = character | kind = len=512 - dimensions = () - intent = out -[ errflg ] - standard_name = ccpp_error_code - units = 1 - type = integer - dimensions = () - intent = out diff --git a/test/var_compatibility_test/module_rad_ddt.F90 b/test/var_compatibility_test/module_rad_ddt.F90 deleted file mode 100644 index 6e992250..00000000 --- a/test/var_compatibility_test/module_rad_ddt.F90 +++ /dev/null @@ -1,23 +0,0 @@ -module mod_rad_ddt - use ccpp_kinds, only: kind_phys - implicit none - - public ty_rad_lw, ty_rad_sw - - !> \section arg_table_ty_rad_lw Argument Table - !! \htmlinclude arg_table_ty_rad_lw.html - !! - type ty_rad_lw - real(kind=kind_phys) :: sfc_up_lw - real(kind=kind_phys) :: sfc_down_lw - end type ty_rad_lw - - !> \section arg_table_ty_rad_sw Argument Table - !! \htmlinclude arg_table_ty_rad_sw.html - !! - type ty_rad_sw - real(kind=kind_phys), pointer :: sfc_up_sw(:) => null() - real(kind=kind_phys), pointer :: sfc_down_sw(:) => null() - end type ty_rad_sw - -end module mod_rad_ddt diff --git a/test/var_compatibility_test/module_rad_ddt.meta b/test/var_compatibility_test/module_rad_ddt.meta deleted file mode 100644 index c4792547..00000000 --- a/test/var_compatibility_test/module_rad_ddt.meta +++ /dev/null @@ -1,40 +0,0 @@ -[ccpp-table-properties] - name = ty_rad_lw - type = ddt - dependencies = - module_name = mod_rad_ddt -[ccpp-arg-table] - name = ty_rad_lw - type = ddt -[ sfc_up_lw ] - standard_name = surface_upwelling_longwave_radiation_flux - units = W m2 - dimensions = () - type = real - kind = kind_phys -[ sfc_down_lw ] - standard_name = surface_downwelling_longwave_radiation_flux - units = W m2 - dimensions = () - type = real - kind = kind_phys - -[ccpp-table-properties] - name = ty_rad_sw - type = ddt - module_name = mod_rad_ddt -[ccpp-arg-table] - name = ty_rad_sw - type = ddt -[ sfc_up_sw ] - standard_name = surface_upwelling_shortwave_radiation_flux - units = W m2 - dimensions = (horizontal_dimension) - type = real - kind = kind_phys -[ sfc_down_sw ] - standard_name = surface_downwelling_shortwave_radiation_flux - units = W m2 - dimensions = (horizontal_dimension) - type = real - kind = kind_phys diff --git a/test/var_compatibility_test/rad_lw.F90 b/test/var_compatibility_test/rad_lw.F90 deleted file mode 100644 index ded4861f..00000000 --- a/test/var_compatibility_test/rad_lw.F90 +++ /dev/null @@ -1,35 +0,0 @@ -module rad_lw - use ccpp_kinds, only: kind_phys - use mod_rad_ddt, only: ty_rad_lw - - implicit none - private - - public :: rad_lw_run - -contains - - !> \section arg_table_rad_lw_run Argument Table - !! \htmlinclude arg_table_rad_lw_run.html - !! - subroutine rad_lw_run(ncol, fluxlw, errmsg, errflg) - - integer, intent(in) :: ncol - type(ty_rad_lw), intent(inout) :: fluxlw(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! Locals - integer :: icol - - errmsg = '' - errflg = 0 - - do icol = 1, ncol - fluxlw(icol)%sfc_up_lw = 300._kind_phys - fluxlw(icol)%sfc_down_lw = 50._kind_phys - end do - - end subroutine rad_lw_run - -end module rad_lw diff --git a/test/var_compatibility_test/rad_lw.meta b/test/var_compatibility_test/rad_lw.meta deleted file mode 100644 index 883edf1b..00000000 --- a/test/var_compatibility_test/rad_lw.meta +++ /dev/null @@ -1,35 +0,0 @@ -[ccpp-table-properties] - name = rad_lw - type = scheme - dependencies = module_rad_ddt.F90 -[ccpp-arg-table] - name = rad_lw_run - type = scheme -[ ncol ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[fluxLW] - standard_name = longwave_radiation_fluxes - long_name = longwave radiation fluxes - units = W m-2 - dimensions = (horizontal_loop_extent) - type = ty_rad_lw - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/var_compatibility_test/rad_sw.F90 b/test/var_compatibility_test/rad_sw.F90 deleted file mode 100644 index 64756217..00000000 --- a/test/var_compatibility_test/rad_sw.F90 +++ /dev/null @@ -1,35 +0,0 @@ -module rad_sw - use ccpp_kinds, only: kind_phys - - implicit none - private - - public :: rad_sw_run - -contains - - !> \section arg_table_rad_sw_run Argument Table - !! \htmlinclude arg_table_rad_sw_run.html - !! - subroutine rad_sw_run(ncol, sfc_up_sw, sfc_down_sw, errmsg, errflg) - - integer, intent(in) :: ncol - real(kind=kind_phys), intent(inout) :: sfc_up_sw(:) - real(kind=kind_phys), intent(inout) :: sfc_down_sw(:) - character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg - - ! Locals - integer :: icol - - errmsg = '' - errflg = 0 - - do icol = 1, ncol - sfc_up_sw(icol) = 100._kind_phys - sfc_down_sw(icol) = 400._kind_phys - end do - - end subroutine rad_sw_run - -end module rad_sw diff --git a/test/var_compatibility_test/rad_sw.meta b/test/var_compatibility_test/rad_sw.meta deleted file mode 100644 index d88b9acc..00000000 --- a/test/var_compatibility_test/rad_sw.meta +++ /dev/null @@ -1,41 +0,0 @@ -[ccpp-table-properties] - name = rad_sw - type = scheme -[ccpp-arg-table] - name = rad_sw_run - type = scheme -[ ncol ] - standard_name = horizontal_loop_extent - type = integer - units = count - dimensions = () - intent = in -[ sfc_up_sw ] - standard_name = surface_upwelling_shortwave_radiation_flux - units = W m2 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ sfc_down_sw ] - standard_name = surface_downwelling_shortwave_radiation_flux - units = W m2 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=512 - intent = out -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test/var_compatibility_test/test_host.F90 b/test/var_compatibility_test/test_host.F90 deleted file mode 100644 index 67c7a1ac..00000000 --- a/test/var_compatibility_test/test_host.F90 +++ /dev/null @@ -1,264 +0,0 @@ -module test_prog - - use ccpp_kinds, only: kind_phys - - implicit none - private - - public test_host - - ! Public data and interfaces - integer, public, parameter :: cs = 32 - integer, public, parameter :: cm = 60 - - !> \section arg_table_suite_info Argument Table - !! \htmlinclude arg_table_suite_info.html - !! - type, public :: suite_info - character(len=cs) :: suite_name = '' - character(len=cs), pointer :: suite_parts(:) => null() - character(len=cm), pointer :: suite_input_vars(:) => null() - character(len=cm), pointer :: suite_output_vars(:) => null() - character(len=cm), pointer :: suite_required_vars(:) => null() - end type suite_info - -contains - - logical function check_suite(test_suite) - use test_host_ccpp_cap, only: ccpp_physics_suite_part_list - use test_host_ccpp_cap, only: ccpp_physics_suite_variables - use test_utils, only: check_list - - ! Dummy argument - type(suite_info), intent(in) :: test_suite - ! Local variables - integer :: sind - logical :: check - integer :: errflg - character(len=512) :: errmsg - character(len=128), allocatable :: test_list(:) - - check_suite = .true. - write(6, *) "Checking suite ", trim(test_suite%suite_name) - ! First, check the suite parts - call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_parts, 'part names', & - suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check the input variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.true., output_vars=.false.) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_input_vars, & - 'input variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check the output variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.false., output_vars=.true.) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_output_vars, & - 'output variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - ! Check all required variables - call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then - check = check_list(test_list, test_suite%suite_required_vars, & - 'required variable names', suite_name=test_suite%suite_name) - else - check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) - end if - check_suite = check_suite .and. check - if (allocated(test_list)) then - deallocate(test_list) - end if - end function check_suite - - !> \section arg_table_test_host Argument Table - !! \htmlinclude arg_table_test_host.html - !! - subroutine test_host(retval, test_suites) - - use test_host_mod, only: ncols - use test_host_ccpp_cap, only: test_host_ccpp_physics_initialize - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_initial - use test_host_ccpp_cap, only: test_host_ccpp_physics_run - use test_host_ccpp_cap, only: test_host_ccpp_physics_timestep_final - use test_host_ccpp_cap, only: test_host_ccpp_physics_finalize - use test_host_ccpp_cap, only: ccpp_physics_suite_list - use test_host_mod, only: init_data, & - compare_data - use test_utils, only: check_list - - type(suite_info), intent(in) :: test_suites(:) - logical, intent(out) :: retval - - logical :: check - integer :: col_start, col_end - integer :: index, sind - integer :: num_suites - character(len=128), allocatable :: suite_names(:) - character(len=512) :: errmsg - integer :: errflg - - ! Initialize our 'data' - call init_data() - - ! Gather and test the inspection routines - num_suites = size(test_suites) - call ccpp_physics_suite_list(suite_names) - retval = check_list(suite_names, test_suites(:)%suite_name, & - 'suite names') - write(6, *) 'Available suites are:' - do index = 1, size(suite_names) - do sind = 1, num_suites - if (trim(test_suites(sind)%suite_name) == & - trim(suite_names(index))) then - exit - end if - end do - write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & - ' = test_suites(', sind, ')' - end do - if (retval) then - do sind = 1, num_suites - check = check_suite(test_suites(sind)) - retval = retval .and. check - end do - end if - !!! Return here if any check failed - if (.not. retval) then - return - end if - - ! Use the suite information to setup the run - do sind = 1, num_suites - call test_host_ccpp_physics_initialize(test_suites(sind)%suite_name, & - errmsg, errflg) - if (errflg /= 0) then - write(6, '(4a)') 'ERROR in initialize of ', & - trim(test_suites(sind)%suite_name), ': ', trim(errmsg) - end if - end do - - ! Initialize the timestep - do sind = 1, num_suites - if (errflg /= 0) then - exit - end if - if (errflg == 0) then - call test_host_ccpp_physics_timestep_initial( & - test_suites(sind)%suite_name, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(errmsg) - exit - end if - if (errflg /= 0) then - exit - end if - end do - - do col_start = 1, ncols, 5 - if (errflg /= 0) then - exit - end if - col_end = min(col_start + 4, ncols) - - do sind = 1, num_suites - if (errflg /= 0) then - exit - end if - do index = 1, size(test_suites(sind)%suite_parts) - if (errflg /= 0) then - exit - end if - if (errflg == 0) then - call test_host_ccpp_physics_run( & - test_suites(sind)%suite_name, & - test_suites(sind)%suite_parts(index), & - col_start, col_end, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(5a)') trim(test_suites(sind)%suite_name), & - '/', trim(test_suites(sind)%suite_parts(index)), & - ': ', trim(errmsg) - exit - end if - end do - end do - end do - - do sind = 1, num_suites - if (errflg /= 0) then - exit - end if - if (errflg == 0) then - call test_host_ccpp_physics_timestep_final( & - test_suites(sind)%suite_name, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & - trim(errmsg) - exit - end if - end do - - do sind = 1, num_suites - if (errflg /= 0) then - exit - end if - if (errflg == 0) then - call test_host_ccpp_physics_finalize( & - test_suites(sind)%suite_name, errmsg, errflg) - end if - if (errflg /= 0) then - write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & - trim(errmsg) - write(6, '(2a)') 'An error occurred in ccpp_timestep_final, ', & - 'Exiting...' - exit - end if - end do - - if (errflg == 0) then - ! Run finished without error, check answers - if (compare_data()) then - write(6, *) 'Answers are correct!' - errflg = 0 - else - write(6, *) 'Answers are not correct!' - errflg = -1 - end if - end if - - retval = errflg == 0 - - end subroutine test_host - -end module test_prog diff --git a/test/var_compatibility_test/test_host.meta b/test/var_compatibility_test/test_host.meta deleted file mode 100644 index da71b182..00000000 --- a/test/var_compatibility_test/test_host.meta +++ /dev/null @@ -1,38 +0,0 @@ -[ccpp-table-properties] - name = suite_info - type = ddt -[ccpp-arg-table] - name = suite_info - type = ddt - -[ccpp-table-properties] - name = test_host - type = host -[ccpp-arg-table] - name = test_host - type = host -[ col_start ] - standard_name = horizontal_loop_begin - type = integer - units = count - dimensions = () - protected = True -[ col_end ] - standard_name = horizontal_loop_end - type = integer - units = count - dimensions = () - protected = True -[ errmsg ] - standard_name = ccpp_error_message - long_name = Error message for error handling in CCPP - units = None - dimensions = () - type = character - kind = len=512 -[ errflg ] - standard_name = ccpp_error_code - long_name = Error flag for error handling in CCPP - units = 1 - dimensions = () - type = integer diff --git a/test/var_compatibility_test/test_host_data.F90 b/test/var_compatibility_test/test_host_data.F90 deleted file mode 100644 index ece60034..00000000 --- a/test/var_compatibility_test/test_host_data.F90 +++ /dev/null @@ -1,103 +0,0 @@ -module test_host_data - - use ccpp_kinds, only: kind_phys - use mod_rad_ddt, only: ty_rad_lw, & - ty_rad_sw - - implicit none - private - - !> \section arg_table_physics_state Argument Table - !! \htmlinclude arg_table_physics_state.html - type physics_state - real(kind=kind_phys), dimension(:, :), allocatable :: & - effrr, & ! effective radius of cloud rain - effrl, & ! effective radius of cloud liquid water - effri, & ! effective radius of cloud ice - effrg, & ! effective radius of cloud graupel - ncg, & ! number concentration of cloud graupel - nci ! number concentration of cloud ice - real(kind=kind_phys) :: scalar_var - type(ty_rad_lw), dimension(:), allocatable :: & - fluxlw ! Longwave radiation fluxes - type(ty_rad_sw) :: & - fluxsw ! Shortwave radiation fluxes - real(kind=kind_phys) :: scalar_vara - real(kind=kind_phys) :: scalar_varb - real(kind=kind_phys) :: tke, tke2 - integer :: scalar_varc - integer :: scheme_order - integer :: num_subcycles - end type physics_state - - public :: physics_state - public :: allocate_physics_state - -contains - - subroutine allocate_physics_state(cols, levels, state, has_graupel, has_ice) - integer, intent(in) :: cols - integer, intent(in) :: levels - type(physics_state), intent(out) :: state - logical, intent(in) :: has_graupel - logical, intent(in) :: has_ice - - if (allocated(state%effrr)) then - deallocate(state%effrr) - end if - allocate(state%effrr(cols, levels)) - - if (allocated(state%effrl)) then - deallocate(state%effrl) - end if - allocate(state%effrl(cols, levels)) - - if (has_ice) then - if (allocated(state%effri)) then - deallocate(state%effri) - end if - allocate(state%effri(cols, levels)) - end if - - if (has_graupel) then - if (allocated(state%effrg)) then - deallocate(state%effrg) - end if - allocate(state%effrg(cols, levels)) - - if (allocated(state%ncg)) then - deallocate(state%ncg) - end if - allocate(state%ncg(cols, levels)) - end if - - if (has_ice) then - if (allocated(state%nci)) then - deallocate(state%nci) - end if - allocate(state%nci(cols, levels)) - end if - - if (allocated(state%fluxlw)) then - deallocate(state%fluxlw) - end if - allocate(state%fluxlw(cols)) - - if (associated(state%fluxsw%sfc_up_sw)) then - nullify(state%fluxsw%sfc_up_sw) - end if - allocate(state%fluxsw%sfc_up_sw(cols)) - - if (associated(state%fluxsw%sfc_down_sw)) then - nullify(state%fluxsw%sfc_down_sw) - end if - allocate(state%fluxsw%sfc_down_sw(cols)) - - ! Initialize scheme counter. - state%scheme_order = 1 - ! Initialize subcycle counter. - state%num_subcycles = 3 - - end subroutine allocate_physics_state - -end module test_host_data diff --git a/test/var_compatibility_test/test_host_data.meta b/test/var_compatibility_test/test_host_data.meta deleted file mode 100644 index 59a0fb4d..00000000 --- a/test/var_compatibility_test/test_host_data.meta +++ /dev/null @@ -1,128 +0,0 @@ -[ccpp-table-properties] - name = physics_state - type = ddt - dependencies = module_rad_ddt.F90 -[ccpp-arg-table] - name = physics_state - type = ddt -[effrr] - standard_name = effective_radius_of_stratiform_cloud_rain_particle - long_name = effective radius of cloud rain particle in meter - units = m - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys -[effrl] - standard_name = effective_radius_of_stratiform_cloud_liquid_water_particle - long_name = effective radius of cloud liquid water particle in meter - units = m - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys -[effri] - standard_name = effective_radius_of_stratiform_cloud_ice_particle - long_name = effective radius of cloud ice water particle in meter - units = m - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - active = (flag_indicating_cloud_microphysics_has_ice) -[effrg] - standard_name = effective_radius_of_stratiform_cloud_graupel - long_name = effective radius of cloud graupel in meter - units = m - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - active = (flag_indicating_cloud_microphysics_has_graupel) -[ncg] - standard_name = cloud_graupel_number_concentration - long_name = number concentration of cloud graupel - units = kg-1 - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - active = (flag_indicating_cloud_microphysics_has_graupel) -[nci] - standard_name = cloud_ice_number_concentration - long_name = number concentration of cloud ice - units = kg-1 - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in - active = (flag_indicating_cloud_microphysics_has_ice) -[scalar_var] - standard_name = scalar_variable_for_testing - long_name = unused scalar variable - units = m - dimensions = () - type = real - kind = kind_phys -[ tke ] - standard_name = turbulent_kinetic_energy - long_name = turbulent_kinetic_energy - units = J kg-1 - dimensions = () - type = real - kind = kind_phys -[ tke2 ] - standard_name = turbulent_kinetic_energy2 - long_name = turbulent_kinetic_energy2 - units = m2 s-2 - dimensions = () - type = real - kind = kind_phys -[fluxSW] - standard_name = shortwave_radiation_fluxes - long_name = shortwave radiation fluxes - units = W m-2 - dimensions = () - type = ty_rad_sw -[fluxLW] - standard_name = longwave_radiation_fluxes - long_name = longwave radiation fluxes - units = W m-2 - dimensions = (horizontal_dimension) - type = ty_rad_lw -[scalar_varA] - standard_name = scalar_variable_for_testing_a - long_name = unused scalar variable A - units = m - dimensions = () - type = real - kind = kind_phys -[scalar_varB] - standard_name = scalar_variable_for_testing_b - long_name = unused scalar variable B - units = m - dimensions = () - type = real - kind = kind_phys -[scalar_varC] - standard_name = scalar_variable_for_testing_c - long_name = unused scalar variable C - units = m - dimensions = () - type = integer -[scheme_order] - standard_name = scheme_order_in_suite - long_name = scheme order in suite definition file - units = None - dimensions = () - type = integer -[num_subcycles] - standard_name = num_subcycles_for_effr - long_name = Number of times to subcycle the effr calculation - units = None - dimensions = () - type = integer - -[ccpp-table-properties] - name = test_host_data - type = module - dependencies = module_rad_ddt.F90 -[ccpp-arg-table] - name = test_host_data - type = module diff --git a/test/var_compatibility_test/test_host_mod.F90 b/test/var_compatibility_test/test_host_mod.F90 deleted file mode 100644 index d3bde866..00000000 --- a/test/var_compatibility_test/test_host_mod.F90 +++ /dev/null @@ -1,132 +0,0 @@ -module test_host_mod - - use ccpp_kinds, only: kind_phys - use test_host_data, only: physics_state, & - allocate_physics_state - - implicit none - public - - !> \section arg_table_test_host_mod Argument Table - !! \htmlinclude arg_table_test_host_host.html - !! - integer, parameter :: ncols = 12 - integer, parameter :: pver = 4 - type(physics_state) :: phys_state - real(kind=kind_phys) :: effrs(ncols, pver) - logical, parameter :: has_ice = .true. - logical, parameter :: has_graupel = .true. - - public :: init_data - public :: compare_data - -contains - - subroutine init_data() - - ! Allocate and initialize state - call allocate_physics_state(ncols, pver, phys_state, has_graupel, has_ice) - phys_state%effrr = 1.0E-3 ! 1000 microns, in meter - phys_state%effrl = 1.0E-4 ! 100 microns, in meter - phys_state%scalar_var = 1.0 ! in m - phys_state%scalar_vara = 273.15 ! in K - phys_state%scalar_varb = 1013.0 ! in mb - phys_state%scalar_varc = 380 ! in ppmv - effrs = 5.0E-4 ! 500 microns, in meter - if (has_graupel) then - phys_state%effrg = 2.5E-4 ! 250 microns, in meter - phys_state%ncg = 40 - end if - if (has_ice) then - phys_state%effri = 5.0E-5 ! 50 microns, in meter - phys_state%nci = 80 - end if - phys_state%tke = 10.0 !J kg-1 - phys_state%tke2 = 42.0 !J kg-1 - - end subroutine init_data - - logical function compare_data() - - real(kind=kind_phys), parameter :: effrr_expected = 1.0E-3 ! 1000 microns, in meter - real(kind=kind_phys), parameter :: effrl_expected = 5.0E-5 ! 50 microns, in meter - real(kind=kind_phys), parameter :: effri_expected = 7.5E-5 ! 75 microns, in meter - real(kind=kind_phys), parameter :: effrs_expected = 5.3E-4 ! 530 microns, in meter - real(kind=kind_phys), parameter :: scalar_expected = 2.0E3 ! 2 km, in meter - real(kind=kind_phys), parameter :: tke_expected = 10.0 ! 10 J kg-1 - real(kind=kind_phys), parameter :: tolerance = 1.0E-6 ! used as scaling factor for expected value - real(kind=kind_phys), parameter :: sfc_up_sw_expected = 100. ! W/m2 - real(kind=kind_phys), parameter :: sfc_down_sw_expected = 400. ! W/m2 - real(kind=kind_phys), parameter :: sfc_up_lw_expected = 300. ! W/m2 - real(kind=kind_phys), parameter :: sfc_down_lw_expected = 50. ! W/m2 - - compare_data = .true. - - if (maxval(abs(phys_state%effrr - effrr_expected)) > tolerance * effrr_expected) then - write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effrr from expected value exceeds tolerance: ', & - maxval(abs(phys_state%effrr - effrr_expected)), ' > ', tolerance * effrr_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%effrl - effrl_expected)) > tolerance * effrl_expected) then - write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effrl from expected value exceeds tolerance: ', & - maxval(abs(phys_state%effrl - effrl_expected)), ' > ', tolerance * effrl_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%effri - effri_expected)) > tolerance * effri_expected) then - write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effri from expected value exceeds tolerance: ', & - maxval(abs(phys_state%effri - effri_expected)), ' > ', tolerance * effri_expected - compare_data = .false. - end if - - if (maxval(abs(effrs - effrs_expected)) > tolerance * effrs_expected) then - write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of effrs from expected value exceeds tolerance: ', & - maxval(abs(effrs - effrs_expected)), ' > ', tolerance * effrs_expected - compare_data = .false. - end if - - if (abs(phys_state%scalar_var - scalar_expected) > tolerance * scalar_expected) then - write(6, '(a,e16.7,a,e16.7)') & - 'Error: max diff of scalar_var from expected value exceeds tolerance: ', & - abs(phys_state%scalar_var - scalar_expected), ' > ', tolerance * scalar_expected - compare_data = .false. - end if - - if (abs(phys_state%tke - tke_expected) > tolerance * tke_expected) then - write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of tke from expected value exceeds tolerance: ', & - abs(phys_state%tke - tke_expected), ' > ', tolerance * tke_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%fluxsw%sfc_up_sw - sfc_up_sw_expected)) > tolerance * sfc_up_sw_expected) then - write(6, '(a,e16.7,a,e16.7)') & - 'Error: max diff of sfc_up_sw from expected value exceeds tolerance: ', & - abs(phys_state%fluxsw%sfc_up_sw - sfc_up_sw_expected), ' > ', tolerance * sfc_up_sw_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%fluxsw%sfc_down_sw - sfc_down_sw_expected)) > tolerance * sfc_down_sw_expected) then - write(6, '(a,e16.7,a,e16.7)') & - 'Error: max diff of sfc_down_sw from expected value exceeds tolerance: ', & - abs(phys_state%fluxsw%sfc_down_sw - sfc_down_sw_expected), ' > ', tolerance * sfc_down_sw_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%fluxlw%sfc_up_lw - sfc_up_lw_expected)) > tolerance * sfc_up_lw_expected) then - write(6, '(a,e16.7,a,e16.7)') & - 'Error: max diff of sfc_up_lw from expected value exceeds tolerance: ', & - abs(phys_state%fluxlw%sfc_up_lw - sfc_up_lw_expected), ' > ', tolerance * sfc_up_lw_expected - compare_data = .false. - end if - - if (maxval(abs(phys_state%fluxlw%sfc_down_lw - sfc_down_lw_expected)) > tolerance * sfc_down_lw_expected) then - write(6, '(a,e16.7,a,e16.7)') & - 'Error: max diff of sfc_down_lw from expected value exceeds tolerance: ', & - abs(phys_state%fluxlw%sfc_down_lw - sfc_down_lw_expected), ' > ', tolerance * sfc_down_lw_expected - compare_data = .false. - end if - - end function compare_data - -end module test_host_mod diff --git a/test/var_compatibility_test/test_host_mod.meta b/test/var_compatibility_test/test_host_mod.meta deleted file mode 100644 index 51a2f5c3..00000000 --- a/test/var_compatibility_test/test_host_mod.meta +++ /dev/null @@ -1,42 +0,0 @@ -[ccpp-table-properties] - name = test_host_mod - type = module -[ccpp-arg-table] - name = test_host_mod - type = module -[ ncols] - standard_name = horizontal_dimension - units = count - type = integer - protected = True - dimensions = () -[ pver ] - standard_name = vertical_layer_dimension - units = count - type = integer - protected = True - dimensions = () -[ phys_state ] - standard_name = physics_state_derived_type - long_name = Physics State DDT - type = physics_state - dimensions = () -[effrs] - standard_name = effective_radius_of_stratiform_cloud_snow_particle - long_name = effective radius of cloud snow particle in meter - units = m - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys -[has_ice] - standard_name = flag_indicating_cloud_microphysics_has_ice - long_name = flag indicating that the cloud microphysics produces ice - units = flag - dimensions = () - type = logical -[has_graupel] - standard_name = flag_indicating_cloud_microphysics_has_graupel - long_name = flag indicating that the cloud microphysics produces graupel - units = flag - dimensions = () - type = logical diff --git a/test/var_compatibility_test/test_var_compatibility_integration.F90 b/test/var_compatibility_test/test_var_compatibility_integration.F90 deleted file mode 100644 index 4115face..00000000 --- a/test/var_compatibility_test/test_var_compatibility_integration.F90 +++ /dev/null @@ -1,88 +0,0 @@ -program test_var_compatibility_integration - use test_prog, only: test_host, & - suite_info, & - cm, & - cs - - implicit none - - character(len=cs), target :: test_parts1(1) = (/ 'radiation ' /) - - character(len=cm), target :: test_invars1(18) = (/ & - 'effective_radius_of_stratiform_cloud_rain_particle ', & - 'effective_radius_of_stratiform_cloud_liquid_water_particle', & - 'effective_radius_of_stratiform_cloud_snow_particle ', & - 'effective_radius_of_stratiform_cloud_graupel ', & - 'cloud_graupel_number_concentration ', & - 'scalar_variable_for_testing ', & - 'turbulent_kinetic_energy ', & - 'turbulent_kinetic_energy2 ', & - 'scalar_variable_for_testing_a ', & - 'scalar_variable_for_testing_b ', & - 'scalar_variable_for_testing_c ', & - 'scheme_order_in_suite ', & - 'num_subcycles_for_effr ', & - 'flag_indicating_cloud_microphysics_has_graupel ', & - 'flag_indicating_cloud_microphysics_has_ice ', & - 'surface_downwelling_shortwave_radiation_flux ', & - 'surface_upwelling_shortwave_radiation_flux ', & - 'longwave_radiation_fluxes '/) - - character(len=cm), target :: test_outvars1(14) = (/ & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'effective_radius_of_stratiform_cloud_ice_particle ', & - 'effective_radius_of_stratiform_cloud_liquid_water_particle', & - 'effective_radius_of_stratiform_cloud_rain_particle ', & - 'effective_radius_of_stratiform_cloud_snow_particle ', & - 'cloud_ice_number_concentration ', & - 'scalar_variable_for_testing ', & - 'scheme_order_in_suite ', & - 'surface_downwelling_shortwave_radiation_flux ', & - 'surface_upwelling_shortwave_radiation_flux ', & - 'turbulent_kinetic_energy ', & - 'turbulent_kinetic_energy2 ', & - 'longwave_radiation_fluxes '/) - - character(len=cm), target :: test_reqvars1(22) = (/ & - 'ccpp_error_code ', & - 'ccpp_error_message ', & - 'effective_radius_of_stratiform_cloud_rain_particle ', & - 'effective_radius_of_stratiform_cloud_ice_particle ', & - 'effective_radius_of_stratiform_cloud_liquid_water_particle', & - 'effective_radius_of_stratiform_cloud_snow_particle ', & - 'effective_radius_of_stratiform_cloud_graupel ', & - 'cloud_graupel_number_concentration ', & - 'cloud_ice_number_concentration ', & - 'scalar_variable_for_testing ', & - 'turbulent_kinetic_energy ', & - 'turbulent_kinetic_energy2 ', & - 'scalar_variable_for_testing_a ', & - 'scalar_variable_for_testing_b ', & - 'scalar_variable_for_testing_c ', & - 'scheme_order_in_suite ', & - 'num_subcycles_for_effr ', & - 'flag_indicating_cloud_microphysics_has_graupel ', & - 'flag_indicating_cloud_microphysics_has_ice ', & - 'surface_downwelling_shortwave_radiation_flux ', & - 'surface_upwelling_shortwave_radiation_flux ', & - 'longwave_radiation_fluxes '/) - - type(suite_info) :: test_suites(1) - logical :: run_okay - - ! Setup expected test suite info - test_suites(1)%suite_name = 'var_compatibility_suite' - test_suites(1)%suite_parts => test_parts1 - test_suites(1)%suite_input_vars => test_invars1 - test_suites(1)%suite_output_vars => test_outvars1 - test_suites(1)%suite_required_vars => test_reqvars1 - - call test_host(run_okay, test_suites) - - if (run_okay) then - stop 0 - else - stop -1 - end if -end program test_var_compatibility_integration diff --git a/test/var_compatibility_test/var_compatibility_suite.xml b/test/var_compatibility_test/var_compatibility_suite.xml deleted file mode 100644 index 07d950ef..00000000 --- a/test/var_compatibility_test/var_compatibility_suite.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - effr_pre - - - effr_calc - - - effr_post - - - effrs_calc - - effr_diag - rad_lw - rad_sw - - diff --git a/test/var_compatibility_test/var_compatibility_test_reports.py b/test/var_compatibility_test/var_compatibility_test_reports.py deleted file mode 100755 index ada612c7..00000000 --- a/test/var_compatibility_test/var_compatibility_test_reports.py +++ /dev/null @@ -1,116 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Test capgen database report python interface - - Assumptions: - - Command line arguments: build_dir database_filepath - - Usage: python test_reports ------------------------------------------------------------------------ -""" -import os -import unittest - -from test_stub import BaseTests - -_BUILD_DIR = os.path.join(os.path.abspath(os.environ['BUILD_DIR']), "test", "var_compatibility_test") -_DATABASE = os.path.abspath(os.path.join(_BUILD_DIR, "ccpp", "datatable.xml")) - -_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -_FRAMEWORK_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, os.pardir)) -_SCRIPTS_DIR = os.path.join(_FRAMEWORK_DIR, "scripts") -_SRC_DIR = os.path.join(_FRAMEWORK_DIR, "src") - -# Check data -_HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] -_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_compatibility_suite_cap.F90")] -_UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), - os.path.join(_SRC_DIR, "ccpp_constituent_prop_mod.F90"), - os.path.join(_SRC_DIR, "ccpp_scheme_utils.F90"), - os.path.join(_SRC_DIR, "ccpp_hashable.F90"), - os.path.join(_SRC_DIR, "ccpp_hash_table.F90")] -_CCPP_FILES = _UTILITY_FILES + \ - [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90"), - os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_compatibility_suite_cap.F90")] -_PROCESS_LIST = [""] -_MODULE_LIST = ["effr_calc", "effrs_calc", "effr_diag", "effr_post", "mod_effr_pre", "rad_lw", "rad_sw"] -_SUITE_LIST = ["var_compatibility_suite"] -_DEPENDENCIES = [ os.path.join(_TEST_DIR, "module_rad_ddt.F90")] -_INPUT_VARS_VAR_ACTION = ["horizontal_loop_begin", "horizontal_loop_end", "horizontal_dimension", "vertical_layer_dimension", - "effective_radius_of_stratiform_cloud_liquid_water_particle", - "effective_radius_of_stratiform_cloud_rain_particle", - "effective_radius_of_stratiform_cloud_snow_particle", - "effective_radius_of_stratiform_cloud_graupel", - "cloud_graupel_number_concentration", - "scalar_variable_for_testing", - "turbulent_kinetic_energy", - "turbulent_kinetic_energy2", - "scalar_variable_for_testing_a", - "scalar_variable_for_testing_b", - "scalar_variable_for_testing_c", - "scheme_order_in_suite", - "flag_indicating_cloud_microphysics_has_graupel", - "flag_indicating_cloud_microphysics_has_ice", - "surface_downwelling_shortwave_radiation_flux", - "surface_upwelling_shortwave_radiation_flux", - "longwave_radiation_fluxes", - "num_subcycles_for_effr"] -_OUTPUT_VARS_VAR_ACTION = ["ccpp_error_code", "ccpp_error_message", - "effective_radius_of_stratiform_cloud_ice_particle", - "effective_radius_of_stratiform_cloud_liquid_water_particle", - "effective_radius_of_stratiform_cloud_snow_particle", - "cloud_ice_number_concentration", - "effective_radius_of_stratiform_cloud_rain_particle", - "turbulent_kinetic_energy", - "turbulent_kinetic_energy2", - "scalar_variable_for_testing", - "scalar_variable_for_testing", - "surface_downwelling_shortwave_radiation_flux", - "surface_upwelling_shortwave_radiation_flux", - "longwave_radiation_fluxes", - "scheme_order_in_suite"] -_REQUIRED_VARS_VAR_ACTION = _INPUT_VARS_VAR_ACTION + _OUTPUT_VARS_VAR_ACTION - - -class TestVarCompatibilityHostDataTables(unittest.TestCase, BaseTests.TestHostDataTables): - database = _DATABASE - host_files = _HOST_FILES - suite_files = _SUITE_FILES - utility_files = _UTILITY_FILES - ccpp_files = _CCPP_FILES - process_list = _PROCESS_LIST - module_list = _MODULE_LIST - dependencies = _DEPENDENCIES - suite_list = _SUITE_LIST - - -class CommandLineVarCompatibilityHostDatafileRequiredFiles(unittest.TestCase, BaseTests.TestHostCommandLineDataFiles): - database = _DATABASE - host_files = _HOST_FILES - suite_files = _SUITE_FILES - utility_files = _UTILITY_FILES - ccpp_files = _CCPP_FILES - process_list = _PROCESS_LIST - module_list = _MODULE_LIST - dependencies = _DEPENDENCIES - suite_list = _SUITE_LIST - datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" - - -class TestCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuite): - database = _DATABASE - required_vars = _REQUIRED_VARS_VAR_ACTION - input_vars = _INPUT_VARS_VAR_ACTION - output_vars = _OUTPUT_VARS_VAR_ACTION - suite_name = "var_compatibility_suite" - - -class CommandLineCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuiteCommandLine): - database = _DATABASE - required_vars = _REQUIRED_VARS_VAR_ACTION - input_vars = _INPUT_VARS_VAR_ACTION - output_vars = _OUTPUT_VARS_VAR_ACTION - suite_name = "var_compatibility_suite" - datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" diff --git a/test_prebuild/run_all_tests.sh b/test_prebuild/run_all_tests.sh deleted file mode 100755 index 08e5910a..00000000 --- a/test_prebuild/run_all_tests.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -echo "" && echo "Running unit_tests " && cd unit_tests && ./run_tests.sh && cd .. -echo "" && echo "Running test_opt_arg " && cd test_opt_arg && ./run_test.sh && cd .. -# No longer possible because of https://github.com/NCAR/ccpp-framework/pull/659 -#echo "" && echo "Running test_blocked_data" && cd test_blocked_data && ./run_test.sh && cd .. -echo "" && echo "Running test_chunked_data" && cd test_chunked_data && ./run_test.sh && cd .. -echo "" && echo "Running test_unit_conv" && cd test_unit_conv && ./run_test.sh && cd .. - -echo "" && echo "Running test_track_variables" && pytest test_track_variables.py diff --git a/test_prebuild/test_blocked_data/CMakeLists.txt b/test_prebuild/test_blocked_data/CMakeLists.txt deleted file mode 100644 index edce19a3..00000000 --- a/test_prebuild/test_blocked_data/CMakeLists.txt +++ /dev/null @@ -1,98 +0,0 @@ -#------------------------------------------------------------------------------ -cmake_minimum_required(VERSION 3.10) - -project(ccpp_blocked_data - VERSION 1.0.0 - LANGUAGES Fortran) - -#------------------------------------------------------------------------------ -# Request a static build -option(BUILD_SHARED_LIBS "Build a shared library" OFF) - -#------------------------------------------------------------------------------ -# Set MPI flags for C/C++/Fortran with MPI F08 interface -find_package(MPI REQUIRED Fortran) -if(NOT MPI_Fortran_HAVE_F08_MODULE) - message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") -endif() - -#------------------------------------------------------------------------------ -# Set the sources: physics type definitions -set(TYPEDEFS $ENV{CCPP_TYPEDEFS}) -if(TYPEDEFS) - message(STATUS "Got CCPP TYPEDEFS from environment variable: ${TYPEDEFS}") -else(TYPEDEFS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) - message(STATUS "Got CCPP TYPEDEFS from cmakefile include file: ${TYPEDEFS}") -endif(TYPEDEFS) - -# Generate list of Fortran modules from the CCPP type -# definitions that need need to be installed -foreach(typedef_module ${TYPEDEFS}) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module}) -endforeach() - -#------------------------------------------------------------------------------ -# Set the sources: physics schemes -set(SCHEMES $ENV{CCPP_SCHEMES}) -if(SCHEMES) - message(STATUS "Got CCPP SCHEMES from environment variable: ${SCHEMES}") -else(SCHEMES) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) - message(STATUS "Got CCPP SCHEMES from cmakefile include file: ${SCHEMES}") -endif(SCHEMES) - -# Set the sources: physics scheme caps -set(CAPS $ENV{CCPP_CAPS}) -if(CAPS) - message(STATUS "Got CCPP CAPS from environment variable: ${CAPS}") -else(CAPS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) - message(STATUS "Got CCPP CAPS from cmakefile include file: ${CAPS}") -endif(CAPS) - -# Set the sources: physics scheme caps -set(API $ENV{CCPP_API}) -if(API) - message(STATUS "Got CCPP API from environment variable: ${API}") -else(API) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) - message(STATUS "Got CCPP API from cmakefile include file: ${API}") -endif(API) - -set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -ggdb -fbacktrace -ffree-line-length-none") - -#------------------------------------------------------------------------------ -add_library(ccpp_blocked_data STATIC ${SCHEMES} ${CAPS} ${API}) -target_link_libraries(ccpp_blocked_data PRIVATE MPI::MPI_Fortran) -# Generate list of Fortran modules from defined sources -foreach(source_f90 ${CAPS} ${API}) - get_filename_component(tmp_source_f90 ${source_f90} NAME) - string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90}) - string(TOLOWER ${tmp_module_f90} module_f90) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90}) -endforeach() - -set_target_properties(ccpp_blocked_data PROPERTIES VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) - -add_executable(test_blocked_data.x main.F90) -add_dependencies(test_blocked_data.x ccpp_blocked_data) -target_link_libraries(test_blocked_data.x ccpp_blocked_data) -set_target_properties(test_blocked_data.x PROPERTIES LINKER_LANGUAGE Fortran) - -# Define where to install the library -install(TARGETS ccpp_blocked_data - EXPORT ccpp_blocked_data-targets - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION lib -) -# Export our configuration -install(EXPORT ccpp_blocked_data-targets - FILE ccpp_blocked_data-config.cmake - DESTINATION lib/cmake -) -# Define where to install the C headers and Fortran modules -#install(FILES ${HEADERS_C} DESTINATION include) -install(FILES ${MODULES_F90} DESTINATION include) diff --git a/test_prebuild/test_blocked_data/README.md b/test_prebuild/test_blocked_data/README.md deleted file mode 100644 index 8802e812..00000000 --- a/test_prebuild/test_blocked_data/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# How to build the blocked data test - -1. Set compiler environment as appropriate for your system -2. Run the following commands: -``` -cd test_prebuild/test_blocked_data/ -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make 2>&1 | tee log.make -./test_blocked_data.x -``` diff --git a/test_prebuild/test_blocked_data/blocked_data_scheme.F90 b/test_prebuild/test_blocked_data/blocked_data_scheme.F90 deleted file mode 100644 index 77e1e687..00000000 --- a/test_prebuild/test_blocked_data/blocked_data_scheme.F90 +++ /dev/null @@ -1,126 +0,0 @@ -!>\file blocked_data_scheme.F90 -!! This file contains a blocked_data_scheme CCPP scheme that does nothing -!! except requesting the minimum, mandatory variables. - -module blocked_data_scheme - - use, intrinsic :: iso_fortran_env, only: error_unit - implicit none - - private - public :: blocked_data_scheme_init, & - blocked_data_scheme_timestep_init, & - blocked_data_scheme_run, & - blocked_data_scheme_timestep_finalize, & - blocked_data_scheme_finalize - - ! This is for unit testing only - integer, parameter, dimension(4) :: data_array_sizes = (/6, 6, 6, 3/) - -contains - - !! \section arg_table_blocked_data_scheme_init Argument Table - !! \htmlinclude blocked_data_scheme_init.html - !! - subroutine blocked_data_scheme_init(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In blocked_data_scheme_init: checking size of data array to be', sum(data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine blocked_data_scheme_init - - !! \section arg_table_blocked_data_scheme_timestep_init Argument Table - !! \htmlinclude blocked_data_scheme_timestep_init.html - !! - subroutine blocked_data_scheme_timestep_init(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In blocked_data_scheme_timestep_init: checking size of data array to be', sum(& - data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), " but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine blocked_data_scheme_timestep_init - - !! \section arg_table_blocked_data_scheme_run Argument Table - !! \htmlinclude blocked_data_scheme_run.html - !! - subroutine blocked_data_scheme_run(nb, data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: nb - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(2(a,i3))') 'In blocked_data_scheme_run: checking size of data array for block', nb, & - ' to be', data_array_sizes(nb) - if (size(data_array)/=data_array_sizes(nb)) then - write(errmsg, '(a,i4)') "Error in blocked_data_scheme_run, expected size(data_array)==6, got ", size(data_array) - errflg = 1 - return - end if - end subroutine blocked_data_scheme_run - - !! \section arg_table_blocked_data_scheme_timestep_finalize Argument Table - !! \htmlinclude blocked_data_scheme_timestep_finalize.html - !! - subroutine blocked_data_scheme_timestep_finalize(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In blocked_data_scheme_timestep_finalize: checking size of data array to be', sum(& - data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine blocked_data_scheme_timestep_finalize - - !! \section arg_table_blocked_data_scheme_finalize Argument Table - !! \htmlinclude blocked_data_scheme_finalize.html - !! - subroutine blocked_data_scheme_finalize(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In blocked_data_scheme_finalize: checking size of data array to be', sum(& - data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine blocked_data_scheme_finalize - -end module blocked_data_scheme diff --git a/test_prebuild/test_blocked_data/blocked_data_scheme.meta b/test_prebuild/test_blocked_data/blocked_data_scheme.meta deleted file mode 100644 index d92b0da6..00000000 --- a/test_prebuild/test_blocked_data/blocked_data_scheme.meta +++ /dev/null @@ -1,147 +0,0 @@ -[ccpp-table-properties] - name = blocked_data_scheme - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = blocked_data_scheme_init - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[data_array] - standard_name = blocked_data_array - long_name = blocked data array - units = 1 - dimensions = (horizontal_dimension) - type = integer - intent = in - -######################################################################## -[ccpp-arg-table] - name = blocked_data_scheme_timestep_init - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[data_array] - standard_name = blocked_data_array - long_name = blocked data array - units = 1 - dimensions = (horizontal_dimension) - type = integer - intent = in - -######################################################################## -[ccpp-arg-table] - name = blocked_data_scheme_run - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[nb] - standard_name = ccpp_block_number - long_name = number of block for explicit data blocking in CCPP - units = index - dimensions = () - type = integer - intent = in -[data_array] - standard_name = blocked_data_array - long_name = blocked data array - units = 1 - dimensions = (horizontal_loop_extent) - type = integer - intent = in - -######################################################################## -[ccpp-arg-table] - name = blocked_data_scheme_timestep_finalize - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[data_array] - standard_name = blocked_data_array - long_name = blocked data array - units = 1 - dimensions = (horizontal_dimension) - type = integer - intent = in - -######################################################################## -[ccpp-arg-table] - name = blocked_data_scheme_finalize - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[data_array] - standard_name = blocked_data_array - long_name = blocked data array - units = 1 - dimensions = (horizontal_dimension) - type = integer - intent = in - diff --git a/test_prebuild/test_blocked_data/blocked_data_suite.xml b/test_prebuild/test_blocked_data/blocked_data_suite.xml deleted file mode 100644 index cf4fe9a4..00000000 --- a/test_prebuild/test_blocked_data/blocked_data_suite.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - blocked_data_scheme - - - diff --git a/test_prebuild/test_blocked_data/ccpp_prebuild_config.py b/test_prebuild/test_blocked_data/ccpp_prebuild_config.py deleted file mode 100644 index 700d9f76..00000000 --- a/test_prebuild/test_blocked_data/ccpp_prebuild_config.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -# CCPP prebuild config for GFDL Finite-Volume Cubed-Sphere Model (FV3) - -############################################################################### -# Definitions # -############################################################################### - -HOST_MODEL_IDENTIFIER = "FV3" - -# Add all files with metadata tables on the host model side and in CCPP, -# relative to basedir = top-level directory of host model. This includes -# kind and type definitions used in CCPP physics. Also add any internal -# dependencies of these files to the list. -VARIABLE_DEFINITION_FILES = [ - # actual variable definition files - '../../src/ccpp_types.F90', - 'data.F90', - ] - -TYPEDEFS_NEW_METADATA = { - 'ccpp_types' : { - 'ccpp_t' : 'cdata', - 'ccpp_types' : '', - }, - 'data' : { - 'blocked_data_type' : 'blocked_data_instance(cdata%blk_no)', - 'data' : '', - }, - } - -# Add all physics scheme files relative to basedir -SCHEME_FILES = [ - 'blocked_data_scheme.F90', - ] - -# Default build dir, relative to current working directory, -# if not specified as command-line argument -DEFAULT_BUILD_DIR = 'build' - -# Auto-generated makefile/cmakefile snippets that contain all type definitions -TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' -TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' -TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' - -# Auto-generated makefile/cmakefile snippets that contain all schemes -SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' -SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' -SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' - -# Auto-generated makefile/cmakefile snippets that contain all caps -CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' -CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' -CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' - -# Directory where to put all auto-generated physics caps -CAPS_DIR = '{build_dir}' - -# Directory where the suite definition files are stored -SUITES_DIR = '.' - -# Optional arguments - only required for schemes that use -# optional arguments. ccpp_prebuild.py will throw an exception -# if it encounters a scheme subroutine with optional arguments -# if no entry is made here. Possible values are: 'all', 'none', -# or a list of standard_names: [ 'var1', 'var3' ]. -OPTIONAL_ARGUMENTS = {} - -# Directory where to write static API to -STATIC_API_DIR = '{build_dir}' -STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' -STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' - -# Directory for writing HTML pages generated from metadata files -METADATA_HTML_OUTPUT_DIR = '{build_dir}' - -# HTML document containing the model-defined CCPP variables -HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_BLOCKED_DATA.html' - -# LaTeX document containing the provided vs requested CCPP variables -LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_BLOCKED_DATA.tex' diff --git a/test_prebuild/test_blocked_data/data.F90 b/test_prebuild/test_blocked_data/data.F90 deleted file mode 100644 index 0d399f27..00000000 --- a/test_prebuild/test_blocked_data/data.F90 +++ /dev/null @@ -1,41 +0,0 @@ -module data - - !! \section arg_table_data Argument Table - !! \htmlinclude data.html - !! - use ccpp_types, only: ccpp_t - - implicit none - - private - - public nblks, blksz, ncols - public ccpp_data_domain, ccpp_data_blocks, blocked_data_type, blocked_data_instance - - integer, parameter :: nblks = 4 - type(ccpp_t), target :: ccpp_data_domain - type(ccpp_t), dimension(nblks), target :: ccpp_data_blocks - - integer, parameter, dimension(nblks) :: blksz = (/6, 6, 6, 3/) - integer, parameter :: ncols = sum(blksz) - - !! \section arg_table_blocked_data_type - !! \htmlinclude blocked_data_type.html - !! - type blocked_data_type - integer, dimension(:), allocatable :: array_data - contains - procedure :: create => blocked_data_create - end type blocked_data_type - - type(blocked_data_type), dimension(nblks) :: blocked_data_instance - -contains - - subroutine blocked_data_create(blocked_data_instance, ncol) - class(blocked_data_type), intent(inout) :: blocked_data_instance - integer, intent(in) :: ncol - allocate(blocked_data_instance%array_data(ncol)) - end subroutine blocked_data_create - -end module data diff --git a/test_prebuild/test_blocked_data/data.meta b/test_prebuild/test_blocked_data/data.meta deleted file mode 100644 index c5fa2842..00000000 --- a/test_prebuild/test_blocked_data/data.meta +++ /dev/null @@ -1,69 +0,0 @@ -[ccpp-table-properties] - name = blocked_data_type - type = ddt - dependencies = -[ccpp-arg-table] - name = blocked_data_type - type = ddt -[array_data] - standard_name = blocked_data_array - long_name = blocked data array - units = 1 - dimensions = (horizontal_loop_extent) - type = integer - -[ccpp-table-properties] - name = data - type = module - dependencies = -[ccpp-arg-table] - name = data - type = module -[cdata] - standard_name = ccpp_t_instance - long_name = instance of derived data type ccpp_t - units = DDT - dimensions = () - type = ccpp_t -[nblks] - standard_name = ccpp_block_count - long_name = for explicit data blocking: number of blocks - units = count - dimensions = () - type = integer -[blksz] - standard_name = ccpp_block_sizes - long_name = for explicit data blocking: block sizes of all blocks - units = count - dimensions = (ccpp_block_count) - type = integer -[blksz(ccpp_block_number)] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - dimensions = () - type = integer -[ncols] - standard_name = horizontal_dimension - long_name = horizontal dimension - units = count - dimensions = () - type = integer -[blocked_data_type] - standard_name = blocked_data_type - long_name = definition of type blocked_data_type - units = DDT - dimensions = () - type = blocked_data_type -[blocked_data_instance(ccpp_block_number)] - standard_name = blocked_data_type_instance - long_name = instance of derived data type blocked_data_type - units = DDT - dimensions = () - type = blocked_data_type -[blocked_data_instance] - standard_name = blocked_data_type_instance_all_blocks - long_name = instance of derived data type blocked_data_type - units = DDT - dimensions = (ccpp_block_count) - type = blocked_data_type diff --git a/test_prebuild/test_blocked_data/main.F90 b/test_prebuild/test_blocked_data/main.F90 deleted file mode 100644 index a6d86a35..00000000 --- a/test_prebuild/test_blocked_data/main.F90 +++ /dev/null @@ -1,117 +0,0 @@ -program test_blocked_data - - use, intrinsic :: iso_fortran_env, only: error_unit - - use ccpp_types, only: ccpp_t - use data, only: nblks, & - blksz, & - ncols - use data, only: ccpp_data_domain, & - ccpp_data_blocks, & - blocked_data_type, & - blocked_data_instance - - use ccpp_static_api, only: ccpp_physics_init, & - ccpp_physics_timestep_init, & - ccpp_physics_run, & - ccpp_physics_timestep_finalize, & - ccpp_physics_finalize - - implicit none - - character(len=*), parameter :: ccpp_suite = 'blocked_data_suite' - integer :: ib, ierr - type(ccpp_t), pointer :: cdata => null() - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - ! For physics running over the entire domain, - ! ccpp_thread_number and ccpp_chunk_number are - ! set to 1, indicating that arrays are to be sent - ! following their dimension specification in the - ! metadata (must match horizontal_dimension). - ccpp_data_domain%blk_no = 1 - ccpp_data_domain%thrd_no = 1 - ccpp_data_domain%thrd_cnt = 1 - - ! Loop over all blocks and threads for ccpp_data_blocks - do ib = 1, nblks - ! Assign the correct block numbers, only one thread - ccpp_data_blocks(ib)%blk_no = ib - ccpp_data_blocks(ib)%thrd_no = 1 - ccpp_data_blocks(ib)%thrd_cnt = 1 - end do - - do ib = 1, size(blocked_data_instance) - allocate(blocked_data_instance(ib)%array_data(blksz(ib))) - write(error_unit, '(2(a,i3))') "Allocated array_data for block", ib, " to size", size(blocked_data_instance(ib)%& - array_data) - end do - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics run step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - do ib = 1, nblks - cdata => ccpp_data_blocks(ib) - call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a,i3,a)') "An error occurred in ccpp_physics_run for block", ib, ":" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - end do - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_finalize:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_finalize:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - -contains - -end program test_blocked_data diff --git a/test_prebuild/test_blocked_data/run_test.sh b/test_prebuild/test_blocked_data/run_test.sh deleted file mode 100755 index ee67d183..00000000 --- a/test_prebuild/test_blocked_data/run_test.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --debug --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make 2>&1 | tee log.make -./test_blocked_data.x -cd .. -rm -fr build diff --git a/test_prebuild/test_chunked_data/CMakeLists.txt b/test_prebuild/test_chunked_data/CMakeLists.txt deleted file mode 100644 index e2e7cf93..00000000 --- a/test_prebuild/test_chunked_data/CMakeLists.txt +++ /dev/null @@ -1,98 +0,0 @@ -#------------------------------------------------------------------------------ -cmake_minimum_required(VERSION 3.10) - -project(ccpp_chunked_data - VERSION 1.0.0 - LANGUAGES Fortran) - -#------------------------------------------------------------------------------ -# Request a static build -option(BUILD_SHARED_LIBS "Build a shared library" OFF) - -#------------------------------------------------------------------------------ -# Set MPI flags for C/C++/Fortran with MPI F08 interface -find_package(MPI REQUIRED Fortran) -if(NOT MPI_Fortran_HAVE_F08_MODULE) - message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") -endif() - -#------------------------------------------------------------------------------ -# Set the sources: physics type definitions -set(TYPEDEFS $ENV{CCPP_TYPEDEFS}) -if(TYPEDEFS) - message(STATUS "Got CCPP TYPEDEFS from environment variable: ${TYPEDEFS}") -else(TYPEDEFS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) - message(STATUS "Got CCPP TYPEDEFS from cmakefile include file: ${TYPEDEFS}") -endif(TYPEDEFS) - -# Generate list of Fortran modules from the CCPP type -# definitions that need need to be installed -foreach(typedef_module ${TYPEDEFS}) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module}) -endforeach() - -#------------------------------------------------------------------------------ -# Set the sources: physics schemes -set(SCHEMES $ENV{CCPP_SCHEMES}) -if(SCHEMES) - message(STATUS "Got CCPP SCHEMES from environment variable: ${SCHEMES}") -else(SCHEMES) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) - message(STATUS "Got CCPP SCHEMES from cmakefile include file: ${SCHEMES}") -endif(SCHEMES) - -# Set the sources: physics scheme caps -set(CAPS $ENV{CCPP_CAPS}) -if(CAPS) - message(STATUS "Got CCPP CAPS from environment variable: ${CAPS}") -else(CAPS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) - message(STATUS "Got CCPP CAPS from cmakefile include file: ${CAPS}") -endif(CAPS) - -# Set the sources: physics scheme caps -set(API $ENV{CCPP_API}) -if(API) - message(STATUS "Got CCPP API from environment variable: ${API}") -else(API) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) - message(STATUS "Got CCPP API from cmakefile include file: ${API}") -endif(API) - -set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -ggdb -fbacktrace -ffree-line-length-none") - -#------------------------------------------------------------------------------ -add_library(ccpp_chunked_data STATIC ${SCHEMES} ${CAPS} ${API}) -target_link_libraries(ccpp_chunked_data PRIVATE MPI::MPI_Fortran) -# Generate list of Fortran modules from defined sources -foreach(source_f90 ${CAPS} ${API}) - get_filename_component(tmp_source_f90 ${source_f90} NAME) - string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90}) - string(TOLOWER ${tmp_module_f90} module_f90) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90}) -endforeach() - -set_target_properties(ccpp_chunked_data PROPERTIES VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) - -add_executable(test_chunked_data.x main.F90) -add_dependencies(test_chunked_data.x ccpp_chunked_data) -target_link_libraries(test_chunked_data.x ccpp_chunked_data) -set_target_properties(test_chunked_data.x PROPERTIES LINKER_LANGUAGE Fortran) - -# Define where to install the library -install(TARGETS ccpp_chunked_data - EXPORT ccpp_chunked_data-targets - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION lib -) -# Export our configuration -install(EXPORT ccpp_chunked_data-targets - FILE ccpp_chunked_data-config.cmake - DESTINATION lib/cmake -) -# Define where to install the C headers and Fortran modules -#install(FILES ${HEADERS_C} DESTINATION include) -install(FILES ${MODULES_F90} DESTINATION include) diff --git a/test_prebuild/test_chunked_data/README.md b/test_prebuild/test_chunked_data/README.md deleted file mode 100644 index 16db6fc5..00000000 --- a/test_prebuild/test_chunked_data/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# How to build the chunked data test - -1. Set compiler environment as appropriate for your system -2. Run the following commands: -``` -cd test_prebuild/test_chunked_data/ -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make 2>&1 | tee log.make -./test_chunked_data.x -# On systems where linking against the MPI library requires a parallel launcher, -# use 'mpirun -np 1 ./test_chunked_data.x' or 'srun -n 1 ./test_chunked_data.x' etc. -``` diff --git a/test_prebuild/test_chunked_data/ccpp_prebuild_config.py b/test_prebuild/test_chunked_data/ccpp_prebuild_config.py deleted file mode 100755 index 4e32d37d..00000000 --- a/test_prebuild/test_chunked_data/ccpp_prebuild_config.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -# CCPP prebuild config for GFDL Finite-Volume Cubed-Sphere Model (FV3) - -############################################################################### -# Definitions # -############################################################################### - -HOST_MODEL_IDENTIFIER = "FV3" - -# Add all files with metadata tables on the host model side and in CCPP, -# relative to basedir = top-level directory of host model. This includes -# kind and type definitions used in CCPP physics. Also add any internal -# dependencies of these files to the list. -VARIABLE_DEFINITION_FILES = [ - # actual variable definition files - '../../src/ccpp_types.F90', - 'data.F90', - ] - -TYPEDEFS_NEW_METADATA = { - 'ccpp_types' : { - 'ccpp_t' : 'cdata', - 'ccpp_types' : '', - }, - 'data' : { - 'chunked_data_type' : 'chunked_data_instance', - 'data' : '', - }, - } - -# Add all physics scheme files relative to basedir -SCHEME_FILES = [ - 'chunked_data_scheme.F90', - ] - -# Default build dir, relative to current working directory, -# if not specified as command-line argument -DEFAULT_BUILD_DIR = 'build' - -# Auto-generated makefile/cmakefile snippets that contain all type definitions -TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' -TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' -TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' - -# Auto-generated makefile/cmakefile snippets that contain all schemes -SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' -SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' -SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' - -# Auto-generated makefile/cmakefile snippets that contain all caps -CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' -CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' -CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' - -# Directory where to put all auto-generated physics caps -CAPS_DIR = '{build_dir}' - -# Directory where the suite definition files are stored -SUITES_DIR = '.' - -# Optional arguments - only required for schemes that use -# optional arguments. ccpp_prebuild.py will throw an exception -# if it encounters a scheme subroutine with optional arguments -# if no entry is made here. Possible values are: 'all', 'none', -# or a list of standard_names: [ 'var1', 'var3' ]. -OPTIONAL_ARGUMENTS = {} - -# Directory where to write static API to -STATIC_API_DIR = '{build_dir}' -STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' -STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' - -# Directory for writing HTML pages generated from metadata files -METADATA_HTML_OUTPUT_DIR = '{build_dir}' - -# HTML document containing the model-defined CCPP variables -HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_CHUNKED_DATA.html' - -# LaTeX document containing the provided vs requested CCPP variables -LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_CHUNKED_DATA.tex' diff --git a/test_prebuild/test_chunked_data/chunked_data_scheme.F90 b/test_prebuild/test_chunked_data/chunked_data_scheme.F90 deleted file mode 100644 index 392167b2..00000000 --- a/test_prebuild/test_chunked_data/chunked_data_scheme.F90 +++ /dev/null @@ -1,126 +0,0 @@ -!>\file chunked_data_scheme.F90 -!! This file contains a chunked_data_scheme CCPP scheme that does nothing -!! except requesting the minimum, mandatory variables. - -module chunked_data_scheme - - use, intrinsic :: iso_fortran_env, only: error_unit - implicit none - - private - public :: chunked_data_scheme_init, & - chunked_data_scheme_timestep_init, & - chunked_data_scheme_run, & - chunked_data_scheme_timestep_finalize, & - chunked_data_scheme_finalize - - ! This is for unit testing only - integer, parameter, dimension(4) :: data_array_sizes = (/6, 6, 6, 3/) - -contains - - !! \section arg_table_chunked_data_scheme_init Argument Table - !! \htmlinclude chunked_data_scheme_init.html - !! - subroutine chunked_data_scheme_init(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In chunked_data_scheme_init: checking size of data array to be', sum(data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine chunked_data_scheme_init - - !! \section arg_table_chunked_data_scheme_timestep_init Argument Table - !! \htmlinclude chunked_data_scheme_timestep_init.html - !! - subroutine chunked_data_scheme_timestep_init(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In chunked_data_scheme_timestep_init: checking size of data array to be', sum(& - data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), " but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine chunked_data_scheme_timestep_init - - !! \section arg_table_chunked_data_scheme_run Argument Table - !! \htmlinclude chunked_data_scheme_run.html - !! - subroutine chunked_data_scheme_run(nchunk, data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: nchunk - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(2(a,i3))') 'In chunked_data_scheme_run: checking size of data array for chunk', nchunk, & - ' to be', data_array_sizes(nchunk) - if (size(data_array)/=data_array_sizes(nchunk)) then - write(errmsg, '(a,i4)') "Error in chunked_data_scheme_run, expected size(data_array)==6, got ", size(data_array) - errflg = 1 - return - end if - end subroutine chunked_data_scheme_run - - !! \section arg_table_chunked_data_scheme_timestep_finalize Argument Table - !! \htmlinclude chunked_data_scheme_timestep_finalize.html - !! - subroutine chunked_data_scheme_timestep_finalize(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In chunked_data_scheme_timestep_finalize: checking size of data array to be', sum(& - data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine chunked_data_scheme_timestep_finalize - - !! \section arg_table_chunked_data_scheme_finalize Argument Table - !! \htmlinclude chunked_data_scheme_finalize.html - !! - subroutine chunked_data_scheme_finalize(data_array, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: data_array(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check size of data array - write(error_unit, '(a,i3)') 'In chunked_data_scheme_finalize: checking size of data array to be', sum(& - data_array_sizes) - if (size(data_array)/=sum(data_array_sizes)) then - write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& - data_array) - errflg = 1 - return - end if - end subroutine chunked_data_scheme_finalize - -end module chunked_data_scheme diff --git a/test_prebuild/test_chunked_data/chunked_data_scheme.meta b/test_prebuild/test_chunked_data/chunked_data_scheme.meta deleted file mode 100644 index 13830dbf..00000000 --- a/test_prebuild/test_chunked_data/chunked_data_scheme.meta +++ /dev/null @@ -1,147 +0,0 @@ -[ccpp-table-properties] - name = chunked_data_scheme - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = chunked_data_scheme_init - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[data_array] - standard_name = chunked_data_array - long_name = chunked data array - units = 1 - dimensions = (horizontal_dimension) - type = integer - intent = in - -######################################################################## -[ccpp-arg-table] - name = chunked_data_scheme_timestep_init - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[data_array] - standard_name = chunked_data_array - long_name = chunked data array - units = 1 - dimensions = (horizontal_dimension) - type = integer - intent = in - -######################################################################## -[ccpp-arg-table] - name = chunked_data_scheme_run - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[nchunk] - standard_name = ccpp_chunk_number - long_name = number of chunk for chunked arrays in CCPP - units = index - dimensions = () - type = integer - intent = in -[data_array] - standard_name = chunked_data_array - long_name = chunked data array - units = 1 - dimensions = (horizontal_loop_extent) - type = integer - intent = in - -######################################################################## -[ccpp-arg-table] - name = chunked_data_scheme_timestep_finalize - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[data_array] - standard_name = chunked_data_array - long_name = chunked data array - units = 1 - dimensions = (horizontal_dimension) - type = integer - intent = in - -######################################################################## -[ccpp-arg-table] - name = chunked_data_scheme_finalize - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[data_array] - standard_name = chunked_data_array - long_name = chunked data array - units = 1 - dimensions = (horizontal_dimension) - type = integer - intent = in - diff --git a/test_prebuild/test_chunked_data/data.F90 b/test_prebuild/test_chunked_data/data.F90 deleted file mode 100644 index 82c4abac..00000000 --- a/test_prebuild/test_chunked_data/data.F90 +++ /dev/null @@ -1,43 +0,0 @@ -module data - - !! \section arg_table_dATa Argument Table - !! \htmlinclude datA.Html - !! - use ccpp_types, only: ccpp_t - - implicit none - - private - - public nchunks, chunksize, chunk_begin, chunk_end, ncols - public ccpp_data_domain, ccpp_data_chunks, chunked_data_type, chunked_data_instance - - integer, parameter :: nchunks = 4 - type(ccpp_t), target :: ccpp_data_domain - type(ccpp_t), dimension(nchunks), target :: ccpp_data_chunks - - integer, parameter, dimension(nchunks) :: chunksize = (/6, 6, 6, 3/) - integer, parameter, dimension(nchunks) :: chunk_begin = (/1, 7, 13, 19/) - integer, parameter, dimension(nchunks) :: chunk_end = (/6, 12, 18, 21/) - integer, parameter :: ncols = sum(chunksize) - - !! \section arg_table_cHuNkEd_dATa_TYPe - !! \htmlinclude CHuNKed_Data_tYpe.hTMl - !! - type chunked_data_type - integer, dimension(:), allocatable :: array_data - contains - procedure :: create => chunked_data_create - end type chunked_data_type - - type(chunked_data_type) :: chunked_data_instance - -contains - - subroutine chunked_data_create(chunked_data_instance, ncol) - class(chunked_data_type), intent(inout) :: chunked_data_instance - integer, intent(in) :: ncol - allocate(chunked_data_instance%array_data(ncol)) - end subroutine chunked_data_create - -end module data diff --git a/test_prebuild/test_chunked_data/data.meta b/test_prebuild/test_chunked_data/data.meta deleted file mode 100644 index c14217df..00000000 --- a/test_prebuild/test_chunked_data/data.meta +++ /dev/null @@ -1,76 +0,0 @@ -[ccpp-table-properties] - name = chunked_data_type - type = ddt - dependencies = -[ccpp-arg-table] - name = chunked_data_type - type = ddt -[array_data] - standard_name = chunked_data_array - long_name = chunked data array - units = 1 - dimensions = (ccpp_constant_one:horizontal_dimension) - type = integer -# Todo: define an additional array running from -1 to ncols-2 - -[ccpp-table-properties] - name = data - type = module - dependencies = -[ccpp-arg-table] - name = data - type = module -[cdata] - standard_name = ccpp_t_instance - long_name = instance of derived data type ccpp_t - units = DDT - dimensions = () - type = ccpp_t -[ncols] - standard_name = horizontal_dimension - long_name = horizontal dimension - units = count - dimensions = () - type = integer -[nchunks] - standard_name = ccpp_chunk_extent - long_name = number of chunks of array data used in run phase - units = count - dimensions = () - type = integer -[chunk_begin] - standard_name = horizontal_loop_begin_all_chunks - long_name = first index for horizontal loop extent in run phase - units = index - dimensions = (ccpp_chunk_extent) - type = integer -[chunk_begin(ccpp_chunk_number)] - standard_name = horizontal_loop_begin - long_name = first index for horizontal loop extent in run phase - units = index - dimensions = () - type = integer -[chunk_end] - standard_name = horizontal_loop_end_all_chunks - long_name = last index for horizontal loop extent in run phase - units = index - dimensions = (ccpp_chunk_extent) - type = integer -[chunk_end(ccpp_chunk_number)] - standard_name = horizontal_loop_end - long_name = last index for horizontal loop extent in run phase - units = index - dimensions = () - type = integer -[chunked_data_type] - standard_name = chunked_data_type - long_name = definition of type chunked_data_type - units = DDT - dimensions = () - type = chunked_data_type -[chunked_data_instance] - standard_name = chunked_data_type_instance - long_name = instance of derived data type chunked_data_type - units = DDT - dimensions = () - type = chunked_data_type diff --git a/test_prebuild/test_chunked_data/main.F90 b/test_prebuild/test_chunked_data/main.F90 deleted file mode 100644 index 739ebf8b..00000000 --- a/test_prebuild/test_chunked_data/main.F90 +++ /dev/null @@ -1,114 +0,0 @@ -program test_chunked_data - - use, intrinsic :: iso_fortran_env, only: error_unit - - use ccpp_types, only: ccpp_t - use data, only: nchunks, & - chunksize, & - ncols - use data, only: ccpp_data_domain, & - ccpp_data_chunks, & - chunked_data_type, & - chunked_data_instance - - use ccpp_static_api, only: ccpp_physics_init, & - ccpp_physics_timestep_init, & - ccpp_physics_run, & - ccpp_physics_timestep_finalize, & - ccpp_physics_finalize - - implicit none - - character(len=*), parameter :: ccpp_suite = 'chunked_data_suite' - integer :: ic, ierr - type(ccpp_t), pointer :: cdata => null() - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - ! For physics running over the entire domain, - ! ccpp_thread_number and ccpp_chunk_number are - ! set to 1, indicating that arrays are to be sent - ! following their dimension specification in the - ! metadata (must match horizontal_dimension). - ccpp_data_domain%thrd_no = 1 - ccpp_data_domain%chunk_no = 1 - ccpp_data_domain%thrd_cnt = 1 - - ! Loop over all chunks and threads for ccpp_data_chunks - do ic = 1, nchunks - ! Assign the correct chunk numbers, only one thread - ccpp_data_chunks(ic)%chunk_no = ic - ccpp_data_chunks(ic)%thrd_no = 1 - ccpp_data_chunks(ic)%thrd_cnt = 1 - end do - - call chunked_data_instance%create(ncols) - write(error_unit, '(2(a,i3))') "Chunked_data_instance%array_data to size", size(chunked_data_instance%array_data) - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics run step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - do ic = 1, nchunks - cdata => ccpp_data_chunks(ic) - call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a,i3,a)') "An error occurred in ccpp_physics_run for chunk", ic, ":" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - end do - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_finalize:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata => ccpp_data_domain - call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_finalize:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - -contains - -end program test_chunked_data diff --git a/test_prebuild/test_chunked_data/run_test.sh b/test_prebuild/test_chunked_data/run_test.sh deleted file mode 100755 index 9ba2a36e..00000000 --- a/test_prebuild/test_chunked_data/run_test.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e - -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --debug --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make 2>&1 | tee log.make -./test_chunked_data.x -cd .. -rm -fr build diff --git a/test_prebuild/test_chunked_data/suite_chunked_data_suite.xml b/test_prebuild/test_chunked_data/suite_chunked_data_suite.xml deleted file mode 100644 index 923e5fb7..00000000 --- a/test_prebuild/test_chunked_data/suite_chunked_data_suite.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - chunked_data_scheme - - - diff --git a/test_prebuild/test_opt_arg/CMakeLists.txt b/test_prebuild/test_opt_arg/CMakeLists.txt deleted file mode 100644 index 58e6e6b5..00000000 --- a/test_prebuild/test_opt_arg/CMakeLists.txt +++ /dev/null @@ -1,98 +0,0 @@ -#------------------------------------------------------------------------------ -cmake_minimum_required(VERSION 3.10) - -project(ccpp_opt_arg - VERSION 1.0.0 - LANGUAGES Fortran) - -#------------------------------------------------------------------------------ -# Request a static build -option(BUILD_SHARED_LIBS "Build a shared library" OFF) - -#------------------------------------------------------------------------------ -# Set MPI flags for C/C++/Fortran with MPI F08 interface -find_package(MPI REQUIRED Fortran) -if(NOT MPI_Fortran_HAVE_F08_MODULE) - message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") -endif() - -#------------------------------------------------------------------------------ -# Set the sources: physics type definitions -set(TYPEDEFS $ENV{CCPP_TYPEDEFS}) -if(TYPEDEFS) - message(STATUS "Got CCPP TYPEDEFS from environment variable: ${TYPEDEFS}") -else(TYPEDEFS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) - message(STATUS "Got CCPP TYPEDEFS from cmakefile include file: ${TYPEDEFS}") -endif(TYPEDEFS) - -# Generate list of Fortran modules from the CCPP type -# definitions that need need to be installed -foreach(typedef_module ${TYPEDEFS}) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module}) -endforeach() - -#------------------------------------------------------------------------------ -# Set the sources: physics schemes -set(SCHEMES $ENV{CCPP_SCHEMES}) -if(SCHEMES) - message(STATUS "Got CCPP SCHEMES from environment variable: ${SCHEMES}") -else(SCHEMES) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) - message(STATUS "Got CCPP SCHEMES from cmakefile include file: ${SCHEMES}") -endif(SCHEMES) - -# Set the sources: physics scheme caps -set(CAPS $ENV{CCPP_CAPS}) -if(CAPS) - message(STATUS "Got CCPP CAPS from environment variable: ${CAPS}") -else(CAPS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) - message(STATUS "Got CCPP CAPS from cmakefile include file: ${CAPS}") -endif(CAPS) - -# Set the sources: physics scheme caps -set(API $ENV{CCPP_API}) -if(API) - message(STATUS "Got CCPP API from environment variable: ${API}") -else(API) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) - message(STATUS "Got CCPP API from cmakefile include file: ${API}") -endif(API) - -set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fcheck=all -ggdb -fbacktrace -ffree-line-length-none") - -#------------------------------------------------------------------------------ -add_library(ccpp_opt_arg STATIC ${SCHEMES} ${CAPS} ${API}) -target_link_libraries(ccpp_opt_arg PRIVATE MPI::MPI_Fortran) -# Generate list of Fortran modules from defined sources -foreach(source_f90 ${CAPS} ${API}) - get_filename_component(tmp_source_f90 ${source_f90} NAME) - string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90}) - string(TOLOWER ${tmp_module_f90} module_f90) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90}) -endforeach() - -set_target_properties(ccpp_opt_arg PROPERTIES VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) - -add_executable(test_opt_arg.x main.F90) -add_dependencies(test_opt_arg.x ccpp_opt_arg) -target_link_libraries(test_opt_arg.x ccpp_opt_arg) -set_target_properties(test_opt_arg.x PROPERTIES LINKER_LANGUAGE Fortran) - -# Define where to install the library -install(TARGETS ccpp_opt_arg - EXPORT ccpp_opt_arg-targets - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION lib -) -# Export our configuration -install(EXPORT ccpp_opt_arg-targets - FILE ccpp_opt_arg-config.cmake - DESTINATION lib/cmake -) -# Define where to install the C headers and Fortran modules -#install(FILES ${HEADERS_C} DESTINATION include) -install(FILES ${MODULES_F90} DESTINATION include) diff --git a/test_prebuild/test_opt_arg/ccpp_kinds.F90 b/test_prebuild/test_opt_arg/ccpp_kinds.F90 deleted file mode 100644 index a07ded9b..00000000 --- a/test_prebuild/test_opt_arg/ccpp_kinds.F90 +++ /dev/null @@ -1,13 +0,0 @@ -module ccpp_kinds - - !! \section arg_table_ccpp_kinds - !! \htmlinclude ccpp_kinds.html - !! - - use iso_fortran_env, only: real64 - - implicit none - - integer, parameter :: kind_phys = real64 - -end module ccpp_kinds diff --git a/test_prebuild/test_opt_arg/ccpp_kinds.meta b/test_prebuild/test_opt_arg/ccpp_kinds.meta deleted file mode 100644 index 0e95702e..00000000 --- a/test_prebuild/test_opt_arg/ccpp_kinds.meta +++ /dev/null @@ -1,15 +0,0 @@ -[ccpp-table-properties] - name = ccpp_kinds - type = module - dependencies = - -######################################################################## -[ccpp-arg-table] - name = ccpp_kinds - type = module -[kind_phys] - standard_name = kind_phys - long_name = definition of kind_phys - units = none - dimensions = () - type = integer diff --git a/test_prebuild/test_opt_arg/ccpp_prebuild_config.py b/test_prebuild/test_opt_arg/ccpp_prebuild_config.py deleted file mode 100755 index bf3bc1cc..00000000 --- a/test_prebuild/test_opt_arg/ccpp_prebuild_config.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python - -# CCPP prebuild config for GFDL Finite-Volume Cubed-Sphere Model (FV3) - -############################################################################### -# Definitions # -############################################################################### - -HOST_MODEL_IDENTIFIER = "FV3" - -# Add all files with metadata tables on the host model side and in CCPP, -# relative to basedir = top-level directory of host model. This includes -# kind and type definitions used in CCPP physics. Also add any internal -# dependencies of these files to the list. -VARIABLE_DEFINITION_FILES = [ - # actual variable definition files - '../../src/ccpp_types.F90', - 'ccpp_kinds.F90', - 'data.F90', - ] - -TYPEDEFS_NEW_METADATA = { - 'ccpp_types' : { - 'ccpp_t' : 'cdata', - 'ccpp_types' : '', - }, - 'data' : { - 'data' : '', - }, - } - -# Add all physics scheme files relative to basedir -SCHEME_FILES = [ - 'opt_arg_scheme.F90', - ] - -# Default build dir, relative to current working directory, -# if not specified as command-line argument -DEFAULT_BUILD_DIR = 'build' - -# Auto-generated makefile/cmakefile snippets that contain all type definitions -TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' -TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' -TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' - -# Auto-generated makefile/cmakefile snippets that contain all schemes -SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' -SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' -SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' - -# Auto-generated makefile/cmakefile snippets that contain all caps -CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' -CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' -CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' - -# Directory where to put all auto-generated physics caps -CAPS_DIR = '{build_dir}' - -# Directory where the suite definition files are stored -SUITES_DIR = '.' - -# Optional arguments - only required for schemes that use -# optional arguments. ccpp_prebuild.py will throw an exception -# if it encounters a scheme subroutine with optional arguments -# if no entry is made here. Possible values are: 'all', 'none', -# or a list of standard_names: [ 'var1', 'var3' ]. -OPTIONAL_ARGUMENTS = {} - -# Directory where to write static API to -STATIC_API_DIR = '{build_dir}' -STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' -STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' - -# Directory for writing HTML pages generated from metadata files -METADATA_HTML_OUTPUT_DIR = '{build_dir}' - -# HTML document containing the model-defined CCPP variables -HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_opt_arg.html' - -# LaTeX document containing the provided vs requested CCPP variables -LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_opt_arg.tex' diff --git a/test_prebuild/test_opt_arg/data.F90 b/test_prebuild/test_opt_arg/data.F90 deleted file mode 100644 index f66cf8c1..00000000 --- a/test_prebuild/test_opt_arg/data.F90 +++ /dev/null @@ -1,23 +0,0 @@ -module data - - !! \section arg_table_data Argument Table - !! \htmlinclude data.html - !! - use ccpp_types, only: ccpp_t - use ccpp_kinds, only: kind_phys - - implicit none - - private - - public cdata, nx, flag_for_opt_arg, std_arg, opt_arg, opt_arg_2 - - type(ccpp_t), target :: cdata - integer, parameter :: nx = 3 - logical :: flag_for_opt_arg - - integer, dimension(nx) :: std_arg - integer, dimension(:), allocatable :: opt_arg - real(kind=kind_phys), dimension(:), allocatable :: opt_arg_2 - -end module data diff --git a/test_prebuild/test_opt_arg/data.meta b/test_prebuild/test_opt_arg/data.meta deleted file mode 100644 index 03f3c472..00000000 --- a/test_prebuild/test_opt_arg/data.meta +++ /dev/null @@ -1,46 +0,0 @@ -[ccpp-table-properties] - name = data - type = module - dependencies = -[ccpp-arg-table] - name = data - type = module -[cdata] - standard_name = ccpp_t_instance - long_name = instance of derived data type ccpp_t - units = DDT - dimensions = () - type = ccpp_t -[nx] - standard_name = size_of_std_arg - long_name = size of std_arg - units = count - dimensions = () - type = integer -[std_arg] - standard_name = std_arg - long_name = mandatory variable - units = 1 - dimensions = (size_of_std_arg) - type = integer -[opt_arg] - standard_name = opt_arg - long_name = optional variable - units = 1 - dimensions = (size_of_std_arg) - type = integer - active = flag_for_opt_arg -[opt_arg_2] - standard_name = opt_arg_2 - long_name = optional variable with unit conversions - units = km - dimensions = (size_of_std_arg) - type = real - kind = kind_phys - active = flag_for_opt_arg -[flag_for_opt_arg] - standard_name = flag_for_opt_arg - long_name = flag for optional variable - units = 1 - dimensions = () - type = logical diff --git a/test_prebuild/test_opt_arg/main.F90 b/test_prebuild/test_opt_arg/main.F90 deleted file mode 100644 index 8d08619a..00000000 --- a/test_prebuild/test_opt_arg/main.F90 +++ /dev/null @@ -1,125 +0,0 @@ -program test_opt_arg - - use, intrinsic :: iso_fortran_env, only: output_unit, & - error_unit - - use ccpp_types, only: ccpp_t - use data, only: cdata, & - nx, & - flag_for_opt_arg, & - std_arg, & - opt_arg, & - opt_arg_2 - - use ccpp_static_api, only: ccpp_physics_init, & - ccpp_physics_timestep_init, & - ccpp_physics_run, & - ccpp_physics_timestep_finalize, & - ccpp_physics_finalize - - implicit none - - character(len=*), parameter :: ccpp_suite = 'opt_arg_suite' - integer :: ierr - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - cdata%blk_no = 1 - cdata%thrd_no = 1 - cdata%thrd_cnt = 1 - - std_arg = 1 - flag_for_opt_arg = .true. - allocate(opt_arg(nx)) - allocate(opt_arg_2(nx)) - - ! std_arg must all be 1, opt_arg must all be 0 - write(output_unit, '(a)') "After ccpp_init: check std_arg(:)==1, opt_arg(:)==0, opt_arg_2(:)==0" - if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_init: std_arg=", std_arg - if (.not. all(opt_arg == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_init: opt_arg=", opt_arg - if (.not. all(opt_arg_2 == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_init: opt_arg_2=", opt_arg_2 - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - ! std_arg must all be 1, opt_arg must all be 0 - write(output_unit, '(a)') "After ccpp_physics_init: check std_arg(:)==1 and opt_arg(:)==0" - if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_init: std_arg=", std_arg - if (.not. all(opt_arg == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_init: opt_arg=", opt_arg - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - ! std_arg must all be 1, opt_arg must all be 2 - write(output_unit, '(a)') "After ccpp_physics_timestep_init: check std_arg(:)==1 and opt_arg(:)==2" - if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_init: std_arg=", std_arg - if (.not. all(opt_arg == 2)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_init: opt_arg=", opt_arg - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics run step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_run:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - ! std_arg must all be 1, opt_arg must all be 3 - write(output_unit, '(a)') "After ccpp_physics_run: check std_arg(:)==1 and opt_arg(:)==3" - if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_run: std_arg=", std_arg - if (.not. all(opt_arg == 3)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_run: opt_arg=", opt_arg - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - deallocate(opt_arg) - flag_for_opt_arg = .false. - - call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - ! std_arg must all be 7, opt_arg no longer allocated - write(output_unit, '(a)') "After ccpp_physics_timestep_final: check std_arg(:)==7; opt_arg not allocated" - if (.not. all(std_arg == 7)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_final: std_arg=", std_arg - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - ! std_arg must all be 7, opt_arg no longer allocated - write(output_unit, '(a)') "After ccpp_physics_timestep_final: check std_arg(:)==7; opt_arg not allocated" - if (.not. all(std_arg == 7)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_final: std_arg=", std_arg - -end program test_opt_arg diff --git a/test_prebuild/test_opt_arg/opt_arg_scheme.F90 b/test_prebuild/test_opt_arg/opt_arg_scheme.F90 deleted file mode 100644 index 33be0973..00000000 --- a/test_prebuild/test_opt_arg/opt_arg_scheme.F90 +++ /dev/null @@ -1,90 +0,0 @@ -!>\file opt_arg_scheme.F90 -!! This file contains a opt_arg_scheme CCPP scheme that does nothing -!! except requesting the minimum, mandatory variables. - -module opt_arg_scheme - - use, intrinsic :: iso_fortran_env, only: error_unit - use ccpp_kinds, only: kind_phys - - implicit none - - private - public :: opt_arg_scheme_timestep_init, & - opt_arg_scheme_run, & - opt_arg_scheme_timestep_finalize - -contains - - !! \section arg_table_opt_arg_scheme_timestep_init Argument Table - !! \htmlinclude opt_arg_scheme_timestep_init.html - !! - subroutine opt_arg_scheme_timestep_init(nx, var, opt_var, opt_var_2, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: nx - integer, intent(in) :: var(:) - integer, optional, intent(out) :: opt_var(:) - real(kind=kind_phys), optional, intent(out) :: opt_var_2(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Initialize opt_var from var if opt_var if present - if (present(opt_var)) then - opt_var = 2 * var - end if - ! Initialize opt_var_2 from var if opt_var_2 present - if (present(opt_var_2)) then - opt_var_2 = 3.0_kind_phys * var - end if - end subroutine opt_arg_scheme_timestep_init - - !! \section arg_table_opt_arg_scheme_run Argument Table - !! \htmlinclude opt_arg_scheme_run.html - !! - subroutine opt_arg_scheme_run(nx, var, opt_var, opt_var_2, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: nx - integer, intent(in) :: var(:) - integer, optional, intent(inout) :: opt_var(:) - real(kind=kind_phys), optional, intent(inout) :: opt_var_2(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Update opt_var from var if opt_var present - if (present(opt_var)) then - opt_var = 3 * var - end if - ! Update opt_var_2 from var if opt_var_2 present - if (present(opt_var_2)) then - opt_var_2 = 4.0_kind_phys * var - end if - end subroutine opt_arg_scheme_run - - !! \section arg_table_opt_arg_scheme_timestep_finalize Argument Table - !! \htmlinclude opt_arg_scheme_timestep_finalize.html - !! - subroutine opt_arg_scheme_timestep_finalize(nx, var, opt_var, opt_var_2, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - integer, intent(in) :: nx - integer, intent(inout) :: var(:) - integer, optional, intent(in) :: opt_var(:) - real(kind=kind_phys), optional, intent(inout) :: opt_var_2(:) - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Update var from opt_var if opt_var is present - if (present(opt_var)) then - var = 4 * opt_var - else - var = 7 * var - end if - ! Update opt_var_2 if present - if (present(opt_var_2)) then - opt_var_2 = opt_var_2 + 5.0_kind_phys - end if - end subroutine opt_arg_scheme_timestep_finalize - -end module opt_arg_scheme diff --git a/test_prebuild/test_opt_arg/opt_arg_scheme.meta b/test_prebuild/test_opt_arg/opt_arg_scheme.meta deleted file mode 100644 index a00519ec..00000000 --- a/test_prebuild/test_opt_arg/opt_arg_scheme.meta +++ /dev/null @@ -1,157 +0,0 @@ -[ccpp-table-properties] - name = opt_arg_scheme - type = scheme - dependencies = ccpp_kinds.F90 - -######################################################################## -[ccpp-arg-table] - name = opt_arg_scheme_timestep_init - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[nx] - standard_name = size_of_std_arg - long_name = size of std_arg - units = count - dimensions = () - type = integer - intent = in -[var] - standard_name = std_arg - long_name = mandatory variable - units = 1 - dimensions = (size_of_std_arg) - type = integer - intent = in -[opt_var] - standard_name = opt_arg - long_name = optional variable - units = 1 - dimensions = (size_of_std_arg) - type = integer - intent = out - optional = True -[opt_var_2] - standard_name = opt_arg_2 - long_name = optional variable with unit conversions - units = m - dimensions = (size_of_std_arg) - type = real - kind = kind_phys - intent = out - optional = True - -######################################################################## -[ccpp-arg-table] - name = opt_arg_scheme_run - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[nx] - standard_name = size_of_std_arg - long_name = size of std_arg - units = count - dimensions = () - type = integer - intent = in -[var] - standard_name = std_arg - long_name = mandatory variable - units = 1 - dimensions = (size_of_std_arg) - type = integer - intent = in -[opt_var] - standard_name = opt_arg - long_name = optional variable - units = 1 - dimensions = (size_of_std_arg) - type = integer - intent = inout - optional = True -[opt_var_2] - standard_name = opt_arg_2 - long_name = optional variable with unit conversions - units = m - dimensions = (size_of_std_arg) - type = real - kind = kind_phys - intent = inout - optional = True - -######################################################################## -[ccpp-arg-table] - name = opt_arg_scheme_timestep_finalize - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[nx] - standard_name = size_of_std_arg - long_name = size of std_arg - units = count - dimensions = () - type = integer - intent = in -[var] - standard_name = std_arg - long_name = mandatory variable - units = 1 - dimensions = (size_of_std_arg) - type = integer - intent = inout -[opt_var] - standard_name = opt_arg - long_name = optional variable - units = 1 - dimensions = (size_of_std_arg) - type = integer - intent = in - optional = True -[opt_var_2] - standard_name = opt_arg_2 - long_name = optional variable with unit conversions - units = m - dimensions = (size_of_std_arg) - type = real - kind = kind_phys - intent = inout - optional = True diff --git a/test_prebuild/test_opt_arg/run_test.sh b/test_prebuild/test_opt_arg/run_test.sh deleted file mode 100755 index cbe50e6d..00000000 --- a/test_prebuild/test_opt_arg/run_test.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e - -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --debug --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make 2>&1 | tee log.make -./test_opt_arg.x -cd .. -rm -fr build diff --git a/test_prebuild/test_opt_arg/suite_opt_arg_suite.xml b/test_prebuild/test_opt_arg/suite_opt_arg_suite.xml deleted file mode 100644 index e66514a4..00000000 --- a/test_prebuild/test_opt_arg/suite_opt_arg_suite.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - opt_arg_scheme - - - diff --git a/test_prebuild/test_track_variables.py b/test_prebuild/test_track_variables.py deleted file mode 100755 index 6f2b1963..00000000 --- a/test_prebuild/test_track_variables.py +++ /dev/null @@ -1,130 +0,0 @@ -#! /usr/bin/env python3 -""" ------------------------------------------------------------------------ - Description: Contains unit tests for ccpp_track_variables.py script - - Assumptions: Assumes user has correct environment for running ccpp_track_variables.py script. - This script should not be run directly, but rather invoked with pytest. - - Command line arguments: none - - Usage: pytest test_track_variables.py # run the unit tests ------------------------------------------------------------------------ -""" -import sys -import os -import pytest - -TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -SCRIPTS_DIR = os.path.abspath(os.path.join(TEST_DIR, os.pardir, "scripts")) -SAMPLE_FILES_DIR = "test_track_variables" -SUITE_FILE = f'{SAMPLE_FILES_DIR}/suite_TEST_SUITE.xml' -SMALL_SUITE_FILE = f'{SAMPLE_FILES_DIR}/suite_small_suite.xml' -CONFIG_FILE = f'{SAMPLE_FILES_DIR}/ccpp_prebuild_config.py' -if not os.path.exists(SCRIPTS_DIR): - raise ImportError(f"Cannot find scripts directory {SCRIPTS_DIR}") - -sys.path.append(SCRIPTS_DIR) - -from ccpp_track_variables import track_variables - -def test_successful_match(capsys): - """Tests whether test_track_variables.py produces expected output from sample suite and - metadata files for a case with a successful match (user provided a variable that exists - within the schemes specified by the test suite)""" - expected_output = """For suite test_track_variables/suite_small_suite.xml, the following schemes (in order for each group) use the variable air_pressure: -In group group1 - scheme_1_run (intent in) - scheme_1_run (intent in)""" - track_variables(SMALL_SUITE_FILE,SAMPLE_FILES_DIR,CONFIG_FILE,'air_pressure',False) - streams = capsys.readouterr() - expected_output_list = expected_output.splitlines() - streams_err_list = streams.err.splitlines() - for (err, expected) in zip(streams_err_list, expected_output_list): - assert err.strip() == expected.strip() - -def test_successful_match_with_subcycles(capsys): - """Tests whether test_track_variables.py produces expected output from sample suite and - metadata files for a case with a successful match (user provided a variable that exists - within the schemes specified by the test suite). In this case, the test suite file - contains subcycles, so the output should reflect this.""" - - expected_output = """For suite test_track_variables/suite_TEST_SUITE.xml, the following schemes (in order for each group) use the variable surface_air_pressure: -In group group1 - scheme_3_run (intent inout) - scheme_3_timestep_finalize (intent inout) - scheme_3_timestep_finalize (intent out) - scheme_4_run (intent in) - scheme_3_run (intent inout) - scheme_3_timestep_finalize (intent inout) - scheme_3_timestep_finalize (intent out) - scheme_4_run (intent in) -In group group2 - scheme_4_run (intent in) - scheme_4_run (intent in) - scheme_4_run (intent in)""" - track_variables(SUITE_FILE,SAMPLE_FILES_DIR,CONFIG_FILE,'surface_air_pressure',False) - streams = capsys.readouterr() - expected_output_list = expected_output.splitlines() - streams_err_list = streams.err.splitlines() - for (err, expected) in zip(streams_err_list, expected_output_list): - assert err.strip() == expected.strip() - - -def test_partial_match(capsys): - """Tests whether test_track_variables.py produces expected output from sample suite and - metadata files for a case with a partial match: user provided a variable that does not - exist in the test suite, but is a substring of one or more other variables that do - exist.""" - - expected_output = """Variable surface not found in any suites for sdf test_track_variables/suite_TEST_SUITE.xml - -ERROR:ccpp_track_variables:Variable surface not found in any suites for sdf test_track_variables/suite_TEST_SUITE.xml - -Did find partial matches that may be of interest: - -In scheme_2_init found variable(s) ['surface_emissivity_data_file'] -In scheme_2_run found variable(s) ['surface_roughness_length', 'surface_ground_temperature_for_radiation', 'surface_air_temperature_for_radiation', 'surface_skin_temperature_over_ice', 'baseline_surface_longwave_emissivity', 'surface_longwave_emissivity', 'surface_albedo_components', 'surface_albedo_for_diffused_shortwave_on_radiation_timestep'] -In scheme_3_init found variable(s) ['flag_for_mellor_yamada_nakanishi_niino_surface_layer_scheme'] -In scheme_3_timestep_init found variable(s) ['flag_for_mellor_yamada_nakanishi_niino_surface_layer_scheme'] -In scheme_3_run found variable(s) ['do_compute_surface_scalar_fluxes', 'do_compute_surface_diagnostics', 'surface_air_pressure', 'reference_air_pressure_normalized_by_surface_air_pressure'] -In scheme_3_timestep_finalize found variable(s) ['surface_air_pressure'] -In scheme_4_run found variable(s) ['surface_air_pressure'] -In scheme_B_run found variable(s) ['flag_nonzero_wet_surface_fraction', 'sea_surface_temperature', 'surface_skin_temperature_after_iteration_over_water'] -""" - track_variables(SUITE_FILE,SAMPLE_FILES_DIR,CONFIG_FILE,'surface',False) - streams = capsys.readouterr() - expected_output_list = expected_output.splitlines() - streams_err_list = streams.err.splitlines() - for (err, expected) in zip(streams_err_list, expected_output_list): - assert err.strip() == expected.strip() - - -def test_no_match(capsys): - """Tests whether test_track_variables.py produces expected output from sample suite and - metadata files for a case with no match (user provided a variable that does not exist - within the schemes specified by the test suite)""" - - expected_output = """Variable abc not found in any suites for sdf test_track_variables/suite_TEST_SUITE.xml - -ERROR:ccpp_track_variables:Variable abc not found in any suites for sdf test_track_variables/suite_TEST_SUITE.xml""" - track_variables(SUITE_FILE,SAMPLE_FILES_DIR,CONFIG_FILE,'abc',False) - streams = capsys.readouterr() - expected_output_list = expected_output.splitlines() - streams_err_list = streams.err.splitlines() - for (err, expected) in zip(streams_err_list, expected_output_list): - assert err.strip() == expected.strip() - - -def test_bad_config(capsys): - """Tests whether test_track_variables.py fails gracefully when provided a config file that does - not exist.""" - with pytest.raises(Exception) as excinfo: - track_variables(SUITE_FILE,SAMPLE_FILES_DIR,f'{SAMPLE_FILES_DIR}/nofile','abc',False) - assert str(excinfo.value) == "Call to import_config failed." - - -if __name__ == "__main__": - print("This test file is designed to be run with pytest; can not be run directly") - sys.exit(1) - diff --git a/test_prebuild/test_track_variables/ccpp_prebuild_config.py b/test_prebuild/test_track_variables/ccpp_prebuild_config.py deleted file mode 100755 index 83b20fd1..00000000 --- a/test_prebuild/test_track_variables/ccpp_prebuild_config.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python - -# CCPP prebuild config for CCPP track variables tool test - - -############################################################################### -# Definitions # -############################################################################### - -HOST_MODEL_IDENTIFIER = "TEST" - -# Add all files with metadata tables on the host model side and in CCPP, -# relative to basedir = top-level directory of host model. This includes -# kind and type definitions used in CCPP physics. Also add any internal -# dependencies of these files to the list. -VARIABLE_DEFINITION_FILES = [ - # actual variable definition files - '../src/ccpp_types.F90', - ] - -# Add all physics scheme files relative to basedir -SCHEME_FILES = [ ] - -# Default build dir, relative to current working directory, -# if not specified as command-line argument -DEFAULT_BUILD_DIR = 'build' - -# Auto-generated makefile/cmakefile snippets that contain all type definitions -TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' -TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' -TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' - -# Auto-generated makefile/cmakefile snippets that contain all schemes -SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' -SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' -SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' - -# Auto-generated makefile/cmakefile snippets that contain all caps -CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' -CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' -CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' - -# Directory where to put all auto-generated physics caps -CAPS_DIR = '{build_dir}' - -# Directory where the suite definition files are stored -SUITES_DIR = '.' - -# Optional arguments - only required for schemes that use -# optional arguments. ccpp_prebuild.py will throw an exception -# if it encounters a scheme subroutine with optional arguments -# if no entry is made here. Possible values are: 'all', 'none', -# or a list of standard_names: [ 'var1', 'var3' ]. -OPTIONAL_ARGUMENTS = {} - -# Directory where to write static API to -STATIC_API_DIR = '{build_dir}' -STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' -STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' - -# Directory for writing HTML pages generated from metadata files -METADATA_HTML_OUTPUT_DIR = '{build_dir}' - -# HTML document containing the model-defined CCPP variables -HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_BLOCKED_DATA.html' - -# LaTeX document containing the provided vs requested CCPP variables -LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_BLOCKED_DATA.tex' - diff --git a/test_prebuild/test_track_variables/scheme_1.meta b/test_prebuild/test_track_variables/scheme_1.meta deleted file mode 100644 index 8ee0800a..00000000 --- a/test_prebuild/test_track_variables/scheme_1.meta +++ /dev/null @@ -1,25 +0,0 @@ -[ccpp-table-properties] - name = scheme_1 - type = scheme - dependencies = ../../hooks/machine.F - -######################################################################## -[ccpp-arg-table] - name = scheme_1_run - type = scheme -[p_lay] - standard_name = air_pressure - long_name = mean layer pressure - units = Pa - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = in -[p_lev] - standard_name = air_pressure_at_interface - long_name = air pressure at model layer interfaces - units = Pa - dimensions = (horizontal_loop_extent,vertical_interface_dimension) - type = real - kind = kind_phys - intent = in diff --git a/test_prebuild/test_track_variables/scheme_2.meta b/test_prebuild/test_track_variables/scheme_2.meta deleted file mode 100644 index 2a03f096..00000000 --- a/test_prebuild/test_track_variables/scheme_2.meta +++ /dev/null @@ -1,87 +0,0 @@ -[ccpp-table-properties] - name = scheme_2 - type = scheme - dependencies_path = ../../ - dependencies = hooks/machine.F - -######################################################################## -[ccpp-arg-table] - name = scheme_2_init - type = scheme -[semis_file] - standard_name = surface_emissivity_data_file - long_name = surface emissivity data file for radiation - units = none - dimensions = () - type = character - kind = len=26 - intent = in - -######################################################################## -[ccpp-arg-table] - name = scheme_2_run - type = scheme -[zorl] - standard_name = surface_roughness_length - long_name = surface roughness length - units = cm - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[tsfg] - standard_name = surface_ground_temperature_for_radiation - long_name = surface ground temperature for radiation - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[tsfa] - standard_name = surface_air_temperature_for_radiation - long_name = lowest model layer air temperature for radiation - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[tisfc] - standard_name = surface_skin_temperature_over_ice - long_name = surface_skin_temperature_over_ice - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[semisbase] - standard_name = baseline_surface_longwave_emissivity - long_name = baseline surface lw emissivity in fraction - units = frac - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[semis] - standard_name = surface_longwave_emissivity - long_name = surface lw emissivity in fraction - units = frac - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[sfcalb] - standard_name = surface_albedo_components - long_name = surface albedo IR/UV/VIS components - units = frac - dimensions = (horizontal_loop_extent,number_of_components_for_surface_albedo) - type = real - kind = kind_phys - intent = inout -[sfc_alb_dif] - standard_name = surface_albedo_for_diffused_shortwave_on_radiation_timestep - long_name = mean surface diffused sw albedo - units = frac - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout diff --git a/test_prebuild/test_track_variables/scheme_3.meta b/test_prebuild/test_track_variables/scheme_3.meta deleted file mode 100644 index f7d568dd..00000000 --- a/test_prebuild/test_track_variables/scheme_3.meta +++ /dev/null @@ -1,143 +0,0 @@ -[ccpp-table-properties] - name = scheme_3 - type = scheme - dependencies = ../../hooks/machine.F,../../hooks/physcons.F90,module_sf_mynn.F90 -######################################################################## -[ccpp-arg-table] - name = scheme_3_init - type = scheme -[do_mynnsfclay] - standard_name = flag_for_mellor_yamada_nakanishi_niino_surface_layer_scheme - long_name = flag to activate MYNN surface layer - units = flag - dimensions = () - type = logical - intent = in -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out - -######################################################################## -[ccpp-arg-table] - name = scheme_3_timestep_init - type = scheme -[do_mynnsfclay] - standard_name = flag_for_mellor_yamada_nakanishi_niino_surface_layer_scheme - long_name = flag to activate MYNN surface layer - units = flag - dimensions = () - type = logical - intent = in -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out - -######################################################################## -[ccpp-arg-table] - name = scheme_3_run - type = scheme -[sfclay_compute_flux] - standard_name = do_compute_surface_scalar_fluxes - long_name = flag for computing surface scalar fluxes in mynnsfclay - units = flag - dimensions = () - type = logical - intent = in -[sfclay_compute_diag] - standard_name = do_compute_surface_diagnostics - long_name = flag for computing surface diagnostics in mynnsfclay - units = flag - dimensions = () - type = logical - intent = in -[prsl] - standard_name = air_pressure - long_name = mean layer pressure - units = Pa - dimensions = (horizontal_loop_extent,vertical_layer_dimension) - type = real - kind = kind_phys - intent = inout -[ps] - standard_name = surface_air_pressure - long_name = surface pressure - units = Pa - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[pr_ps] - standard_name = reference_air_pressure_normalized_by_surface_air_pressure - long_name = reference pressure normalized by surface pressure - units = cm - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = out - -######################################################################## -[ccpp-arg-table] - name = scheme_3_timestep_finalize - type = scheme -[prsl] - standard_name = air_pressure - long_name = mean layer pressure - units = Pa - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - intent = out -[ps] - standard_name = surface_air_pressure - long_name = surface pressure - units = Pa - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = inout - -######################################################################## -[ccpp-arg-table] - name = scheme_3_timestep_finalize - type = scheme -[prsl] - standard_name = air_pressure - long_name = mean layer pressure - units = Pa - dimensions = (horizontal_dimension,vertical_layer_dimension) - type = real - kind = kind_phys - intent = out -[ps] - standard_name = surface_air_pressure - long_name = surface pressure - units = Pa - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - intent = out - diff --git a/test_prebuild/test_track_variables/scheme_4.meta b/test_prebuild/test_track_variables/scheme_4.meta deleted file mode 100644 index 5464693b..00000000 --- a/test_prebuild/test_track_variables/scheme_4.meta +++ /dev/null @@ -1,31 +0,0 @@ -[ccpp-table-properties] - name = scheme_4 - type = scheme - -######################################################################## -[ccpp-arg-table] - name = scheme_4_run - type = scheme -[ps] - standard_name = surface_air_pressure - long_name = surface pressure - units = Pa - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test_prebuild/test_track_variables/scheme_A.meta b/test_prebuild/test_track_variables/scheme_A.meta deleted file mode 100644 index 4fc6118c..00000000 --- a/test_prebuild/test_track_variables/scheme_A.meta +++ /dev/null @@ -1,31 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = scheme_A - type = scheme - -######################################################################## -[ccpp-arg-table] - name = scheme_A_run - type = scheme -[im] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - dimensions = () - type = integer - intent = in -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out diff --git a/test_prebuild/test_track_variables/scheme_B.meta b/test_prebuild/test_track_variables/scheme_B.meta deleted file mode 100644 index 3bd0f070..00000000 --- a/test_prebuild/test_track_variables/scheme_B.meta +++ /dev/null @@ -1,48 +0,0 @@ -######################################################################## -[ccpp-table-properties] - name = scheme_B - type = scheme - dependencies = ../../hooks/machine.F,module_nst_parameters.f90,module_nst_water_prop.f90 - -######################################################################## -[ccpp-arg-table] - name = scheme_B_run - type = scheme -[im] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - dimensions = () - type = integer - intent = in -[wet] - standard_name = flag_nonzero_wet_surface_fraction - long_name = flag indicating presence of some ocean or lake surface area fraction - units = flag - dimensions = (horizontal_loop_extent) - type = logical - intent = in -[tgice] - standard_name = freezing_point_temperature_of_seawater - long_name = freezing point temperature of seawater - units = K - dimensions = () - type = real - kind = kind_phys - intent = in -[tsfco] - standard_name = sea_surface_temperature - long_name = sea surface temperature - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = in -[tsurf_wat] - standard_name = surface_skin_temperature_after_iteration_over_water - long_name = surface skin temperature after iteration over water - units = K - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout diff --git a/test_prebuild/test_track_variables/suite_TEST_SUITE.xml b/test_prebuild/test_track_variables/suite_TEST_SUITE.xml deleted file mode 100644 index 92d7af46..00000000 --- a/test_prebuild/test_track_variables/suite_TEST_SUITE.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - - - scheme_1 - scheme_2 - - - scheme_3 - scheme_4 - - - - - scheme_1 - scheme_A - - - scheme_B - scheme_4 - - - diff --git a/test_prebuild/test_track_variables/suite_small_suite.xml b/test_prebuild/test_track_variables/suite_small_suite.xml deleted file mode 100644 index 057936b3..00000000 --- a/test_prebuild/test_track_variables/suite_small_suite.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - scheme_1 - - - - - scheme_4 - - - diff --git a/test_prebuild/test_unit_conv/CMakeLists.txt b/test_prebuild/test_unit_conv/CMakeLists.txt deleted file mode 100644 index 97c1730c..00000000 --- a/test_prebuild/test_unit_conv/CMakeLists.txt +++ /dev/null @@ -1,98 +0,0 @@ -#------------------------------------------------------------------------------ -cmake_minimum_required(VERSION 3.10) - -project(ccpp_unit_conv - VERSION 1.0.0 - LANGUAGES Fortran) - -#------------------------------------------------------------------------------ -# Request a static build -option(BUILD_SHARED_LIBS "Build a shared library" OFF) - -#------------------------------------------------------------------------------ -# Set MPI flags for C/C++/Fortran with MPI F08 interface -find_package(MPI REQUIRED Fortran) -if(NOT MPI_Fortran_HAVE_F08_MODULE) - message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") -endif() - -#------------------------------------------------------------------------------ -# Set the sources: physics type definitions -set(TYPEDEFS $ENV{CCPP_TYPEDEFS}) -if(TYPEDEFS) - message(STATUS "Got CCPP TYPEDEFS from environment variable: ${TYPEDEFS}") -else(TYPEDEFS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) - message(STATUS "Got CCPP TYPEDEFS from cmakefile include file: ${TYPEDEFS}") -endif(TYPEDEFS) - -# Generate list of Fortran modules from the CCPP type -# definitions that need need to be installed -foreach(typedef_module ${TYPEDEFS}) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${typedef_module}) -endforeach() - -#------------------------------------------------------------------------------ -# Set the sources: physics schemes -set(SCHEMES $ENV{CCPP_SCHEMES}) -if(SCHEMES) - message(STATUS "Got CCPP SCHEMES from environment variable: ${SCHEMES}") -else(SCHEMES) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) - message(STATUS "Got CCPP SCHEMES from cmakefile include file: ${SCHEMES}") -endif(SCHEMES) - -# Set the sources: physics scheme caps -set(CAPS $ENV{CCPP_CAPS}) -if(CAPS) - message(STATUS "Got CCPP CAPS from environment variable: ${CAPS}") -else(CAPS) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) - message(STATUS "Got CCPP CAPS from cmakefile include file: ${CAPS}") -endif(CAPS) - -# Set the sources: physics scheme caps -set(API $ENV{CCPP_API}) -if(API) - message(STATUS "Got CCPP API from environment variable: ${API}") -else(API) - include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) - message(STATUS "Got CCPP API from cmakefile include file: ${API}") -endif(API) - -set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -O0 -fno-unsafe-math-optimizations -frounding-math -fsignaling-nans -ffpe-trap=invalid,zero,overflow -fbounds-check -ggdb -fbacktrace -ffree-line-length-none") - -#------------------------------------------------------------------------------ -add_library(ccpp_unit_conv STATIC ${SCHEMES} ${CAPS} ${API}) -target_link_libraries(ccpp_unit_conv PRIVATE MPI::MPI_Fortran) -# Generate list of Fortran modules from defined sources -foreach(source_f90 ${CAPS} ${API}) - get_filename_component(tmp_source_f90 ${source_f90} NAME) - string(REGEX REPLACE ".F90" ".mod" tmp_module_f90 ${tmp_source_f90}) - string(TOLOWER ${tmp_module_f90} module_f90) - list(APPEND MODULES_F90 ${CMAKE_CURRENT_BINARY_DIR}/${module_f90}) -endforeach() - -set_target_properties(ccpp_unit_conv PROPERTIES VERSION ${PROJECT_VERSION} - SOVERSION ${PROJECT_VERSION_MAJOR}) - -add_executable(test_unit_conv.x main.F90) -add_dependencies(test_unit_conv.x ccpp_unit_conv) -target_link_libraries(test_unit_conv.x ccpp_unit_conv) -set_target_properties(test_unit_conv.x PROPERTIES LINKER_LANGUAGE Fortran) - -# Define where to install the library -install(TARGETS ccpp_unit_conv - EXPORT ccpp_unit_conv-targets - ARCHIVE DESTINATION lib - LIBRARY DESTINATION lib - RUNTIME DESTINATION lib -) -# Export our configuration -install(EXPORT ccpp_unit_conv-targets - FILE ccpp_unit_conv-config.cmake - DESTINATION lib/cmake -) -# Define where to install the C headers and Fortran modules -#install(FILES ${HEADERS_C} DESTINATION include) -install(FILES ${MODULES_F90} DESTINATION include) diff --git a/test_prebuild/test_unit_conv/README.md b/test_prebuild/test_unit_conv/README.md deleted file mode 100644 index 17ca1c58..00000000 --- a/test_prebuild/test_unit_conv/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# How to build the unit conv test - -1. Set compiler environment as appropriate for your system -2. Run the following commands: -``` -cd test_prebuild/test_unit_conv/ -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make 2>&1 | tee log.make -./test_unit_conv.x -# On systems where linking against the MPI library requires a parallel launcher, -# use 'mpirun -np 1 ./test_unit_conv.x' or 'srun -n 1 ./test_unit_conv.x' etc. -``` diff --git a/test_prebuild/test_unit_conv/ccpp_kinds.F90 b/test_prebuild/test_unit_conv/ccpp_kinds.F90 deleted file mode 100644 index a07ded9b..00000000 --- a/test_prebuild/test_unit_conv/ccpp_kinds.F90 +++ /dev/null @@ -1,13 +0,0 @@ -module ccpp_kinds - - !! \section arg_table_ccpp_kinds - !! \htmlinclude ccpp_kinds.html - !! - - use iso_fortran_env, only: real64 - - implicit none - - integer, parameter :: kind_phys = real64 - -end module ccpp_kinds diff --git a/test_prebuild/test_unit_conv/ccpp_kinds.meta b/test_prebuild/test_unit_conv/ccpp_kinds.meta deleted file mode 100644 index 0e95702e..00000000 --- a/test_prebuild/test_unit_conv/ccpp_kinds.meta +++ /dev/null @@ -1,15 +0,0 @@ -[ccpp-table-properties] - name = ccpp_kinds - type = module - dependencies = - -######################################################################## -[ccpp-arg-table] - name = ccpp_kinds - type = module -[kind_phys] - standard_name = kind_phys - long_name = definition of kind_phys - units = none - dimensions = () - type = integer diff --git a/test_prebuild/test_unit_conv/ccpp_prebuild_config.py b/test_prebuild/test_unit_conv/ccpp_prebuild_config.py deleted file mode 100755 index 3ee45942..00000000 --- a/test_prebuild/test_unit_conv/ccpp_prebuild_config.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python - -# CCPP prebuild config for GFDL Finite-Volume Cubed-Sphere Model (FV3) - -############################################################################### -# Definitions # -############################################################################### - -HOST_MODEL_IDENTIFIER = "FV3" - -# Add all files with metadata tables on the host model side and in CCPP, -# relative to basedir = top-level directory of host model. This includes -# kind and type definitions used in CCPP physics. Also add any internal -# dependencies of these files to the list. -VARIABLE_DEFINITION_FILES = [ - # actual variable definition files - '../../src/ccpp_types.F90', - 'ccpp_kinds.F90', - 'data.F90', - ] - -TYPEDEFS_NEW_METADATA = { - 'ccpp_types' : { - 'ccpp_t' : 'cdata', - 'ccpp_types' : '', - }, - 'data' : { - 'data' : '', - }, - } - -# Add all physics scheme files relative to basedir -SCHEME_FILES = [ - 'unit_conv_scheme_1.F90', - 'unit_conv_scheme_2.F90', - ] - -# Default build dir, relative to current working directory, -# if not specified as command-line argument -DEFAULT_BUILD_DIR = 'build' - -# Auto-generated makefile/cmakefile snippets that contain all type definitions -TYPEDEFS_MAKEFILE = '{build_dir}/CCPP_TYPEDEFS.mk' -TYPEDEFS_CMAKEFILE = '{build_dir}/CCPP_TYPEDEFS.cmake' -TYPEDEFS_SOURCEFILE = '{build_dir}/CCPP_TYPEDEFS.sh' - -# Auto-generated makefile/cmakefile snippets that contain all schemes -SCHEMES_MAKEFILE = '{build_dir}/CCPP_SCHEMES.mk' -SCHEMES_CMAKEFILE = '{build_dir}/CCPP_SCHEMES.cmake' -SCHEMES_SOURCEFILE = '{build_dir}/CCPP_SCHEMES.sh' - -# Auto-generated makefile/cmakefile snippets that contain all caps -CAPS_MAKEFILE = '{build_dir}/CCPP_CAPS.mk' -CAPS_CMAKEFILE = '{build_dir}/CCPP_CAPS.cmake' -CAPS_SOURCEFILE = '{build_dir}/CCPP_CAPS.sh' - -# Directory where to put all auto-generated physics caps -CAPS_DIR = '{build_dir}' - -# Directory where the suite definition files are stored -SUITES_DIR = '.' - -# Optional arguments - only required for schemes that use -# optional arguments. ccpp_prebuild.py will throw an exception -# if it encounters a scheme subroutine with optional arguments -# if no entry is made here. Possible values are: 'all', 'none', -# or a list of standard_names: [ 'var1', 'var3' ]. -OPTIONAL_ARGUMENTS = {} - -# Directory where to write static API to -STATIC_API_DIR = '{build_dir}' -STATIC_API_CMAKEFILE = '{build_dir}/CCPP_API.cmake' -STATIC_API_SOURCEFILE = '{build_dir}/CCPP_API.sh' - -# Directory for writing HTML pages generated from metadata files -METADATA_HTML_OUTPUT_DIR = '{build_dir}' - -# HTML document containing the model-defined CCPP variables -HTML_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_UNIT_CONV.html' - -# LaTeX document containing the provided vs requested CCPP variables -LATEX_VARTABLE_FILE = '{build_dir}/CCPP_VARIABLES_UNIT_CONV.tex' diff --git a/test_prebuild/test_unit_conv/data.F90 b/test_prebuild/test_unit_conv/data.F90 deleted file mode 100644 index ad6db921..00000000 --- a/test_prebuild/test_unit_conv/data.F90 +++ /dev/null @@ -1,24 +0,0 @@ -module data - - !! \section arg_table_data Argument Table - !! \htmlinclude data.html - !! - use ccpp_kinds, only : kind_phys - use ccpp_types, only: ccpp_t - - implicit none - - private - - public ncols, ncolsrun, nspecies - public cdata, data_array, data_array2, opt_array_flag - - integer, parameter :: ncols = 4 - integer, parameter :: ncolsrun = ncols - integer, parameter :: nspecies = 2 - type(ccpp_t), target :: cdata - real(kind=kind_phys), dimension(1:ncols, 1:nspecies) :: data_array - real(kind=kind_phys), dimension(1:ncols) :: data_array2 - logical :: opt_array_flag - -end module data diff --git a/test_prebuild/test_unit_conv/data.meta b/test_prebuild/test_unit_conv/data.meta deleted file mode 100644 index 9c9ea5e7..00000000 --- a/test_prebuild/test_unit_conv/data.meta +++ /dev/null @@ -1,66 +0,0 @@ -[ccpp-table-properties] - name = data - type = module - dependencies = -[ccpp-arg-table] - name = data - type = module -[cdata] - standard_name = ccpp_t_instance - long_name = instance of derived data type ccpp_t - units = DDT - dimensions = () - type = ccpp_t -[ncols] - standard_name = horizontal_dimension - long_name = horizontal dimension - units = count - dimensions = () - type = integer -[ncolsrun] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - dimensions = () - type = integer -[nspecies] - standard_name = number_of_species - long_name = number of species in data array - units = count - dimensions = () - type = integer -[data_array] - standard_name = data_array_all_species - long_name = data array in module - units = m - dimensions = (horizontal_dimension, number_of_species) - type = real - kind = kind_phys -[data_array(:,2)] - standard_name = data_array - long_name = data array in module - units = m - dimensions = (horizontal_dimension) - type = real - kind = kind_phys -[data_array2] - standard_name = data_array2 - long_name = data array 2 in module - units = m2 s-2 - dimensions = (horizontal_dimension) - type = real - kind = kind_phys -[opt_array_flag] - standard_name = flag_for_opt_array - long_name = flag for passing optional data array - units = 1 - dimensions = () - type = logical -[data_array(:,1)] - standard_name = data_array_opt - long_name = optional data array in km - units = m - dimensions = (horizontal_dimension) - type = real - kind = kind_phys - active = (flag_for_opt_array) diff --git a/test_prebuild/test_unit_conv/main.F90 b/test_prebuild/test_unit_conv/main.F90 deleted file mode 100644 index f414eeda..00000000 --- a/test_prebuild/test_unit_conv/main.F90 +++ /dev/null @@ -1,97 +0,0 @@ -program test_unit_conv - - use, intrinsic :: iso_fortran_env, only: error_unit - - use ccpp_types, only: ccpp_t - use data, only: ncols, & - nspecies - use data, only: cdata, & - data_array, & - data_array2, & - opt_array_flag - - use ccpp_static_api, only: ccpp_physics_init, & - ccpp_physics_timestep_init, & - ccpp_physics_run, & - ccpp_physics_timestep_finalize, & - ccpp_physics_finalize - - implicit none - - character(len=*), parameter :: ccpp_suite = 'unit_conv_suite' - integer :: ierr - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - ! For physics running over the entire domain, - ! ccpp_thread_number and ccpp_chunk_number are - ! set to 1, indicating that arrays are to be sent - ! following their dimension specification in the - ! metadata (must match horizontal_dimension). - cdata%thrd_no = 1 - cdata%chunk_no = 1 - cdata%thrd_cnt = 1 - - data_array = 1.0_8 - data_array2 = 42.0_8 - opt_array_flag = .true. - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep init step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_timestep_init(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics run step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_run(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_run:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics timestep finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_timestep_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_finalize:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ! CCPP physics finalize step ! - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - - call ccpp_physics_finalize(cdata, suite_name=trim(ccpp_suite), ierr=ierr) - if (ierr/=0) then - write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" - write(error_unit, '(a)') trim(cdata%errmsg) - stop 1 - end if - -contains - -end program test_unit_conv diff --git a/test_prebuild/test_unit_conv/run_test.sh b/test_prebuild/test_unit_conv/run_test.sh deleted file mode 100755 index ab7e8c31..00000000 --- a/test_prebuild/test_unit_conv/run_test.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -e - -rm -fr build -mkdir build -../../scripts/ccpp_prebuild.py --verbose --debug --config=ccpp_prebuild_config.py --builddir=build -cd build -cmake .. 2>&1 | tee log.cmake -make VERBOSE=1 -j1 2>&1 | tee log.make -./test_unit_conv.x -cd .. -rm -fr build diff --git a/test_prebuild/test_unit_conv/suite_unit_conv_suite.xml b/test_prebuild/test_unit_conv/suite_unit_conv_suite.xml deleted file mode 100644 index 68d90109..00000000 --- a/test_prebuild/test_unit_conv/suite_unit_conv_suite.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - unit_conv_scheme_1 - unit_conv_scheme_2 - unit_conv_scheme_1 - - - diff --git a/test_prebuild/test_unit_conv/unit_conv_scheme_1.F90 b/test_prebuild/test_unit_conv/unit_conv_scheme_1.F90 deleted file mode 100644 index 42df267e..00000000 --- a/test_prebuild/test_unit_conv/unit_conv_scheme_1.F90 +++ /dev/null @@ -1,70 +0,0 @@ -!>\file unit_conv_scheme_1.F90 -!! This file contains a unit_conv_scheme_1 CCPP scheme that does nothing -!! except requesting the minimum, mandatory variables. - -module unit_conv_scheme_1 - - use, intrinsic :: iso_fortran_env, only: error_unit - use ccpp_kinds, only : kind_phys - implicit none - - private - public :: unit_conv_scheme_1_run - - !! This is for unit testing only - real(kind=kind_phys), parameter :: target_value = 1.0_kind_phys - real(kind=kind_phys), parameter :: target_value2 = 42.0_kind_phys - -contains - - !! \section arg_table_unit_conv_scheme_1_run Argument Table - !! \htmlinclude unit_conv_scheme_1_run.html - !! - subroutine unit_conv_scheme_1_run(data_array, data_array2, data_array_opt, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - real(kind=kind_phys), intent(inout) :: data_array(:) - real(kind=kind_phys), intent(inout) :: data_array2(:) - real(kind=kind_phys), intent(inout), optional :: data_array_opt(:) - - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check values in data array - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_1_run: checking min/max values of data array to be approximately ', target_value - if (minval(data_array) < 0.99 * target_value .or. maxval(data_array) > 1.01 * target_value) then - write(errmsg, '(3(a,e12.4),a)') & - "Error in unit_conv_scheme_1_run, expected values for data_array of approximately ", & - target_value, " but got [ ", minval(data_array), " : ", maxval(data_array), " ]" - errflg = 1 - return - end if - ! Check values in data array2 - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_1_run: checking min/max values of data array 2 to be approximately ', target_value2 - if (minval(data_array2) < 0.99 * target_value2 .or. maxval(data_array2) > 1.01 * target_value2) then - write(errmsg, '(3(a,e12.4),a)') & - "Error in unit_conv_scheme_1_run, expected values for data array 2 of approximately ", & - target_value2, " but got [ ", minval(data_array2), " : ", maxval(data_array2), " ]" - errflg = 1 - return - end if - ! Check for presence of optional data array, then check its values - write(error_unit, '(a)') 'In unit_conv_scheme_1_run: checking for presence of optional data array' - if (.not. present(data_array_opt)) then - write(error_unit, '(a)') 'Error in unit_conv_scheme_1_run, optional data array expected but not present' - errflg = 1 - return - end if - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_1_run: checking min/max values of optional data array to be approximately ', target_value - if (minval(data_array_opt) < 0.99 * target_value .or. maxval(data_array_opt) > 1.01 * target_value) then - write(errmsg, '(3(a,e12.4),a)') 'Error in unit_conv_scheme_1_run, expected values of approximately ', & - target_value, ' but got [ ', minval(data_array_opt), ' : ', maxval(data_array_opt), ' ]' - errflg = 1 - return - end if - end subroutine unit_conv_scheme_1_run - -end module unit_conv_scheme_1 diff --git a/test_prebuild/test_unit_conv/unit_conv_scheme_1.meta b/test_prebuild/test_unit_conv/unit_conv_scheme_1.meta deleted file mode 100644 index befb19bd..00000000 --- a/test_prebuild/test_unit_conv/unit_conv_scheme_1.meta +++ /dev/null @@ -1,49 +0,0 @@ -[ccpp-table-properties] - name = unit_conv_scheme_1 - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = unit_conv_scheme_1_run - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[data_array] - standard_name = data_array - long_name = data array in m - units = m - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[data_array2] - standard_name = data_array2 - long_name = data array in J kg-1 - units = J kg-1 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[data_array_opt] - standard_name = data_array_opt - long_name = optional data array in m - units = m - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout - optional = True diff --git a/test_prebuild/test_unit_conv/unit_conv_scheme_2.F90 b/test_prebuild/test_unit_conv/unit_conv_scheme_2.F90 deleted file mode 100644 index 76f6ef2f..00000000 --- a/test_prebuild/test_unit_conv/unit_conv_scheme_2.F90 +++ /dev/null @@ -1,69 +0,0 @@ -!>\file unit_conv_scheme_2.F90 -!! This file contains a unit_conv_scheme_2 CCPP scheme that does nothing -!! except requesting the minimum, mandatory variables. - -module unit_conv_scheme_2 - - use, intrinsic :: iso_fortran_env, only: error_unit - use ccpp_kinds, only : kind_phys - implicit none - - private - public :: unit_conv_scheme_2_run - - !! This is for unit testing only - real(kind=kind_phys), parameter :: target_value = 1.0E-3_kind_phys - real(kind=kind_phys), parameter :: target_value2 = 42.0_kind_phys - -contains - - !! \section arg_table_unit_conv_scheme_2_run Argument Table - !! \htmlinclude unit_conv_scheme_2_run.html - !! - subroutine unit_conv_scheme_2_run(data_array, data_array2, data_array_opt, errmsg, errflg) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg - real(kind=kind_phys), intent(inout) :: data_array(:) - real(kind=kind_phys), intent(inout) :: data_array2(:) - real(kind=kind_phys), intent(inout), optional :: data_array_opt(:) - - ! Initialize CCPP error handling variables - errmsg = '' - errflg = 0 - ! Check values in data array - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_2_run: checking min/max values of data array to be approximately ', target_value - if (minval(data_array) < 0.99 * target_value .or. maxval(data_array) > 1.01 * target_value) then - write(errmsg, '(3(a,e12.4),a)') 'Error in unit_conv_scheme_2_run, expected values of approximately ', & - target_value, ' but got [ ', minval(data_array), ' : ', maxval(data_array), ' ]' - errflg = 1 - return - end if - ! Check values in data array2 - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_2_run: checking min/max values of data array 2 to be approximately ', target_value2 - if (minval(data_array2) < 0.99 * target_value2 .or. maxval(data_array2) > 1.01 * target_value2) then - write(errmsg, '(3(a,e12.4),a)') & - "Error in unit_conv_scheme_2_run, expected values for data array 2 of approximately ", & - target_value2, " but got [ ", minval(data_array2), " : ", maxval(data_array2), " ]" - errflg = 1 - return - end if - ! Check for presence of optional data array, then check its values - write(error_unit, '(a)') 'In unit_conv_scheme_2_run: checking for presence of optional data array' - if (.not. present(data_array_opt)) then - write(error_unit, '(a)') 'Error in unit_conv_scheme_2_run, optional data array expected but not present' - errflg = 1 - return - end if - write(error_unit, '(a,e12.4)') & - 'In unit_conv_scheme_2_run: checking min/max values of optional data array to be approximately ', target_value - if (minval(data_array_opt) < 0.99 * target_value .or. maxval(data_array_opt) > 1.01 * target_value) then - write(errmsg, '(3(a,e12.4),a)') 'Error in unit_conv_scheme_2_run, expected values of approximately ', & - target_value, ' but got [ ', minval(data_array_opt), ' : ', maxval(data_array_opt), ' ]' - errflg = 1 - return - end if - end subroutine unit_conv_scheme_2_run - -end module unit_conv_scheme_2 diff --git a/test_prebuild/test_unit_conv/unit_conv_scheme_2.meta b/test_prebuild/test_unit_conv/unit_conv_scheme_2.meta deleted file mode 100644 index 68e4b063..00000000 --- a/test_prebuild/test_unit_conv/unit_conv_scheme_2.meta +++ /dev/null @@ -1,49 +0,0 @@ -[ccpp-table-properties] - name = unit_conv_scheme_2 - type = scheme - dependencies = - -######################################################################## -[ccpp-arg-table] - name = unit_conv_scheme_2_run - type = scheme -[errmsg] - standard_name = ccpp_error_message - long_name = error message for error handling in CCPP - units = none - dimensions = () - type = character - kind = len=* - intent = out -[errflg] - standard_name = ccpp_error_code - long_name = error code for error handling in CCPP - units = 1 - dimensions = () - type = integer - intent = out -[data_array] - standard_name = data_array - long_name = data array in km - units = km - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[data_array2] - standard_name = data_array2 - long_name = data array in m+2 s-2 - units = m+2 s-2 - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout -[data_array_opt] - standard_name = data_array_opt - long_name = optional data array in km - units = km - dimensions = (horizontal_loop_extent) - type = real - kind = kind_phys - intent = inout - optional = True diff --git a/test_prebuild/unit_tests/run_tests.sh b/test_prebuild/unit_tests/run_tests.sh deleted file mode 100755 index 3b80c071..00000000 --- a/test_prebuild/unit_tests/run_tests.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -THIS_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -FRAMEWORK_DIR="${THIS_DIR}/../.." - -export PYTHONPATH="${FRAMEWORK_DIR}/scripts/parse_tools:${FRAMEWORK_DIR}/scripts:${PYTHONPATH}" - -set -ex -python3 ./test_metadata_parser.py -python3 ./test_mkstatic.py diff --git a/test_prebuild/unit_tests/test_metadata_parser.py b/test_prebuild/unit_tests/test_metadata_parser.py deleted file mode 100644 index b4e36dfc..00000000 --- a/test_prebuild/unit_tests/test_metadata_parser.py +++ /dev/null @@ -1,57 +0,0 @@ -import logging -import os -import sys - -from parse_checkers import registered_fortran_ddt_names -from metadata_table import MetadataTable, parse_metadata_file, Var -from framework_env import CCPPFrameworkEnv - -example_table = """ -[ccpp-table-properties] - name = - type = scheme - dependencies_path = path - dependencies = a.f,b.f - -[ccpp-arg-table] - name = - type = scheme -[ im ] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent, start at 1 - units = index - type = integer - dimensions = () - intent = in -""" - - -def test_MetadataTable_parse_table(tmpdir): - path = str(tmpdir.join("table.meta")) - with open(path, "w") as f: - f.write(example_table) - - dummy_run_env = CCPPFrameworkEnv(None, ndict={'host_files':'', - 'scheme_files':'', - 'suites':''}) - - - metadata_headers = parse_metadata_file(path, known_ddts=registered_fortran_ddt_names(), - run_env=dummy_run_env) - - # check metadata header - assert len(metadata_headers) == 1 - metadata_header = metadata_headers[0] - assert metadata_header.table_name == "" - assert metadata_header.table_type == "scheme" - assert metadata_header.dependencies_path == "path" - assert metadata_header.dependencies == [os.path.join(tmpdir, metadata_header.dependencies_path,"a.f"), os.path.join(tmpdir, metadata_header.dependencies_path,"b.f")] - - # check metadata section - assert len(metadata_header.sections()) == 1 - metadata_section = metadata_header.sections()[0] - assert metadata_section.name == "" - assert metadata_section.ptype == "scheme" - (im_data,) = metadata_section.variable_list() - assert isinstance(im_data, Var) - assert im_data.get_dimensions() == [] diff --git a/test_prebuild/unit_tests/test_mkstatic.py b/test_prebuild/unit_tests/test_mkstatic.py deleted file mode 100644 index af66d575..00000000 --- a/test_prebuild/unit_tests/test_mkstatic.py +++ /dev/null @@ -1,22 +0,0 @@ -from mkstatic import extract_parents_and_indices_from_local_name - -import pytest - - -@pytest.mark.parametrize( - "input_,expected", - [ - ( - r"Atm(mytile)%q(:,:,:,Atm2(mytile2)%graupel)", - ("Atm", ["Atm2", "mytile", "mytile2"]), - ), - (r"Atm(mytile)%q(:,:,:,simple_ind)", ("Atm", ["mytile", "simple_ind"])), - (r"Atm%q(random)", ("Atm", ["random"])), - ], -) -def test_extract_parents_and_indices_from_local_name(input_, expected): - expected_parent, expected_inputs = expected - parent, inputs = extract_parents_and_indices_from_local_name(input_) - - assert parent == expected_parent - assert set(inputs) == set(expected_inputs) From 9a9fbc1321cc43d825b86aca67216bfe39724fa5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 07:25:48 -0600 Subject: [PATCH 02/74] Update GitHub workflows for capgen-ng --- .../{ => TMP_OFF}/fortran-formatting.yaml | 0 ..._unit_tests.yaml => end-to-end-tests.yaml} | 42 ++++++------ .github/workflows/prebuild.yaml | 68 ------------------- .github/workflows/python.yaml | 56 --------------- .github/workflows/unit-tests.yaml | 34 ++++++++++ 5 files changed, 55 insertions(+), 145 deletions(-) rename .github/workflows/{ => TMP_OFF}/fortran-formatting.yaml (100%) rename .github/workflows/{capgen_unit_tests.yaml => end-to-end-tests.yaml} (55%) delete mode 100644 .github/workflows/prebuild.yaml delete mode 100644 .github/workflows/python.yaml create mode 100644 .github/workflows/unit-tests.yaml diff --git a/.github/workflows/fortran-formatting.yaml b/.github/workflows/TMP_OFF/fortran-formatting.yaml similarity index 100% rename from .github/workflows/fortran-formatting.yaml rename to .github/workflows/TMP_OFF/fortran-formatting.yaml diff --git a/.github/workflows/capgen_unit_tests.yaml b/.github/workflows/end-to-end-tests.yaml similarity index 55% rename from .github/workflows/capgen_unit_tests.yaml rename to .github/workflows/end-to-end-tests.yaml index 4b8598f0..ea6bee2a 100644 --- a/.github/workflows/capgen_unit_tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -1,4 +1,4 @@ -name: Capgen Unit Tests +name: capgen-ng end-to-end tests on: workflow_dispatch: @@ -10,7 +10,7 @@ concurrency: cancel-in-progress: true jobs: - unit_tests: + end-to-end-tests: strategy: matrix: os: [ubuntu-22.04] @@ -37,27 +37,27 @@ jobs: - name: Build the framework run: | - cmake --fresh -S. -B./build -DOPENMP=ON -DCCPP_FRAMEWORK_ENABLE_TESTS=ON + cmake --fresh -S./end-to-end-tests -B./build cd build make - - name: Run unit tests + - name: Run end-to-end tests run: | cd build - ctest --rerun-failed --output-on-failure . --verbose - - - name: Run python tests - run: | - BUILD_DIR=./build \ - PYTHONPATH=test/:scripts/ \ - pytest \ - test/capgen_test/capgen_test_reports.py \ - test/advection_test/advection_test_reports.py \ - test/ddthost_test/ddthost_test_reports.py \ - test/var_compatibility_test/var_compatibility_test_reports.py - - - name: Run Fortran to metadata test - run: cd test && ./test_fortran_to_metadata.sh - - - name: Run offline metadata parser test - run: cd test && ./test_offline_metadata_checker.sh + ctest --rerun-failed --output-on-failure . + +# - name: Run python tests +# run: | +# BUILD_DIR=./build \ +# PYTHONPATH=test/:scripts/ \ +# pytest \ +# test/capgen_test/capgen_test_reports.py \ +# test/advection_test/advection_test_reports.py \ +# test/ddthost_test/ddthost_test_reports.py \ +# test/var_compatibility_test/var_compatibility_test_reports.py +# +# - name: Run Fortran to metadata test +# run: cd test && ./test_fortran_to_metadata.sh +# +# - name: Run offline metadata parser test +# run: cd test && ./test_offline_metadata_checker.sh diff --git a/.github/workflows/prebuild.yaml b/.github/workflows/prebuild.yaml deleted file mode 100644 index 6ef3841c..00000000 --- a/.github/workflows/prebuild.yaml +++ /dev/null @@ -1,68 +0,0 @@ -name: ccpp-prebuild - -on: - pull_request: - branches: [develop] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - unit-tests: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.8","3.9","3.10","3.11","3.12"] - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Install open mpi - run: | - wget https://github.com/open-mpi/ompi/archive/refs/tags/v4.1.6.tar.gz - tar -xvf v4.1.6.tar.gz - cd ompi-4.1.6 - ./autogen.pl - ./configure --prefix=/home/runner/ompi-4.1.6 - make -j4 - make install - echo "LD_LIBRARY_PATH=/home/runner/ompi-4.1.6/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV - echo "PATH=/home/runner/ompi-4.1.6/bin:$PATH" >> $GITHUB_ENV - - name: ccpp-prebuild unit tests - run: | - export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools - cd test_prebuild - pytest - # No longer possible because of https://github.com/NCAR/ccpp-framework/pull/659 - #- name: ccpp-prebuild blocked data tests - # run: | - # cd test_prebuild/test_blocked_data - # python3 ../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build - # cd build - # cmake .. - # make - # ./test_blocked_data.x - - name: ccpp-prebuild chunked data tests - run: | - cd test_prebuild/test_chunked_data - python3 ../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build - cd build - cmake .. - make - ./test_chunked_data.x - diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml deleted file mode 100644 index b8508f2a..00000000 --- a/.github/workflows/python.yaml +++ /dev/null @@ -1,56 +0,0 @@ -name: Python package - -on: - workflow_dispatch: - pull_request: - branches: [develop] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y \ - libxml2-utils - python -m pip install --upgrade pip - pip install pytest - which pytest - - - name: Test with pytest - run: | - export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools - which xmllint - xmllint --version - pytest -v test/ - - - name: Test with pytest using bad xmllint (xmllint wrapper) - run: | - export XMLLINT_REAL=$(which xmllint) - export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools - export PATH=$(pwd)/test/unit_tests/xmllint_wrapper:${PATH} - export | grep PATH - which xmllint - xmllint --version - pytest -v test/ - - - name: Test with doctest - run: | - export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools - which xmllint - xmllint --version - pytest -v scripts/ --doctest-modules diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml new file mode 100644 index 00000000..cd97315e --- /dev/null +++ b/.github/workflows/unit-tests.yaml @@ -0,0 +1,34 @@ +name: capgen-ng unit tests + +on: + workflow_dispatch: + pull_request: + branches: [develop] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + + unit-tests: + name: capgen-ng unit tests + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install pytest + which pytest + - name: Run capgen-ng unit tests + run: | + pytest -v unit-tests/ \ No newline at end of file From fc007c6bd105c2ced053dd8c70cf1c67f404526e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 07:26:43 -0600 Subject: [PATCH 03/74] Add directories capgen-ng, unit-tests, end-to-end-tests --- capgen-ng/__init__.py | 1 + capgen-ng/ccpp_capgen_ng.py | 1068 ++++++ capgen-ng/ccpp_datafile.py | 873 +++++ capgen-ng/ccpp_validator.py | 623 ++++ capgen-ng/generator/__init__.py | 1 + capgen-ng/generator/datatable.py | 430 +++ capgen-ng/generator/group_cap.py | 1186 +++++++ capgen-ng/generator/host_constituents.py | 642 ++++ capgen-ng/generator/kinds_writer.py | 234 ++ capgen-ng/generator/static_api.py | 982 ++++++ capgen-ng/generator/suite_cap.py | 1056 ++++++ capgen-ng/generator/suite_data.py | 446 +++ capgen-ng/generator/suite_resolver.py | 2039 ++++++++++++ capgen-ng/generator/suite_types.py | 252 ++ capgen-ng/generator/suite_xml.py | 639 ++++ capgen-ng/metadata/__init__.py | 1 + capgen-ng/metadata/legacy_compat.py | 159 + capgen-ng/metadata/metadata_table.py | 1332 ++++++++ capgen-ng/metadata/parse_tools/__init__.py | 39 + .../parse_tools/fortran_conditional.py | 13 + .../metadata/parse_tools/parse_checkers.py | 1100 +++++++ capgen-ng/metadata/parse_tools/parse_log.py | 68 + .../metadata/parse_tools/parse_object.py | 166 + .../metadata/parse_tools/parse_source.py | 347 ++ capgen-ng/metadata/parse_tools/xml_tools.py | 571 ++++ capgen-ng/metadata/unit_conversion.py | 206 ++ capgen-ng/metadata/variable_resolver.py | 687 ++++ capgen-ng/schema/suite_v1_0.xsd | 128 + capgen-ng/schema/suite_v2_0.xsd | 119 + capgen-ng/src/ccpp_constituent_prop_mod.F90 | 2679 +++++++++++++++ capgen-ng/src/ccpp_constituent_prop_mod.meta | 63 + capgen-ng/src/ccpp_hash_table.F90 | 520 +++ capgen-ng/src/ccpp_hashable.F90 | 98 + capgen-ng/src/ccpp_scheme_utils.F90 | 122 + end-to-end-tests/CMakeLists.txt | 82 + end-to-end-tests/README.md.tobeupdated | 16 + end-to-end-tests/advection/CMakeLists.txt | 71 + end-to-end-tests/advection/README.md | 10 + .../advection/advection_test_reports.py | 127 + .../apply_constituent_tendencies.F90 | 39 + .../apply_constituent_tendencies.meta | 36 + end-to-end-tests/advection/cld_ice.F90 | 125 + end-to-end-tests/advection/cld_ice.meta | 136 + end-to-end-tests/advection/cld_liq.F90 | 107 + end-to-end-tests/advection/cld_liq.meta | 135 + end-to-end-tests/advection/cld_suite.xml | 11 + .../advection/cld_suite_error.xml | 9 + end-to-end-tests/advection/const_indices.F90 | 95 + end-to-end-tests/advection/const_indices.meta | 108 + end-to-end-tests/advection/dlc_liq.F90 | 41 + end-to-end-tests/advection/dlc_liq.meta | 29 + .../test_advection_host_integration.F90 | 79 + end-to-end-tests/advection/test_host.F90 | 1172 +++++++ end-to-end-tests/advection/test_host.meta | 70 + end-to-end-tests/advection/test_host_data.F90 | 96 + .../advection/test_host_data.meta | 67 + end-to-end-tests/advection/test_host_mod.F90 | 176 + end-to-end-tests/advection/test_host_mod.meta | 64 + end-to-end-tests/capgen_ng/CMakeLists.txt | 95 + end-to-end-tests/capgen_ng/README.md | 14 + .../capgen_ng/adjust/temp_kinds.F90 | 12 + .../capgen_ng/capgen_test_reports.py | 151 + end-to-end-tests/capgen_ng/ddt2.F90 | 12 + end-to-end-tests/capgen_ng/ddt_suite.xml | 8 + .../capgen_ng/environ_conditions.meta | 110 + end-to-end-tests/capgen_ng/make_ddt.F90 | 142 + end-to-end-tests/capgen_ng/make_ddt.meta | 128 + end-to-end-tests/capgen_ng/setup_coeffs.F90 | 24 + end-to-end-tests/capgen_ng/setup_coeffs.meta | 29 + .../source_dir1/environ_conditions.F90 | 96 + .../capgen_ng/source_dir2/temp_set.F90 | 125 + end-to-end-tests/capgen_ng/temp_adjust.F90 | 127 + end-to-end-tests/capgen_ng/temp_adjust.meta | 155 + .../capgen_ng/temp_calc_adjust.F90 | 111 + .../capgen_ng/temp_calc_adjust.meta | 111 + end-to-end-tests/capgen_ng/temp_set.meta | 213 ++ end-to-end-tests/capgen_ng/temp_suite.xml | 12 + .../test_capgen_host_integration.F90 | 91 + end-to-end-tests/capgen_ng/test_host.F90 | 349 ++ end-to-end-tests/capgen_ng/test_host.meta | 70 + end-to-end-tests/capgen_ng/test_host_data.F90 | 60 + .../capgen_ng/test_host_data.meta | 53 + end-to-end-tests/capgen_ng/test_host_mod.F90 | 164 + end-to-end-tests/capgen_ng/test_host_mod.meta | 133 + end-to-end-tests/chunked_data/CMakeLists.txt | 64 + .../chunked_data/chunked_data_scheme.F90 | 126 + .../chunked_data/chunked_data_scheme.meta | 154 + end-to-end-tests/chunked_data/data.F90 | 47 + end-to-end-tests/chunked_data/data.meta | 45 + end-to-end-tests/chunked_data/main.F90 | 124 + end-to-end-tests/chunked_data/main.meta | 65 + .../chunked_data/suite_chunked_data_suite.xml | 9 + end-to-end-tests/cmake/ccpp_capgen.cmake | 205 ++ end-to-end-tests/ddthost/CMakeLists.txt | 71 + end-to-end-tests/ddthost/README.md | 16 + end-to-end-tests/ddthost/ddt_suite.xml | 8 + .../ddthost/ddthost_test_reports.py | 139 + .../ddthost/environ_conditions.F90 | 96 + .../ddthost/environ_conditions.meta | 109 + end-to-end-tests/ddthost/host_ccpp_ddt.F90 | 16 + end-to-end-tests/ddthost/host_ccpp_ddt.meta | 31 + end-to-end-tests/ddthost/make_ddt.F90 | 131 + end-to-end-tests/ddthost/make_ddt.meta | 129 + end-to-end-tests/ddthost/setup_coeffs.F90 | 24 + end-to-end-tests/ddthost/setup_coeffs.meta | 29 + end-to-end-tests/ddthost/temp_adjust.F90 | 84 + end-to-end-tests/ddthost/temp_adjust.meta | 118 + end-to-end-tests/ddthost/temp_calc_adjust.F90 | 95 + .../ddthost/temp_calc_adjust.meta | 88 + end-to-end-tests/ddthost/temp_set.F90 | 112 + end-to-end-tests/ddthost/temp_set.meta | 171 + end-to-end-tests/ddthost/temp_suite.xml | 12 + .../ddthost/test_ddt_host_integration.F90 | 82 + end-to-end-tests/ddthost/test_host.F90 | 333 ++ end-to-end-tests/ddthost/test_host.meta | 64 + end-to-end-tests/ddthost/test_host_data.F90 | 51 + end-to-end-tests/ddthost/test_host_data.meta | 46 + end-to-end-tests/ddthost/test_host_mod.F90 | 141 + end-to-end-tests/ddthost/test_host_mod.meta | 98 + end-to-end-tests/instances/CMakeLists.txt | 59 + end-to-end-tests/instances/README.md | 16 + end-to-end-tests/instances/data.F90 | 30 + end-to-end-tests/instances/data.meta | 77 + end-to-end-tests/instances/main.F90 | 177 + end-to-end-tests/instances/main.meta | 71 + .../instances/suite_unit_conv_suite.xml | 11 + .../instances/unit_conv_scheme_1.F90 | 86 + .../instances/unit_conv_scheme_1.meta | 56 + .../instances/unit_conv_scheme_2.F90 | 86 + .../instances/unit_conv_scheme_2.meta | 56 + end-to-end-tests/nested_suite/CMakeLists.txt | 71 + end-to-end-tests/nested_suite/README.md | 19 + end-to-end-tests/nested_suite/effr_calc.F90 | 84 + end-to-end-tests/nested_suite/effr_calc.meta | 163 + end-to-end-tests/nested_suite/effr_diag.F90 | 68 + end-to-end-tests/nested_suite/effr_diag.meta | 65 + end-to-end-tests/nested_suite/effr_post.F90 | 61 + end-to-end-tests/nested_suite/effr_post.meta | 65 + end-to-end-tests/nested_suite/effr_pre.F90 | 60 + end-to-end-tests/nested_suite/effr_pre.meta | 66 + end-to-end-tests/nested_suite/effrs_calc.F90 | 32 + end-to-end-tests/nested_suite/effrs_calc.meta | 25 + end-to-end-tests/nested_suite/main_suite.xml | 20 + .../nested_suite/module_rad_ddt.F90 | 23 + .../nested_suite/module_rad_ddt.meta | 40 + end-to-end-tests/nested_suite/rad_lw.F90 | 35 + end-to-end-tests/nested_suite/rad_lw.meta | 35 + end-to-end-tests/nested_suite/rad_sw.F90 | 35 + end-to-end-tests/nested_suite/rad_sw.meta | 41 + .../nested_suite/radiation2_suite.xml | 10 + .../nested_suite/radiation3_subsuite.xml | 7 + .../nested_suite/radiation3_suite.xml | 7 + .../nested_suite/radiation4_suite.xml | 7 + .../nested_suite/suite_lifecycle.F90 | 34 + .../nested_suite/suite_lifecycle.meta | 49 + end-to-end-tests/nested_suite/test_host.F90 | 314 ++ end-to-end-tests/nested_suite/test_host.meta | 63 + .../nested_suite/test_host_data.F90 | 103 + .../nested_suite/test_host_data.meta | 129 + .../nested_suite/test_host_mod.F90 | 141 + .../nested_suite/test_host_mod.meta | 47 + .../test_nested_suite_integration.F90 | 91 + end-to-end-tests/opt_arg/CMakeLists.txt | 59 + end-to-end-tests/opt_arg/data.F90 | 21 + end-to-end-tests/opt_arg/data.meta | 40 + end-to-end-tests/opt_arg/main.F90 | 152 + end-to-end-tests/opt_arg/main.meta | 65 + end-to-end-tests/opt_arg/opt_arg_scheme.F90 | 90 + end-to-end-tests/opt_arg/opt_arg_scheme.meta | 157 + .../opt_arg/suite_opt_arg_suite.xml | 9 + end-to-end-tests/utils/test_utils.F90 | 100 + end-to-end-tests/var_compat/CMakeLists.txt | 69 + end-to-end-tests/var_compat/README.md | 23 + end-to-end-tests/var_compat/effr_calc.F90 | 84 + end-to-end-tests/var_compat/effr_calc.meta | 163 + end-to-end-tests/var_compat/effr_diag.F90 | 68 + end-to-end-tests/var_compat/effr_diag.meta | 65 + end-to-end-tests/var_compat/effr_post.F90 | 61 + end-to-end-tests/var_compat/effr_post.meta | 65 + end-to-end-tests/var_compat/effr_pre.F90 | 60 + end-to-end-tests/var_compat/effr_pre.meta | 66 + end-to-end-tests/var_compat/effrs_calc.F90 | 32 + end-to-end-tests/var_compat/effrs_calc.meta | 25 + .../var_compat/module_rad_ddt.F90 | 23 + .../var_compat/module_rad_ddt.meta | 40 + end-to-end-tests/var_compat/rad_lw.F90 | 35 + end-to-end-tests/var_compat/rad_lw.meta | 35 + end-to-end-tests/var_compat/rad_sw.F90 | 35 + end-to-end-tests/var_compat/rad_sw.meta | 41 + end-to-end-tests/var_compat/test_host.F90 | 323 ++ end-to-end-tests/var_compat/test_host.meta | 64 + .../var_compat/test_host_data.F90 | 103 + .../var_compat/test_host_data.meta | 128 + end-to-end-tests/var_compat/test_host_mod.F90 | 132 + .../var_compat/test_host_mod.meta | 42 + .../test_var_compatibility_integration.F90 | 88 + .../var_compat/var_compatibility_suite.xml | 21 + .../var_compatibility_test_reports.py | 116 + unit-tests/__init__.py | 1 + unit-tests/conftest.py | 30 + unit-tests/run_tests.py | 43 + .../sample_files/bad_ctrl_in_host_table.meta | 17 + .../bad_ctrl_missing_suite_name.meta | 60 + .../sample_files/bad_ctrl_missing_vars.meta | 18 + .../sample_files/bad_ctrl_nonscalar.meta | 66 + .../sample_files/bad_ctrl_wrong_type.meta | 66 + .../sample_files/bad_dim_loop_begin.meta | 33 + .../sample_files/bad_dim_loop_extent.meta | 23 + .../sample_files/bad_duplicate_stdname.meta | 21 + .../sample_files/bad_finalize_phase.meta | 23 + unit-tests/sample_files/bad_invalid_type.meta | 14 + unit-tests/sample_files/bad_module_type.meta | 15 + .../sample_files/control_chunked_data.meta | 79 + unit-tests/sample_files/control_full.meta | 73 + .../sample_files/control_no_instance.meta | 67 + unit-tests/sample_files/control_opt_arg.meta | 73 + unit-tests/sample_files/control_simple.meta | 70 + .../sample_files/control_unit_conv.meta | 66 + unit-tests/sample_files/ddt_chunked_data.meta | 16 + unit-tests/sample_files/ddt_nested_inner.meta | 22 + unit-tests/sample_files/ddt_nested_outer.meta | 22 + unit-tests/sample_files/ddt_simple.meta | 21 + .../sample_files/ddt_subcycle_stdname.meta | 17 + .../sample_files/host_chunked_data.meta | 31 + unit-tests/sample_files/host_full.meta | 56 + unit-tests/sample_files/host_no_instance.meta | 49 + unit-tests/sample_files/host_opt_arg.meta | 52 + unit-tests/sample_files/host_simple.meta | 25 + .../sample_files/host_subcycle_stdname.meta | 18 + .../host_subcycle_stdname_ddt.meta | 19 + unit-tests/sample_files/host_unit_conv.meta | 51 + .../sample_files/host_with_constituents.meta | 36 + .../sample_files/host_with_ddt_instance.meta | 16 + .../sample_files/host_with_dependencies.meta | 59 + .../sample_files/host_with_nested_ddt.meta | 16 + .../sample_files/scheme_chunked_data.meta | 152 + .../scheme_consume_constituent.meta | 41 + .../scheme_interstitial_consumer.meta | 32 + .../scheme_interstitial_producer.meta | 32 + .../scheme_module_name_override.meta | 32 + unit-tests/sample_files/scheme_multipart.meta | 87 + .../sample_files/scheme_multipart_correct.F90 | 34 + .../scheme_multipart_wrong_args.F90 | 28 + unit-tests/sample_files/scheme_opt_arg.meta | 163 + .../scheme_register_constituents.meta | 33 + .../scheme_register_dim_consumer.meta | 31 + .../scheme_register_dim_producer.meta | 32 + .../sample_files/scheme_suite_init_final.meta | 42 + .../sample_files/scheme_top_at_one.meta | 34 + .../sample_files/scheme_unit_conv_1.meta | 45 + .../sample_files/scheme_unit_conv_2.meta | 45 + .../sample_suite_files/another_suite.xml | 10 + .../sample_suite_files/another_suite2.xml | 16 + .../sample_suite_files/nested_full_suite.xml | 10 + unit-tests/sample_suite_files/subsuite1.xml | 7 + .../sample_suite_files/subsuite_inline.xml | 9 + .../suite_bad_v2_duplicate_group.xml | 16 + .../suite_bad_v2_suite_tag.xml | 7 + .../suite_bad_version01.xml | 8 + .../suite_bad_version02.xml | 8 + .../suite_bad_version03.xml | 8 + .../suite_bad_version04.xml | 8 + .../sample_suite_files/suite_chunked_data.xml | 9 + .../suite_consume_constituent.xml | 6 + .../suite_good_v1_test01.xml | 8 + .../suite_good_v1_test02.xml | 11 + .../suite_good_v2_test01.xml | 9 + .../suite_good_v2_test01_exp.xml | 11 + .../suite_good_v2_test02.xml | 10 + .../suite_good_v2_test02_exp.xml | 13 + .../suite_good_v2_test03.xml | 19 + .../suite_good_v2_test03_exp.xml | 30 + .../suite_good_v2_test04.xml | 18 + .../suite_good_v2_test04_exp.xml | 26 + .../sample_suite_files/suite_interstitial.xml | 7 + .../suite_invalid_group_fortran_id.xml | 8 + .../suite_invalid_scheme_fortran_id.xml | 8 + .../suite_invalid_suite_fortran_id.xml | 8 + .../sample_suite_files/suite_missing_file.xml | 9 + .../suite_missing_group.xml | 7 + .../suite_missing_loaded_suite.xml | 16 + .../suite_missing_version.xml | 8 + .../suite_module_name_override.xml | 6 + .../suite_nested_subcycle.xml | 10 + .../sample_suite_files/suite_opt_arg.xml | 9 + .../suite_recurse_level2.xml | 10 + .../suite_recurse_level2a.xml | 10 + .../suite_recurse_level3.xml | 10 + .../suite_recurse_level3a.xml | 10 + .../sample_suite_files/suite_recurse_top1.xml | 18 + .../sample_suite_files/suite_recurse_top2.xml | 18 + .../suite_register_constituents.xml | 6 + .../sample_suite_files/suite_register_dim.xml | 7 + .../suite_subcycle_stdname.xml | 8 + .../suite_subcycle_stdname_ddt.xml | 8 + .../sample_suite_files/suite_test_simple.xml | 6 + .../suite_test_subcycle.xml | 8 + .../sample_suite_files/suite_top_at_one.xml | 6 + .../sample_suite_files/suite_unit_conv.xml | 10 + .../suite_with_init_final.xml | 8 + unit-tests/test_ccpp_datafile.py | 334 ++ unit-tests/test_control_validation.py | 336 ++ unit-tests/test_datatable.py | 600 ++++ unit-tests/test_host_constituents.py | 548 ++++ unit-tests/test_integration.py | 2065 ++++++++++++ unit-tests/test_kinds_writer.py | 156 + unit-tests/test_legacy_compat.py | 280 ++ unit-tests/test_metadata_table.py | 1997 ++++++++++++ unit-tests/test_static_api.py | 1084 +++++++ unit-tests/test_suite_cap.py | 358 ++ unit-tests/test_suite_data.py | 249 ++ unit-tests/test_suite_resolver.py | 2880 +++++++++++++++++ unit-tests/test_suite_xml.py | 767 +++++ unit-tests/test_validator.py | 654 ++++ unit-tests/test_variable_resolver.py | 930 ++++++ 315 files changed, 48955 insertions(+) create mode 100644 capgen-ng/__init__.py create mode 100755 capgen-ng/ccpp_capgen_ng.py create mode 100755 capgen-ng/ccpp_datafile.py create mode 100755 capgen-ng/ccpp_validator.py create mode 100644 capgen-ng/generator/__init__.py create mode 100644 capgen-ng/generator/datatable.py create mode 100644 capgen-ng/generator/group_cap.py create mode 100644 capgen-ng/generator/host_constituents.py create mode 100644 capgen-ng/generator/kinds_writer.py create mode 100644 capgen-ng/generator/static_api.py create mode 100644 capgen-ng/generator/suite_cap.py create mode 100644 capgen-ng/generator/suite_data.py create mode 100644 capgen-ng/generator/suite_resolver.py create mode 100644 capgen-ng/generator/suite_types.py create mode 100644 capgen-ng/generator/suite_xml.py create mode 100644 capgen-ng/metadata/__init__.py create mode 100644 capgen-ng/metadata/legacy_compat.py create mode 100644 capgen-ng/metadata/metadata_table.py create mode 100644 capgen-ng/metadata/parse_tools/__init__.py create mode 100755 capgen-ng/metadata/parse_tools/fortran_conditional.py create mode 100644 capgen-ng/metadata/parse_tools/parse_checkers.py create mode 100644 capgen-ng/metadata/parse_tools/parse_log.py create mode 100644 capgen-ng/metadata/parse_tools/parse_object.py create mode 100644 capgen-ng/metadata/parse_tools/parse_source.py create mode 100644 capgen-ng/metadata/parse_tools/xml_tools.py create mode 100755 capgen-ng/metadata/unit_conversion.py create mode 100644 capgen-ng/metadata/variable_resolver.py create mode 100644 capgen-ng/schema/suite_v1_0.xsd create mode 100644 capgen-ng/schema/suite_v2_0.xsd create mode 100644 capgen-ng/src/ccpp_constituent_prop_mod.F90 create mode 100644 capgen-ng/src/ccpp_constituent_prop_mod.meta create mode 100644 capgen-ng/src/ccpp_hash_table.F90 create mode 100644 capgen-ng/src/ccpp_hashable.F90 create mode 100644 capgen-ng/src/ccpp_scheme_utils.F90 create mode 100644 end-to-end-tests/CMakeLists.txt create mode 100644 end-to-end-tests/README.md.tobeupdated create mode 100644 end-to-end-tests/advection/CMakeLists.txt create mode 100644 end-to-end-tests/advection/README.md create mode 100644 end-to-end-tests/advection/advection_test_reports.py create mode 100644 end-to-end-tests/advection/apply_constituent_tendencies.F90 create mode 100644 end-to-end-tests/advection/apply_constituent_tendencies.meta create mode 100644 end-to-end-tests/advection/cld_ice.F90 create mode 100644 end-to-end-tests/advection/cld_ice.meta create mode 100644 end-to-end-tests/advection/cld_liq.F90 create mode 100644 end-to-end-tests/advection/cld_liq.meta create mode 100644 end-to-end-tests/advection/cld_suite.xml create mode 100644 end-to-end-tests/advection/cld_suite_error.xml create mode 100644 end-to-end-tests/advection/const_indices.F90 create mode 100644 end-to-end-tests/advection/const_indices.meta create mode 100644 end-to-end-tests/advection/dlc_liq.F90 create mode 100644 end-to-end-tests/advection/dlc_liq.meta create mode 100644 end-to-end-tests/advection/test_advection_host_integration.F90 create mode 100644 end-to-end-tests/advection/test_host.F90 create mode 100644 end-to-end-tests/advection/test_host.meta create mode 100644 end-to-end-tests/advection/test_host_data.F90 create mode 100644 end-to-end-tests/advection/test_host_data.meta create mode 100644 end-to-end-tests/advection/test_host_mod.F90 create mode 100644 end-to-end-tests/advection/test_host_mod.meta create mode 100644 end-to-end-tests/capgen_ng/CMakeLists.txt create mode 100644 end-to-end-tests/capgen_ng/README.md create mode 100644 end-to-end-tests/capgen_ng/adjust/temp_kinds.F90 create mode 100644 end-to-end-tests/capgen_ng/capgen_test_reports.py create mode 100644 end-to-end-tests/capgen_ng/ddt2.F90 create mode 100644 end-to-end-tests/capgen_ng/ddt_suite.xml create mode 100644 end-to-end-tests/capgen_ng/environ_conditions.meta create mode 100644 end-to-end-tests/capgen_ng/make_ddt.F90 create mode 100644 end-to-end-tests/capgen_ng/make_ddt.meta create mode 100644 end-to-end-tests/capgen_ng/setup_coeffs.F90 create mode 100644 end-to-end-tests/capgen_ng/setup_coeffs.meta create mode 100644 end-to-end-tests/capgen_ng/source_dir1/environ_conditions.F90 create mode 100644 end-to-end-tests/capgen_ng/source_dir2/temp_set.F90 create mode 100644 end-to-end-tests/capgen_ng/temp_adjust.F90 create mode 100644 end-to-end-tests/capgen_ng/temp_adjust.meta create mode 100644 end-to-end-tests/capgen_ng/temp_calc_adjust.F90 create mode 100644 end-to-end-tests/capgen_ng/temp_calc_adjust.meta create mode 100644 end-to-end-tests/capgen_ng/temp_set.meta create mode 100644 end-to-end-tests/capgen_ng/temp_suite.xml create mode 100644 end-to-end-tests/capgen_ng/test_capgen_host_integration.F90 create mode 100644 end-to-end-tests/capgen_ng/test_host.F90 create mode 100644 end-to-end-tests/capgen_ng/test_host.meta create mode 100644 end-to-end-tests/capgen_ng/test_host_data.F90 create mode 100644 end-to-end-tests/capgen_ng/test_host_data.meta create mode 100644 end-to-end-tests/capgen_ng/test_host_mod.F90 create mode 100644 end-to-end-tests/capgen_ng/test_host_mod.meta create mode 100644 end-to-end-tests/chunked_data/CMakeLists.txt create mode 100644 end-to-end-tests/chunked_data/chunked_data_scheme.F90 create mode 100644 end-to-end-tests/chunked_data/chunked_data_scheme.meta create mode 100644 end-to-end-tests/chunked_data/data.F90 create mode 100644 end-to-end-tests/chunked_data/data.meta create mode 100644 end-to-end-tests/chunked_data/main.F90 create mode 100644 end-to-end-tests/chunked_data/main.meta create mode 100644 end-to-end-tests/chunked_data/suite_chunked_data_suite.xml create mode 100644 end-to-end-tests/cmake/ccpp_capgen.cmake create mode 100644 end-to-end-tests/ddthost/CMakeLists.txt create mode 100644 end-to-end-tests/ddthost/README.md create mode 100644 end-to-end-tests/ddthost/ddt_suite.xml create mode 100644 end-to-end-tests/ddthost/ddthost_test_reports.py create mode 100644 end-to-end-tests/ddthost/environ_conditions.F90 create mode 100644 end-to-end-tests/ddthost/environ_conditions.meta create mode 100644 end-to-end-tests/ddthost/host_ccpp_ddt.F90 create mode 100644 end-to-end-tests/ddthost/host_ccpp_ddt.meta create mode 100644 end-to-end-tests/ddthost/make_ddt.F90 create mode 100644 end-to-end-tests/ddthost/make_ddt.meta create mode 100644 end-to-end-tests/ddthost/setup_coeffs.F90 create mode 100644 end-to-end-tests/ddthost/setup_coeffs.meta create mode 100644 end-to-end-tests/ddthost/temp_adjust.F90 create mode 100644 end-to-end-tests/ddthost/temp_adjust.meta create mode 100644 end-to-end-tests/ddthost/temp_calc_adjust.F90 create mode 100644 end-to-end-tests/ddthost/temp_calc_adjust.meta create mode 100644 end-to-end-tests/ddthost/temp_set.F90 create mode 100644 end-to-end-tests/ddthost/temp_set.meta create mode 100644 end-to-end-tests/ddthost/temp_suite.xml create mode 100644 end-to-end-tests/ddthost/test_ddt_host_integration.F90 create mode 100644 end-to-end-tests/ddthost/test_host.F90 create mode 100644 end-to-end-tests/ddthost/test_host.meta create mode 100644 end-to-end-tests/ddthost/test_host_data.F90 create mode 100644 end-to-end-tests/ddthost/test_host_data.meta create mode 100644 end-to-end-tests/ddthost/test_host_mod.F90 create mode 100644 end-to-end-tests/ddthost/test_host_mod.meta create mode 100644 end-to-end-tests/instances/CMakeLists.txt create mode 100644 end-to-end-tests/instances/README.md create mode 100644 end-to-end-tests/instances/data.F90 create mode 100644 end-to-end-tests/instances/data.meta create mode 100644 end-to-end-tests/instances/main.F90 create mode 100644 end-to-end-tests/instances/main.meta create mode 100644 end-to-end-tests/instances/suite_unit_conv_suite.xml create mode 100644 end-to-end-tests/instances/unit_conv_scheme_1.F90 create mode 100644 end-to-end-tests/instances/unit_conv_scheme_1.meta create mode 100644 end-to-end-tests/instances/unit_conv_scheme_2.F90 create mode 100644 end-to-end-tests/instances/unit_conv_scheme_2.meta create mode 100644 end-to-end-tests/nested_suite/CMakeLists.txt create mode 100644 end-to-end-tests/nested_suite/README.md create mode 100644 end-to-end-tests/nested_suite/effr_calc.F90 create mode 100644 end-to-end-tests/nested_suite/effr_calc.meta create mode 100644 end-to-end-tests/nested_suite/effr_diag.F90 create mode 100644 end-to-end-tests/nested_suite/effr_diag.meta create mode 100644 end-to-end-tests/nested_suite/effr_post.F90 create mode 100644 end-to-end-tests/nested_suite/effr_post.meta create mode 100644 end-to-end-tests/nested_suite/effr_pre.F90 create mode 100644 end-to-end-tests/nested_suite/effr_pre.meta create mode 100644 end-to-end-tests/nested_suite/effrs_calc.F90 create mode 100644 end-to-end-tests/nested_suite/effrs_calc.meta create mode 100644 end-to-end-tests/nested_suite/main_suite.xml create mode 100644 end-to-end-tests/nested_suite/module_rad_ddt.F90 create mode 100644 end-to-end-tests/nested_suite/module_rad_ddt.meta create mode 100644 end-to-end-tests/nested_suite/rad_lw.F90 create mode 100644 end-to-end-tests/nested_suite/rad_lw.meta create mode 100644 end-to-end-tests/nested_suite/rad_sw.F90 create mode 100644 end-to-end-tests/nested_suite/rad_sw.meta create mode 100644 end-to-end-tests/nested_suite/radiation2_suite.xml create mode 100644 end-to-end-tests/nested_suite/radiation3_subsuite.xml create mode 100644 end-to-end-tests/nested_suite/radiation3_suite.xml create mode 100644 end-to-end-tests/nested_suite/radiation4_suite.xml create mode 100644 end-to-end-tests/nested_suite/suite_lifecycle.F90 create mode 100644 end-to-end-tests/nested_suite/suite_lifecycle.meta create mode 100644 end-to-end-tests/nested_suite/test_host.F90 create mode 100644 end-to-end-tests/nested_suite/test_host.meta create mode 100644 end-to-end-tests/nested_suite/test_host_data.F90 create mode 100644 end-to-end-tests/nested_suite/test_host_data.meta create mode 100644 end-to-end-tests/nested_suite/test_host_mod.F90 create mode 100644 end-to-end-tests/nested_suite/test_host_mod.meta create mode 100644 end-to-end-tests/nested_suite/test_nested_suite_integration.F90 create mode 100644 end-to-end-tests/opt_arg/CMakeLists.txt create mode 100644 end-to-end-tests/opt_arg/data.F90 create mode 100644 end-to-end-tests/opt_arg/data.meta create mode 100644 end-to-end-tests/opt_arg/main.F90 create mode 100644 end-to-end-tests/opt_arg/main.meta create mode 100644 end-to-end-tests/opt_arg/opt_arg_scheme.F90 create mode 100644 end-to-end-tests/opt_arg/opt_arg_scheme.meta create mode 100644 end-to-end-tests/opt_arg/suite_opt_arg_suite.xml create mode 100644 end-to-end-tests/utils/test_utils.F90 create mode 100644 end-to-end-tests/var_compat/CMakeLists.txt create mode 100644 end-to-end-tests/var_compat/README.md create mode 100644 end-to-end-tests/var_compat/effr_calc.F90 create mode 100644 end-to-end-tests/var_compat/effr_calc.meta create mode 100644 end-to-end-tests/var_compat/effr_diag.F90 create mode 100644 end-to-end-tests/var_compat/effr_diag.meta create mode 100644 end-to-end-tests/var_compat/effr_post.F90 create mode 100644 end-to-end-tests/var_compat/effr_post.meta create mode 100644 end-to-end-tests/var_compat/effr_pre.F90 create mode 100644 end-to-end-tests/var_compat/effr_pre.meta create mode 100644 end-to-end-tests/var_compat/effrs_calc.F90 create mode 100644 end-to-end-tests/var_compat/effrs_calc.meta create mode 100644 end-to-end-tests/var_compat/module_rad_ddt.F90 create mode 100644 end-to-end-tests/var_compat/module_rad_ddt.meta create mode 100644 end-to-end-tests/var_compat/rad_lw.F90 create mode 100644 end-to-end-tests/var_compat/rad_lw.meta create mode 100644 end-to-end-tests/var_compat/rad_sw.F90 create mode 100644 end-to-end-tests/var_compat/rad_sw.meta create mode 100644 end-to-end-tests/var_compat/test_host.F90 create mode 100644 end-to-end-tests/var_compat/test_host.meta create mode 100644 end-to-end-tests/var_compat/test_host_data.F90 create mode 100644 end-to-end-tests/var_compat/test_host_data.meta create mode 100644 end-to-end-tests/var_compat/test_host_mod.F90 create mode 100644 end-to-end-tests/var_compat/test_host_mod.meta create mode 100644 end-to-end-tests/var_compat/test_var_compatibility_integration.F90 create mode 100644 end-to-end-tests/var_compat/var_compatibility_suite.xml create mode 100755 end-to-end-tests/var_compat/var_compatibility_test_reports.py create mode 100644 unit-tests/__init__.py create mode 100644 unit-tests/conftest.py create mode 100644 unit-tests/run_tests.py create mode 100644 unit-tests/sample_files/bad_ctrl_in_host_table.meta create mode 100644 unit-tests/sample_files/bad_ctrl_missing_suite_name.meta create mode 100644 unit-tests/sample_files/bad_ctrl_missing_vars.meta create mode 100644 unit-tests/sample_files/bad_ctrl_nonscalar.meta create mode 100644 unit-tests/sample_files/bad_ctrl_wrong_type.meta create mode 100644 unit-tests/sample_files/bad_dim_loop_begin.meta create mode 100644 unit-tests/sample_files/bad_dim_loop_extent.meta create mode 100644 unit-tests/sample_files/bad_duplicate_stdname.meta create mode 100644 unit-tests/sample_files/bad_finalize_phase.meta create mode 100644 unit-tests/sample_files/bad_invalid_type.meta create mode 100644 unit-tests/sample_files/bad_module_type.meta create mode 100644 unit-tests/sample_files/control_chunked_data.meta create mode 100644 unit-tests/sample_files/control_full.meta create mode 100644 unit-tests/sample_files/control_no_instance.meta create mode 100644 unit-tests/sample_files/control_opt_arg.meta create mode 100644 unit-tests/sample_files/control_simple.meta create mode 100644 unit-tests/sample_files/control_unit_conv.meta create mode 100644 unit-tests/sample_files/ddt_chunked_data.meta create mode 100644 unit-tests/sample_files/ddt_nested_inner.meta create mode 100644 unit-tests/sample_files/ddt_nested_outer.meta create mode 100644 unit-tests/sample_files/ddt_simple.meta create mode 100644 unit-tests/sample_files/ddt_subcycle_stdname.meta create mode 100644 unit-tests/sample_files/host_chunked_data.meta create mode 100644 unit-tests/sample_files/host_full.meta create mode 100644 unit-tests/sample_files/host_no_instance.meta create mode 100644 unit-tests/sample_files/host_opt_arg.meta create mode 100644 unit-tests/sample_files/host_simple.meta create mode 100644 unit-tests/sample_files/host_subcycle_stdname.meta create mode 100644 unit-tests/sample_files/host_subcycle_stdname_ddt.meta create mode 100644 unit-tests/sample_files/host_unit_conv.meta create mode 100644 unit-tests/sample_files/host_with_constituents.meta create mode 100644 unit-tests/sample_files/host_with_ddt_instance.meta create mode 100644 unit-tests/sample_files/host_with_dependencies.meta create mode 100644 unit-tests/sample_files/host_with_nested_ddt.meta create mode 100644 unit-tests/sample_files/scheme_chunked_data.meta create mode 100644 unit-tests/sample_files/scheme_consume_constituent.meta create mode 100644 unit-tests/sample_files/scheme_interstitial_consumer.meta create mode 100644 unit-tests/sample_files/scheme_interstitial_producer.meta create mode 100644 unit-tests/sample_files/scheme_module_name_override.meta create mode 100644 unit-tests/sample_files/scheme_multipart.meta create mode 100644 unit-tests/sample_files/scheme_multipart_correct.F90 create mode 100644 unit-tests/sample_files/scheme_multipart_wrong_args.F90 create mode 100644 unit-tests/sample_files/scheme_opt_arg.meta create mode 100644 unit-tests/sample_files/scheme_register_constituents.meta create mode 100644 unit-tests/sample_files/scheme_register_dim_consumer.meta create mode 100644 unit-tests/sample_files/scheme_register_dim_producer.meta create mode 100644 unit-tests/sample_files/scheme_suite_init_final.meta create mode 100644 unit-tests/sample_files/scheme_top_at_one.meta create mode 100644 unit-tests/sample_files/scheme_unit_conv_1.meta create mode 100644 unit-tests/sample_files/scheme_unit_conv_2.meta create mode 100644 unit-tests/sample_suite_files/another_suite.xml create mode 100644 unit-tests/sample_suite_files/another_suite2.xml create mode 100644 unit-tests/sample_suite_files/nested_full_suite.xml create mode 100644 unit-tests/sample_suite_files/subsuite1.xml create mode 100644 unit-tests/sample_suite_files/subsuite_inline.xml create mode 100644 unit-tests/sample_suite_files/suite_bad_v2_duplicate_group.xml create mode 100644 unit-tests/sample_suite_files/suite_bad_v2_suite_tag.xml create mode 100644 unit-tests/sample_suite_files/suite_bad_version01.xml create mode 100644 unit-tests/sample_suite_files/suite_bad_version02.xml create mode 100644 unit-tests/sample_suite_files/suite_bad_version03.xml create mode 100644 unit-tests/sample_suite_files/suite_bad_version04.xml create mode 100644 unit-tests/sample_suite_files/suite_chunked_data.xml create mode 100644 unit-tests/sample_suite_files/suite_consume_constituent.xml create mode 100644 unit-tests/sample_suite_files/suite_good_v1_test01.xml create mode 100644 unit-tests/sample_suite_files/suite_good_v1_test02.xml create mode 100644 unit-tests/sample_suite_files/suite_good_v2_test01.xml create mode 100644 unit-tests/sample_suite_files/suite_good_v2_test01_exp.xml create mode 100644 unit-tests/sample_suite_files/suite_good_v2_test02.xml create mode 100644 unit-tests/sample_suite_files/suite_good_v2_test02_exp.xml create mode 100644 unit-tests/sample_suite_files/suite_good_v2_test03.xml create mode 100644 unit-tests/sample_suite_files/suite_good_v2_test03_exp.xml create mode 100644 unit-tests/sample_suite_files/suite_good_v2_test04.xml create mode 100644 unit-tests/sample_suite_files/suite_good_v2_test04_exp.xml create mode 100644 unit-tests/sample_suite_files/suite_interstitial.xml create mode 100644 unit-tests/sample_suite_files/suite_invalid_group_fortran_id.xml create mode 100644 unit-tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml create mode 100644 unit-tests/sample_suite_files/suite_invalid_suite_fortran_id.xml create mode 100644 unit-tests/sample_suite_files/suite_missing_file.xml create mode 100644 unit-tests/sample_suite_files/suite_missing_group.xml create mode 100644 unit-tests/sample_suite_files/suite_missing_loaded_suite.xml create mode 100644 unit-tests/sample_suite_files/suite_missing_version.xml create mode 100644 unit-tests/sample_suite_files/suite_module_name_override.xml create mode 100644 unit-tests/sample_suite_files/suite_nested_subcycle.xml create mode 100644 unit-tests/sample_suite_files/suite_opt_arg.xml create mode 100644 unit-tests/sample_suite_files/suite_recurse_level2.xml create mode 100644 unit-tests/sample_suite_files/suite_recurse_level2a.xml create mode 100644 unit-tests/sample_suite_files/suite_recurse_level3.xml create mode 100644 unit-tests/sample_suite_files/suite_recurse_level3a.xml create mode 100644 unit-tests/sample_suite_files/suite_recurse_top1.xml create mode 100644 unit-tests/sample_suite_files/suite_recurse_top2.xml create mode 100644 unit-tests/sample_suite_files/suite_register_constituents.xml create mode 100644 unit-tests/sample_suite_files/suite_register_dim.xml create mode 100644 unit-tests/sample_suite_files/suite_subcycle_stdname.xml create mode 100644 unit-tests/sample_suite_files/suite_subcycle_stdname_ddt.xml create mode 100644 unit-tests/sample_suite_files/suite_test_simple.xml create mode 100644 unit-tests/sample_suite_files/suite_test_subcycle.xml create mode 100644 unit-tests/sample_suite_files/suite_top_at_one.xml create mode 100644 unit-tests/sample_suite_files/suite_unit_conv.xml create mode 100644 unit-tests/sample_suite_files/suite_with_init_final.xml create mode 100644 unit-tests/test_ccpp_datafile.py create mode 100644 unit-tests/test_control_validation.py create mode 100644 unit-tests/test_datatable.py create mode 100644 unit-tests/test_host_constituents.py create mode 100644 unit-tests/test_integration.py create mode 100644 unit-tests/test_kinds_writer.py create mode 100644 unit-tests/test_legacy_compat.py create mode 100644 unit-tests/test_metadata_table.py create mode 100644 unit-tests/test_static_api.py create mode 100644 unit-tests/test_suite_cap.py create mode 100644 unit-tests/test_suite_data.py create mode 100644 unit-tests/test_suite_resolver.py create mode 100644 unit-tests/test_suite_xml.py create mode 100644 unit-tests/test_validator.py create mode 100644 unit-tests/test_variable_resolver.py diff --git a/capgen-ng/__init__.py b/capgen-ng/__init__.py new file mode 100644 index 00000000..59cfaf70 --- /dev/null +++ b/capgen-ng/__init__.py @@ -0,0 +1 @@ +"""CCPP capgen-ng: next-generation CCPP code generator.""" diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py new file mode 100755 index 00000000..a108987e --- /dev/null +++ b/capgen-ng/ccpp_capgen_ng.py @@ -0,0 +1,1068 @@ +#!/usr/bin/env python3 + +"""ccpp_capgen_ng — next-generation CCPP cap code generator. + +This script replaces both ``ccpp_prebuild.py`` and ``ccpp_capgen.py`` from the +legacy toolchain. It reads host-model metadata files, scheme metadata files, +and suite XML definition files, resolves all variable connections, and writes: + +* ``ccpp_kinds.F90`` — kind parameter definitions +* ``ccpp_static_api.F90`` — static dispatch API +* ``ccpp__cap.F90`` — suite-level cap (state machine, group dispatch) +* ``ccpp___cap.F90`` — group-level cap (scheme call sites) +* ``ccpp__data.F90`` — suite-owned interstitial data module +* ``ccpp__types.F90`` — shared types (pointer wrappers, temp locals) +* ``ccpp_.meta`` — generated suite metadata (for inspection) +* ``datatable.xml`` — generator database for ``ccpp_datafile.py`` + +Usage +----- +:: + + ccpp_capgen_ng.py \\ + --host-name \\ + --host-files \\ + --scheme-files \\ + --suites \\ + --output-root \\ + --kind-type NAME=[MODULE:]SPEC \\ # repeatable, see below + --verbose # once=INFO, twice=DEBUG + +``--kind-type`` +^^^^^^^^^^^^^^^ + +Each ``--kind-type`` entry maps a CCPP-visible kind name to a Fortran +precision constant. The syntax is:: + + --kind-type =[:] + +* ```` is the kind name as it will be published in ``ccpp_kinds`` and + referenced in scheme metadata (e.g. ``kind_phys``). +* ```` is the name of a precision constant (a kind parameter) defined + in some Fortran module. +* ```` is the Fortran module that defines ````. When + ``:`` is omitted, ```` must be a standard + ``ISO_FORTRAN_ENV`` constant (``REAL32``, ``REAL64``, ``INT32`` etc.) and + the module defaults to ``iso_fortran_env``. + +The flag may be specified multiple times. + +Examples:: + + --kind-type kind_phys=REAL64 + # → use iso_fortran_env, only: REAL64 + # integer, parameter, public :: kind_phys = REAL64 + + --kind-type kind_phys=my_host_kinds:kind_r8 + # → use my_host_kinds, only: kind_r8 + # integer, parameter, public :: kind_phys = kind_r8 + +If no ``--kind-type`` is supplied (or ``kind_phys`` is omitted from a +non-empty list), the generator injects ``kind_phys=iso_fortran_env:REAL64`` +and logs an INFO message. ``ccpp_kinds.F90`` is always written. + +Exit codes +---------- +0 — success +1 — user error (metadata problem, missing file, etc.) +2 — internal error (bug in the generator) +""" + +import argparse +import logging +import os +import sys +from typing import Dict, List, Optional, Tuple + +# Ensure the capgen-ng package is importable when invoked directly. +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_PACKAGE_DIR = os.path.dirname(_SCRIPT_DIR) +if _PACKAGE_DIR not in sys.path: + sys.path.insert(0, _PACKAGE_DIR) + +from metadata.parse_tools import CCPPError, init_log, set_log_level +from metadata.metadata_table import parse_metadata_file, MetadataTable +from metadata.variable_resolver import ( + build_ddt_module_map, + build_flat_host_dict, + SchemeStore, +) +from generator.kinds_writer import write_ccpp_kinds +from generator.suite_xml import parse_suite_xml_files +from generator.suite_resolver import resolve_suite +from generator.group_cap import write_group_cap +from generator.suite_data import write_suite_data, write_suite_meta +from generator.suite_cap import write_suite_cap +from generator.suite_types import write_suite_types +from generator.static_api import write_static_api +from generator.host_constituents import write_host_constituents +from generator.datatable import write_datatable + + +######################################################################## +# Logging +######################################################################## + +_LOGGER = init_log('ccpp_capgen_ng') + + +######################################################################## +# Framework-shipped metadata +######################################################################## + +# Path to the framework-shipped constituent module metadata, auto-included +# as a host metadata file so that the constituent DDT types are always known +# to the generator (even when the host metadata does not declare them). +_FRAMEWORK_SRC_DIR = os.path.join(_SCRIPT_DIR, 'src') +_FRAMEWORK_HOST_META = [ + os.path.join(_FRAMEWORK_SRC_DIR, 'ccpp_constituent_prop_mod.meta'), +] + +# Framework Fortran source files that must be compiled alongside the +# generated cap modules whenever any suite touches constituent state. +# Listed in datatable.xml's so host CMake projects pick them +# up via ccpp_datafile.py --utility-files / --ccpp-files queries. All +# of these live in :data:`_FRAMEWORK_SRC_DIR` (capgen-ng's own ``src/``); +# capgen-ng ships self-contained — no external src/ companion needed. +_FRAMEWORK_F90_FILES = [ + 'ccpp_constituent_prop_mod.F90', + 'ccpp_hashable.F90', + 'ccpp_hash_table.F90', + 'ccpp_scheme_utils.F90', +] + + +def _resolve_framework_f90_files() -> List[str]: + """Return absolute paths for the framework F90 files. + + Each name in :data:`_FRAMEWORK_F90_FILES` is looked up under + :data:`_FRAMEWORK_SRC_DIR` (``capgen-ng/src/``). A missing file is + a hard error: capgen-ng/src/ is the canonical (and only) location; + a missing file means the deployment is incomplete and the host + build would fail later with an opaque "Cannot open module file" + error. Surface it now with a precise message instead. + """ + found: List[str] = [] + missing: List[str] = [] + for name in _FRAMEWORK_F90_FILES: + p = os.path.join(_FRAMEWORK_SRC_DIR, name) + if os.path.isfile(p): + found.append(os.path.abspath(p)) + else: + missing.append(p) + if missing: + raise CCPPError( + "capgen-ng deployment is incomplete: required framework " + "Fortran source file(s) not found under {!r}:\n {}\n" + "Vendor the missing file(s) into capgen-ng/src/ (the " + "canonical location for files capgen-ng emits a USE for).".format( + _FRAMEWORK_SRC_DIR, '\n '.join(missing), + ) + ) + return found + + +######################################################################## +# CLI +######################################################################## + +def _build_arg_parser() -> argparse.ArgumentParser: + """Build and return the argument parser. + + Returns + ------- + argparse.ArgumentParser + """ + parser = argparse.ArgumentParser( + prog='ccpp_capgen_ng.py', + description='CCPP next-generation cap code generator', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + '--host-name', + required=True, + metavar='NAME', + help='Host model identifier (used in generated subroutine names)', + ) + parser.add_argument( + '--host-files', + required=True, + metavar='FILE[,FILE...]', + help='Comma-separated list of host-model metadata (.meta) files', + ) + parser.add_argument( + '--scheme-files', + required=True, + metavar='FILE[,FILE...]', + help='Comma-separated list of physics scheme metadata (.meta) files', + ) + parser.add_argument( + '--suites', + required=True, + metavar='FILE[,FILE...]', + help='Comma-separated list of suite XML definition (.xml) files', + ) + parser.add_argument( + '--output-root', + required=True, + metavar='DIR', + help='Output directory for all generated files', + ) + parser.add_argument( + '--kind-type', + action='append', + default=[], + metavar='NAME=[MODULE:]SPEC', + help=( + 'Map a CCPP kind name to a Fortran precision constant. Syntax: ' + '``=[:]``. When ``:`` is omitted, ' + '```` must be an ISO_FORTRAN_ENV constant (REAL32/REAL64/' + 'INT32/...) and the module defaults to ``iso_fortran_env``. ' + 'Examples: ``--kind-type kind_phys=REAL64``, ' + '``--kind-type kind_phys=my_host_kinds:kind_r8``. May be ' + 'specified multiple times. If kind_phys is not supplied, ' + '``kind_phys=iso_fortran_env:REAL64`` is injected automatically.' + ), + ) + parser.add_argument( + '--verbose', '-v', + action='count', + default=0, + help=( + 'Increase verbosity. Use once for INFO messages, ' + 'twice (-vv) for DEBUG messages.' + ), + ) + # legacy-compat: transient migration shim (delete the argument, + # the enable() call below, and the rest of the legacy_compat + # touchpoints when the migration is complete). + parser.add_argument( + '--legacy-mode', + action='store_true', + help=( + "TRANSIENT MIGRATION SHIM. Accept legacy CCPP standard " + "names (currently 'horizontal_loop_extent') in scheme " + "metadata and silently rewrite them to their canonical " + "capgen-ng equivalents ('horizontal_dimension'). Emits a " + "loud warning at startup. Will be removed." + ), + ) + return parser + + +# Standard ISO_FORTRAN_ENV kind constants accepted as a bare ```` (i.e. +# without an explicit ``:`` prefix). Compared case-insensitively. +_ISO_FORTRAN_KINDS = frozenset({ + 'INT8', 'INT16', 'INT32', 'INT64', + 'REAL32', 'REAL64', 'REAL128', +}) + +_ISO_FORTRAN_MODULE = 'iso_fortran_env' + + +def _parse_kind_types( + kind_type_args: List[str], +) -> Dict[str, Tuple[str, str]]: + """Parse ``--kind-type NAME=[MODULE:]SPEC`` arguments into a mapping. + + Parameters + ---------- + kind_type_args : list of str + Each entry must have the form ``=[:]``. + + Returns + ------- + dict + Mapping from kind name to a ``(module, spec)`` tuple. + + Raises + ------ + CCPPError + If any entry is malformed, has a duplicate name, or omits the module + for a non-ISO ````. + + Examples + -------- + Default ``iso_fortran_env`` module when spec is a known ISO kind: + + >>> _parse_kind_types(['kind_phys=REAL64', 'kind_dyn=REAL32']) + {'kind_phys': ('iso_fortran_env', 'REAL64'), 'kind_dyn': ('iso_fortran_env', 'REAL32')} + + Explicit host-supplied module: + + >>> _parse_kind_types(['kind_phys=my_host_kinds:kind_r8']) + {'kind_phys': ('my_host_kinds', 'kind_r8')} + + Mixed: + + >>> sorted(_parse_kind_types([ + ... 'kind_iso=REAL64', + ... 'kind_host=my_kinds:kind_r4', + ... ]).items()) + [('kind_host', ('my_kinds', 'kind_r4')), ('kind_iso', ('iso_fortran_env', 'REAL64'))] + + Malformed (missing ``=``): + + >>> _parse_kind_types(['bad_entry']) + Traceback (most recent call last): + ... + metadata.parse_tools.parse_source.CCPPError: --kind-type 'bad_entry' must have the form NAME=[MODULE:]SPEC + + Duplicate entry: + + >>> _parse_kind_types(['kind_phys=REAL64', 'kind_phys=REAL32']) + Traceback (most recent call last): + ... + metadata.parse_tools.parse_source.CCPPError: Duplicate --kind-type entry for 'kind_phys' + + Non-ISO spec without explicit module: + + >>> _parse_kind_types(['kind_phys=kind_r8']) + Traceback (most recent call last): + ... + metadata.parse_tools.parse_source.CCPPError: --kind-type 'kind_phys=kind_r8': spec 'kind_r8' is not a standard ISO_FORTRAN_ENV constant; supply the module explicitly as : + """ + mapping: Dict[str, Tuple[str, str]] = {} + for entry in kind_type_args: + head, sep, tail = entry.partition('=') + if not sep or not head.strip() or not tail.strip(): + raise CCPPError( + "--kind-type '{}' must have the form NAME=[MODULE:]SPEC".format(entry) + ) + kind_name = head.strip() + rhs = tail.strip() + + # Split the right-hand side on ':'. At most one colon is permitted. + rhs_parts = rhs.split(':') + if len(rhs_parts) == 1: + spec = rhs_parts[0].strip() + module = _ISO_FORTRAN_MODULE + if spec.upper() not in _ISO_FORTRAN_KINDS: + raise CCPPError( + "--kind-type '{}': spec '{}' is not a standard " + "ISO_FORTRAN_ENV constant; supply the module " + "explicitly as :".format(entry, spec) + ) + elif len(rhs_parts) == 2: + module = rhs_parts[0].strip() + spec = rhs_parts[1].strip() + if not module or not spec: + raise CCPPError( + "--kind-type '{}': both and must be " + "non-empty when using the : form".format(entry) + ) + else: + raise CCPPError( + "--kind-type '{}': at most one ':' is permitted " + "(syntax is NAME=[MODULE:]SPEC)".format(entry) + ) + + if kind_name in mapping: + raise CCPPError( + "Duplicate --kind-type entry for '{}'".format(kind_name) + ) + mapping[kind_name] = (module, spec) + return mapping + + +def _ensure_kind_phys_default( + kind_types: Dict[str, Tuple[str, str]], + log: logging.Logger, +) -> Dict[str, Tuple[str, str]]: + """Inject ``kind_phys=iso_fortran_env:REAL64`` if not already mapped. + + Mutates and returns *kind_types*. Logs an INFO message when the default + is injected, so users always know that the fallback is in effect. + """ + if 'kind_phys' not in kind_types: + kind_types['kind_phys'] = (_ISO_FORTRAN_MODULE, 'REAL64') + log.info( + "kind_phys not supplied via --kind-type or metadata kind_spec; " + "defaulting to REAL64 from iso_fortran_env" + ) + return kind_types + + +def _collect_metadata_kind_specs( + tables: List[MetadataTable], +) -> Dict[str, Tuple[str, str]]: + """Aggregate ``kind_spec`` declarations across loaded metadata tables. + + Each table contributes zero or more ``(kind_name, module, spec)`` triples + via :attr:`MetadataTable.kind_specs`. All contributions for the same + ``kind_name`` must agree; identical duplicates are collapsed silently + while a divergent ``(module, spec)`` raises :exc:`CCPPError` with a + message naming both source files. + + Parameters + ---------- + tables : list of MetadataTable + Host and scheme metadata tables, in any order. + + Returns + ------- + dict + Mapping ``kind_name -> (module, spec)``. + + Raises + ------ + CCPPError + If two tables declare the same ``kind_name`` with different + ``(module, spec)`` pairs. + """ + result: Dict[str, Tuple[str, str]] = {} + sources: Dict[str, str] = {} + for tbl in tables: + for kind_name, module, spec in tbl.kind_specs: + pair = (module, spec) + origin = "{} (table '{}')".format(tbl.file_path, tbl.table_name) + existing = result.get(kind_name) + if existing is None: + result[kind_name] = pair + sources[kind_name] = origin + elif existing != pair: + raise CCPPError( + "Conflicting kind_spec for kind '{}': {} declares " + "'{}:{}' but {} declares '{}:{}'".format( + kind_name, sources[kind_name], + existing[0], existing[1], + origin, pair[0], pair[1], + ) + ) + return result + + +def _merge_cli_and_metadata_kinds( + cli_kinds: Dict[str, Tuple[str, str]], + meta_kinds: Dict[str, Tuple[str, str]], +) -> Dict[str, Tuple[str, str]]: + """Combine ``--kind-type`` CLI mappings with metadata-declared kinds. + + For any ``kind_name`` defined in both sides the ``(module, spec)`` pair + must match exactly. Identical pairs collapse silently; mismatches raise + :exc:`CCPPError`. + + Parameters + ---------- + cli_kinds : dict + Mapping from :func:`_parse_kind_types`. + meta_kinds : dict + Mapping from :func:`_collect_metadata_kind_specs`. + + Returns + ------- + dict + Merged mapping ``kind_name -> (module, spec)``. + + Raises + ------ + CCPPError + If CLI and metadata declare the same kind name with different + ``(module, spec)`` pairs. + """ + merged = dict(cli_kinds) + for kind_name, pair in meta_kinds.items(): + existing = merged.get(kind_name) + if existing is None: + merged[kind_name] = pair + elif existing != pair: + raise CCPPError( + "Kind '{}' declared inconsistently: --kind-type says " + "'{}:{}' but metadata kind_spec says '{}:{}'".format( + kind_name, existing[0], existing[1], pair[0], pair[1], + ) + ) + return merged + + +def _split_file_list(arg: str) -> List[str]: + """Split a comma-separated file-list argument, stripping whitespace. + + Parameters + ---------- + arg : str + Comma-separated list of file paths. + + Returns + ------- + list of str + + Examples + -------- + >>> _split_file_list('a.meta, b.meta, c.meta') + ['a.meta', 'b.meta', 'c.meta'] + >>> _split_file_list('single.meta') + ['single.meta'] + >>> _split_file_list('') + [] + """ + return [f.strip() for f in arg.split(',') if f.strip()] + + +######################################################################## +# Metadata loading +######################################################################## + +# Loop-bound standard names that must never appear as variable dimensions. +# These are control variables (scalars passed as subroutine arguments) and +# using them as array dimensions indicates a porting error from the legacy +# toolchain. Remove this guard once migration is complete. +_FORBIDDEN_DIMENSION_NAMES = frozenset({ + 'horizontal_loop_extent', + 'horizontal_loop_begin', + 'horizontal_loop_end', +}) + + +def _check_no_loop_dimensions(tables: list) -> None: + """Raise CCPPError if any variable uses a forbidden dimension name. + + Parameters + ---------- + tables : list of MetadataTable + + Raises + ------ + CCPPError + If any variable's dimensions list contains a name from + ``_FORBIDDEN_DIMENSION_NAMES``. All violations are collected and + reported together. + """ + errors = [] + for tbl in tables: + for sec in tbl.sections(): + for var in sec.variables: + for dim in var.dimensions: + if dim in _FORBIDDEN_DIMENSION_NAMES: + errors.append( + "Variable '{}' (standard_name='{}') in table " + "'{}' (type={}) in '{}' uses '{}' as a " + "dimension. Loop-bound control/legacy vars must " + "not appear in dimension attributes; use " + "horizontal_dimension instead.".format( + var.local_name, var.standard_name, + tbl.table_name, tbl.table_type, + tbl.file_path, dim, + ) + ) + if errors: + raise CCPPError( + "Forbidden dimension names found in metadata:\n\n{}".format( + '\n\n'.join("ERROR: " + e for e in errors) + ) + ) + +def _load_metadata_files( + file_list: List[str], + expected_types: frozenset, + label: str, +) -> List[MetadataTable]: + """Load and validate a list of metadata files. + + Parameters + ---------- + file_list : list of str + Paths to ``.meta`` files. + expected_types : frozenset of str + Table types that are acceptable in these files. Any table with a + different type raises a :exc:`CCPPError`. + label : str + Human-readable description (``'host'`` or ``'scheme'``) used in + error messages. + + Returns + ------- + list of MetadataTable + All tables parsed from all files, in order. + + Raises + ------ + CCPPError + On any parse error or unexpected table type. + """ + tables: List[MetadataTable] = [] + for fpath in file_list: + _LOGGER.info("Reading %s metadata: %s", label, fpath) + file_tables = parse_metadata_file(fpath) + for tbl in file_tables: + if tbl.table_type not in expected_types: + raise CCPPError( + "Unexpected table type '{}' in {} metadata file '{}'; " + "expected one of {}".format( + tbl.table_type, label, fpath, sorted(expected_types) + ) + ) + _check_no_loop_dimensions(file_tables) + tables.extend(file_tables) + return tables + + +######################################################################## +# Control-variable validation +######################################################################## + +# Required control variables: (standard_name, expected_fortran_type, description) +_REQUIRED_CTRL_VARS = [ + ('suite_name', 'character', 'drives suite dispatch'), + ('horizontal_loop_begin', 'integer', 'lower horizontal slice bound at scheme call sites'), + ('horizontal_loop_end', 'integer', 'upper horizontal slice bound at scheme call sites'), + ('thread_number', 'integer', 'current thread number (pass 1 if single-threaded)'), + ('number_of_threads', 'integer', 'total thread count (pass 1 if single-threaded)'), + ('number_of_physics_threads','integer', 'physics-internal thread budget (pass 1 if unused)'), + ('ccpp_error_code', 'integer', 'CCPP error flag'), + ('ccpp_error_message', 'character', 'CCPP error message'), +] + +# Optional control variables that must be declared as a *pair*. Hosts that +# need a multi-instance API declare both ``instance_number`` (the index) and +# ``number_of_instances`` (the bound). Hosts that don't may omit both; the +# generator will emit a single-instance API and dimension all per-instance +# arrays to length 1. Declaring exactly one is an error. +_PAIRED_OPTIONAL_CTRL_VARS = [ + ('instance_number', 'integer', 'current model instance index'), + ('number_of_instances', 'integer', 'total number of model instances'), +] + + +def _validate_required_control_vars( + host_name: str, + host_dict: dict, +) -> None: + """Check that every required control variable is present in *host_dict*. + + Collects all failures and raises a single :exc:`CCPPError` listing them. + + Parameters + ---------- + host_name : str + Host model identifier, used in error messages. + host_dict : dict + Flat host variable dictionary built by :func:`build_flat_host_dict`. + + Raises + ------ + CCPPError + If any required control variable is missing, not marked as a control + variable, has the wrong Fortran type, or is not a scalar. + """ + errors = [] + + def _check_control_var(std_name, expected_type, description, required: bool) -> None: + """Validate a variable that must live in a ``type=control`` table.""" + entry = host_dict.get(std_name) + + if entry is None: + if required: + errors.append( + "Required control variable '{}' not found in host '{}' " + "type=control metadata.\n" + " This variable {}. Add it to a " + "[ccpp-table-properties] / type=control block in your " + "host metadata files.".format(std_name, host_name, description) + ) + return + + if not entry.is_control: + errors.append( + "Variable '{}' must be declared in a type=control table for " + "host '{}', but it was found in a type=host table.\n" + " Move it to a [ccpp-table-properties] / type=control " + "block.".format(std_name, host_name) + ) + return + + if entry.type.lower() != expected_type.lower(): + errors.append( + "Required control variable '{}' in host '{}' has Fortran " + "type '{}' but '{}' is required.".format( + std_name, host_name, entry.type, expected_type + ) + ) + + if entry.dimensions: + errors.append( + "Required control variable '{}' in host '{}' must be a " + "scalar (rank-0) but has dimensions {}.".format( + std_name, host_name, entry.dimensions + ) + ) + + def _check_host_module_var(std_name, expected_type, description) -> None: + """Validate a variable that must live in a ``type=host`` table. + + Used for symbols the generator emits via ``use , only: + `` rather than as call-arg control vars. + """ + entry = host_dict.get(std_name) + if entry is None: + return + + if entry.is_control: + errors.append( + "Variable '{}' must be declared in a type=host table for " + "host '{}' (it is USE'd from the host module), but it was " + "found in a type=control table.\n" + " Move it to a [ccpp-table-properties] / type=host " + "block.".format(std_name, host_name) + ) + return + + if entry.type.lower() != expected_type.lower(): + errors.append( + "Host variable '{}' in host '{}' has Fortran type '{}' but " + "'{}' is required.".format( + std_name, host_name, entry.type, expected_type + ) + ) + + if entry.dimensions: + errors.append( + "Host variable '{}' in host '{}' must be a scalar (rank-0) " + "but has dimensions {}.".format( + std_name, host_name, entry.dimensions + ) + ) + + for std_name, expected_type, description in _REQUIRED_CTRL_VARS: + _check_control_var(std_name, expected_type, description, required=True) + + # Paired optional: instance_number lives in type=control (call-arg); + # number_of_instances lives in type=host (USE'd by the suite cap for + # state-array sizing). Either both declared or neither. + _check_control_var( + 'instance_number', 'integer', + 'current model instance index', required=False, + ) + _check_host_module_var( + 'number_of_instances', 'integer', + 'total number of model instances', + ) + + inst_present = host_dict.get('instance_number') is not None + ninst_present = host_dict.get('number_of_instances') is not None + if inst_present ^ ninst_present: + present, missing, present_table, missing_table = ( + ('instance_number', 'number_of_instances', 'control', 'host') + if inst_present + else ('number_of_instances', 'instance_number', 'host', 'control') + ) + errors.append( + "Host '{}' declares '{}' (in a type={} table) but is missing " + "the paired variable '{}' (which must be declared in a " + "type={} table).\n" + " Declare both for a multi-instance API, or neither for a " + "single-instance API.".format( + host_name, present, present_table, + missing, missing_table, + ) + ) + + if errors: + raise CCPPError( + "Host '{}' is missing required control variables:\n\n{}".format( + host_name, + '\n\n'.join("ERROR: " + e for e in errors), + ) + ) + + +######################################################################## +# Entry point +######################################################################## + +def capgen( + host_name: str, + host_files: List[str], + scheme_files: List[str], + suite_files: List[str], + output_root: str, + kind_types: Dict[str, Tuple[str, str]], + logger: Optional[logging.Logger] = None, +) -> None: + """Programmatic entry point for the cap generator. + + Mirrors the CLI behaviour. Both the CLI and programmatic paths call + this function. + + Parameters + ---------- + host_name : str + Host model identifier. + host_files : list of str + Host metadata (``.meta``) file paths. + scheme_files : list of str + Scheme metadata (``.meta``) file paths. + suite_files : list of str + Suite XML (``.xml``) file paths. + output_root : str + Directory where all generated files are written. + kind_types : dict + Mapping ``kind_name -> (module_name, kind_spec)``. May be empty; + ``kind_phys=(iso_fortran_env, REAL64)`` is injected automatically + when missing. + logger : logging.Logger, optional + Logger to use. Defaults to the module-level logger. + + Raises + ------ + CCPPError + On any user-facing error. + """ + log = logger or _LOGGER + + # Snapshot the CLI-provided kinds; the default ``kind_phys`` and any + # metadata-declared kind_specs are folded in below, after metadata loads. + cli_kind_types = dict(kind_types) + + # ---- validate output directory ----------------------------------------- + os.makedirs(output_root, exist_ok=True) + + # ---- load host metadata (host + control tables) ------------------------- + log.info("Loading host metadata for host '%s'", host_name) + framework_meta = [p for p in _FRAMEWORK_HOST_META if os.path.isfile(p)] + if framework_meta: + log.info("Auto-including framework metadata: %s", framework_meta) + host_tables = _load_metadata_files( + framework_meta + list(host_files), + expected_types=frozenset({'host', 'control', 'ddt'}), + label='host', + ) + log.info("Loaded %d host/control/ddt tables", len(host_tables)) + + # ---- load scheme metadata ----------------------------------------------- + log.info("Loading scheme metadata") + scheme_tables = _load_metadata_files( + scheme_files, + expected_types=frozenset({'scheme', 'ddt'}), + label='scheme', + ) + log.info("Loaded %d scheme/ddt tables", len(scheme_tables)) + + # ---- merge --kind-type with metadata kind_spec declarations ----------- + meta_kind_types = _collect_metadata_kind_specs(host_tables + scheme_tables) + if meta_kind_types: + log.info( + "Found %d kind_spec declaration(s) in metadata: %s", + len(meta_kind_types), sorted(meta_kind_types), + ) + kind_types = _merge_cli_and_metadata_kinds(cli_kind_types, meta_kind_types) + kind_types = _ensure_kind_phys_default(kind_types, log) + + # ---- build flat host variable dictionary -------------------------------- + host_only = [t for t in host_tables if t.table_type == 'host'] + control_only = [t for t in host_tables if t.table_type == 'control'] + ddt_from_host = [t for t in host_tables if t.table_type == 'ddt'] + ddt_from_schemes = [t for t in scheme_tables if t.table_type == 'ddt'] + all_ddt_tables = ddt_from_host + ddt_from_schemes + + host_dict = build_flat_host_dict(host_only, control_only, all_ddt_tables) + log.info("Host dictionary contains %d variables", len(host_dict)) + + # Map DDT type name → defining Fortran module, derived from co-located + # tables in each .meta file. Used by the suite data generator to emit + # USE statements for DDT-typed suite-owned variables. + ddt_module_map = build_ddt_module_map(host_tables + scheme_tables) + + # ---- Phase 1 validation: required control variables --------------------- + _validate_required_control_vars(host_name, host_dict) + + # Signal which instance API the host opted into so users can tell which + # branch the generator took. Paired-presence has already been enforced. + if host_dict.get('instance_number') is not None: + log.info("Host '%s' declares instance_number — generating " + "multi-instance API.", host_name) + else: + log.info("Host '%s' did not declare instance_number — generating " + "single-instance API (per-instance arrays sized to 1).", + host_name) + + # ---- build scheme metadata store ---------------------------------------- + scheme_store = SchemeStore.build_from(scheme_tables) + log.info("Scheme store contains %d schemes: %s", + len(scheme_store.scheme_names()), scheme_store.scheme_names()) + + # ---- write ccpp_kinds.F90 (always generated) --------------------------- + kinds_path = write_ccpp_kinds(kind_types, output_root) + log.info("Wrote %s", kinds_path) + + # ---- parse suite XML files ---------------------------------------------- + suites = parse_suite_xml_files(suite_files, output_root, log) + log.info("Loaded %d suite(s): %s", len(suites), [s.name for s in suites]) + + # ---- resolve and generate per-suite outputs ---------------------------- + suite_names = [] + suite_resolutions = [] + + for suite in suites: + log.info("Resolving suite '%s'", suite.name) + suite_res = resolve_suite(suite, scheme_store, host_dict) + suite_names.append(suite.name) + suite_resolutions.append(suite_res) + + # Group caps + for rg in suite_res.groups: + cap_path = write_group_cap( + suite.name, rg.group_name, rg, host_dict, output_root + ) + log.info("Wrote %s", cap_path) + + # Suite data module + data_path = write_suite_data( + suite.name, suite_res.suite_vars, output_root, host_dict, + ddt_module_map=ddt_module_map, + ) + log.info("Wrote %s", data_path) + + # Suite metadata (for inspection) + meta_path = write_suite_meta(suite.name, suite_res.suite_vars, output_root) + log.info("Wrote %s", meta_path) + + # Suite types module (only when optional args are present) + types_path = write_suite_types(suite.name, suite_res, output_root) + if types_path: + log.info("Wrote %s", types_path) + + # Suite cap + suite_cap_path = write_suite_cap( + suite.name, suite_res, scheme_store, output_root, host_dict + ) + log.info("Wrote %s", suite_cap_path) + + # ---- static API (one file for all suites) ------------------------------ + static_path = write_static_api( + suite_names, suite_resolutions, output_root, host_dict, scheme_store + ) + log.info("Wrote %s", static_path) + + # ---- host-wide constituent module (only when any suite touches + # constituent state) ------------------------------------------------ + host_consts_path = write_host_constituents( + suite_resolutions, output_root, host_dict=host_dict, + ) + if host_consts_path: + log.info("Wrote %s", host_consts_path) + + # ---- datatable.xml ------------------------------------------------------ + abs_root = os.path.abspath(output_root) + utility_paths = [ + os.path.join(abs_root, 'ccpp_kinds.F90'), + ] + if host_consts_path: + utility_paths.append(host_consts_path) + # The generated ccpp_host_constituents.F90 USEs ccpp_constituent_prop_mod + # (and transitively ccpp_hashable / ccpp_hash_table); host code that + # calls ccpp_constituent_index pulls in ccpp_scheme_utils. Add all + # framework F90 dependencies so the host build picks them up. + utility_paths.extend(_resolve_framework_f90_files()) + host_file_paths = [ + os.path.join(abs_root, 'ccpp_static_api.F90'), + ] + suite_file_paths = [] + suite_meta_paths = [] + for sname, sr in zip(suite_names, suite_resolutions): + suite_file_paths.append( + os.path.join(abs_root, 'ccpp_{}_cap.F90'.format(sname)) + ) + suite_file_paths.append( + os.path.join(abs_root, 'ccpp_{}_data.F90'.format(sname)) + ) + # Types module is only present when optional args exist. + types_file = os.path.join(abs_root, 'ccpp_{}_types.F90'.format(sname)) + if os.path.isfile(types_file): + suite_file_paths.append(types_file) + for rg in sr.groups: + suite_file_paths.append( + os.path.join( + abs_root, + 'ccpp_{}_{}_cap.F90'.format(sname, rg.group_name), + ) + ) + suite_meta_paths.append( + os.path.join(abs_root, 'ccpp_{}.meta'.format(sname)) + ) + # Expanded SDFs (one per parsed suite) are inspection artifacts; carry + # the paths set by parse_suite_xml() forward into datatable.xml. + expanded_sdf_paths = [s.expanded_file for s in suites if s.expanded_file] + # Collect dependency paths from EVERY parsed metadata table — + # host, control, ddt, and scheme alike. ``dependencies =`` is + # legal on any [ccpp-table-properties] block per + # ``MetadataTable.apply_table_props``, so all of them must + # contribute to datatable.xml's section. + # Duplicates are collapsed by ``write_datatable``. + dependency_paths = [] + for tbl in host_tables + scheme_tables: + dependency_paths.extend(tbl.dependencies) + + datatable_path = write_datatable( + suite_resolutions, scheme_store, utility_paths, suite_file_paths, + output_root, host_file_paths=host_file_paths, + dependency_paths=dependency_paths, + suite_meta_paths=suite_meta_paths, + expanded_sdf_paths=expanded_sdf_paths, + host_dict=host_dict, host_name=host_name, + ) + log.info("Wrote %s", datatable_path) + + log.info("Cap generation complete.") + + +def main(argv: Optional[List[str]] = None) -> int: + """Command-line entry point. + + Parameters + ---------- + argv : list of str, optional + Override ``sys.argv[1:]`` (used by tests). + + Returns + ------- + int + Exit code: 0 = success, 1 = user error, 2 = internal error. + """ + parser = _build_arg_parser() + args = parser.parse_args(argv) + + # ---- configure logging ------------------------------------------------- + if args.verbose == 0: + set_log_level(_LOGGER, logging.WARNING) + elif args.verbose == 1: + set_log_level(_LOGGER, logging.INFO) + else: + set_log_level(_LOGGER, logging.DEBUG) + + # legacy-compat: transient migration shim. Emit the loud banner + # before any parsing happens so user has fair warning. + if args.legacy_mode: + from metadata import legacy_compat + legacy_compat.enable(_LOGGER) + + # ---- parse kind types -------------------------------------------------- + try: + kind_types = _parse_kind_types(args.kind_type) + except CCPPError as exc: + _LOGGER.error("%s", exc) + return 1 + + # ---- call the generator ------------------------------------------------ + try: + capgen( + host_name=args.host_name, + host_files=_split_file_list(args.host_files), + scheme_files=_split_file_list(args.scheme_files), + suite_files=_split_file_list(args.suites), + output_root=args.output_root, + kind_types=kind_types, + ) + except CCPPError as exc: + _LOGGER.error("%s", exc) + return 1 + except Exception as exc: # pylint: disable=broad-except + _LOGGER.error("Internal error: %s", exc, exc_info=True) + return 2 + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/capgen-ng/ccpp_datafile.py b/capgen-ng/ccpp_datafile.py new file mode 100755 index 00000000..ae11cbab --- /dev/null +++ b/capgen-ng/ccpp_datafile.py @@ -0,0 +1,873 @@ +#!/usr/bin/env python3 + +"""Query CLI for the ``datatable.xml`` produced by ``ccpp_capgen_ng.py``. + +This is the read-side companion to :mod:`generator.datatable`. The writer +half lives in the generator package; this module is a pure-Python, +dependency-free reader (only the stdlib ``xml.etree.ElementTree``) so it +can be invoked from CMake or any other build-time tooling. + +The flag surface mirrors the original ``scripts/ccpp_datafile.py``: + + * Fortran-source enumeration: + ``--host-files`` / ``--suite-files`` / ``--utility-files`` / + ``--capgen-files``. All four return *only* generated Fortran files + (``.F90``). ``--capgen-files`` is the union of the other three. + * Non-Fortran enumeration: + ``--inspection-files`` — generated ``.meta`` and expanded SDF XML. + * ``--process-list`` / ``--module-list`` / ``--dependencies`` + * ``--suite-list`` + * ``--required-variables`` / ``--input-variables`` / + ``--output-variables`` / ``--host-variables`` + * ``--show`` (pretty-print) + * ``--separator``, ``--exclude-protected``, ``--line-wrap``, ``--indent`` + +Exactly one report action is required per invocation. + +Notes specific to capgen-ng +--------------------------- +* ``--host-files`` returns ``ccpp_static_api.F90`` (capgen-ng emits the + static API in lieu of a per-host cap file). +* ``--capgen-files`` enumerates Fortran sources only. Non-Fortran inspection + artifacts (``ccpp_.meta``, ``ccpp__expanded.xml``) are + reported via ``--inspection-files``. +* ``--process-list`` is supported syntactically but will return an empty + string: capgen-ng does not currently record a ``process`` attribute on + scheme entries. +""" + +# Python library imports +import argparse +import sys +import xml.etree.ElementTree as ET +from typing import List, Optional + +# Module-level indent string used by --show; overridden via --indent. +_INDENT_STR = " " + +## datatable_report must have an action for each report type +_VALID_REPORTS = [ + {"report": "host_files", "type": bool, + "help": "Return a list of host CAP Fortran files created by capgen"}, + {"report": "suite_files", "type": bool, + "help": "Return a list of suite CAP Fortran files created by capgen"}, + {"report": "utility_files", "type": bool, + "help": ("Return a list of utility Fortran files created by " + "capgen (e.g., ccpp_kinds.F90)")}, + {"report": "capgen_files", "type": bool, + "help": ("Return a list of all Fortran files created by capgen " + "(union of --host-files, --suite-files, --utility-files); " + "non-Fortran inspection artifacts are reported via " + "--inspection-files")}, + {"report": "inspection_files", "type": bool, + "help": ("Return a list of non-Fortran inspection files created by " + "capgen (suite .meta files and expanded suite definition " + "XML files)")}, + {"report": "process_list", "type": bool, + "help": ("Return a list of process types and implementing " + "scheme name")}, + {"report": "module_list", "type": bool, + "help": + "Return a list of module names used in this set of suites"}, + {"report": "dependencies", "type": bool, + "help": ("Return a list of scheme and host " + "dependency file paths (from the 'dependencies' " + "attribute in metadata tables)")}, + {"report": "suite_list", "type": bool, + "help": "Return a list of configured suite names"}, + {"report": "required_variables", "type": str, + "help": ("Return a list of required variable " + "standard names for suite, "), + "metavar": "SUITE_NAME"}, + {"report": "input_variables", "type": str, + "help": ("Return a list of required input variable " + "standard names for suite, "), + "metavar": "SUITE_NAME"}, + {"report": "output_variables", "type": str, + "help": ("Return a list of required output variable " + "standard names for suite, "), + "metavar": "SUITE_NAME"}, + {"report": "host_variables", "type": bool, + "help": ("Return a list of required host model variable " + "standard names")}, + {"report": "show", "type": bool, + "help": + "Pretty print the database contents to the screen"}, +] + +### +### Utilities +### + + +class CCPPDatatableError(ValueError): + """Error specific to errors found in the CCPP capgen datafile""" + pass + + +class DatatableReport(object): + """A class to hold a database report type and inquiry function""" + + __valid_actions = [x["report"] for x in _VALID_REPORTS] + + def __init__(self, action, value=True): + """Initialize this report as report-type, . + # Test a valid action + >>> DatatableReport('input_variables', False).action + 'input_variables' + + # Test an invalid action + >>> DatatableReport('banana', True).value + Traceback (most recent call last): + ... + ValueError: Invalid action, 'banana' + + """ + if action in DatatableReport.__valid_actions: + self.__action = action + self.__value = value + else: + raise ValueError("Invalid action, '{}'".format(action)) + + def action_is(self, action): + """If matches this report type, return True. + Otherwise, return False + >>> DatatableReport('suite_files', False).action_is('suite_files') + True + + >>> DatatableReport('suite_files', False).action_is('banana') + False + """ + return action == self.__action + + @property + def action(self): + """Return this action's action""" + return self.__action + + @property + def value(self): + """Return this action's value""" + return self.__value + + @classmethod + def valid_actions(cls): + """Return the list of valid actions for this class""" + return cls.__valid_actions + + +### +### Interface for retrieving datatable information +### + + +def _command_line_parser(): + """Create and return an ArgumentParser for parsing the command line.""" + description = """ + Retrieve information about a ccpp_capgen_ng run. + The returned information is controlled by selecting an action from + the list of optional arguments below. + Note that exactly one action is required. + """ + parser = argparse.ArgumentParser(description=description) + parser.add_argument("datatable", type=str, + help="Path to a data table XML file created by capgen") + # Only one action per call + group = parser.add_mutually_exclusive_group(required=True) + for report in _VALID_REPORTS: + rep_type = "--{}".format(report["report"].replace("_", "-")) + if report["type"] is bool: + group.add_argument(rep_type, action='store_true', default=False, + help=report["help"]) + elif report["type"] is str: + if "metavar" in report: + group.add_argument(rep_type, required=False, type=str, + metavar=report["metavar"], default='', + help=report["help"]) + else: + group.add_argument(rep_type, required=False, type=str, + default='', help=report["help"]) + else: + raise ValueError("Unknown report type, '{}'".format(report["type"])) + defval = "," + help_str = "String to separate items in a list (default: '{}')" + parser.add_argument("--separator", type=str, required=False, default=defval, + metavar="SEP", dest="sep", help=help_str.format(defval)) + defval = False + help_str = ("Exclude protected variables (only has an effect if the " + "requested report is returning a list of variables)." + " (default: {})") + parser.add_argument("--exclude-protected", action='store_true', + required=False, + default=defval, help=help_str.format(defval)) + defval = -1 + help_str = ("Screen width for '--show' line wrapping. -1 means do not " + "wrap. (default: {})") + parser.add_argument("--line-wrap", type=int, required=False, + metavar="LINE_WIDTH", dest="line_wrap", + default=defval, help=help_str.format(defval)) + defval = 2 + help_str = "Indent depth for '--show' output (default: {})" + parser.add_argument("--indent", type=int, required=False, default=2, + help=help_str.format(defval)) + return parser + + +def parse_command_line(args): + """Create an ArgumentParser to parse and return command-line arguments.""" + parser = _command_line_parser() + pargs = parser.parse_args(args) + return pargs + + +### +### Accessor functions to retrieve information from a datatable file +### + + +def _read_datatable(datatable): + """Read XML file *datatable* and return its root node.""" + tree = ET.parse(datatable) + return tree.getroot() + + +def _find_table_section(table, elem_type): + """Look for and return an element type, , in . + Raise an exception if the element is not found. + # Test present section + >>> table = ET.fromstring("") + >>> _find_table_section(table, "capgen_files").tag + 'capgen_files' + + # Test missing section + >>> table = ET.fromstring("") + >>> _find_table_section(table, "capgen_files").tag + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Element type, 'capgen_files', not found in table + """ + found = table.find(elem_type) + if found is None: + emsg = "Element type, '{}', not found in table" + raise CCPPDatatableError(emsg.format(elem_type)) + return found + + +def _retrieve_capgen_files(table, file_type=None): + """Find and retrieve a list of generated filenames from
. + If is not None, only return that file type. + # Test valid ccpp files + >>> table = ET.fromstring(""\ + "/path/to/file1"\ + "/path/to/file2"\ + "/path/to/file3"\ + "/path/to/file4"\ + "") + >>> _retrieve_capgen_files(table) + ['/path/to/file1', '/path/to/file2', '/path/to/file3', '/path/to/file4'] + + # Test invalid file type + >>> table = ET.fromstring(""\ + "/path/to/file1"\ + "") + >>> _retrieve_capgen_files(table) + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Invalid file list entry type, 'banana' + """ + capgen_files = list() + for section in _find_table_section(table, "capgen_files"): + if (not file_type) or (section.tag == file_type): + for entry in section: + if entry.tag == "file": + capgen_files.append(entry.text) + else: + emsg = "Invalid file list entry type, '{}'" + raise CCPPDatatableError(emsg.format(entry.tag)) + return capgen_files + + +def _retrieve_inspection_files(table, file_type=None): + """Find and retrieve a list of inspection filenames from
. + + Inspection files are non-Fortran artifacts emitted by capgen-ng for + debugging and downstream tooling: suite ``.meta`` files and expanded + suite-definition XML. Each kind lives in its own subsection of + ````. + + If is not None, only return files in that subsection. + + # Test valid inspection files + >>> table = ET.fromstring(""\ + "/path/to/a.meta"\ + ""\ + "/path/to/a_exp.xml"\ + ""\ + "") + >>> _retrieve_inspection_files(table) + ['/path/to/a.meta', '/path/to/a_exp.xml'] + + # Test file_type filter + >>> _retrieve_inspection_files(table, file_type='suite_meta_files') + ['/path/to/a.meta'] + + # Test invalid entry type + >>> table = ET.fromstring(""\ + "/path/to/a.meta"\ + "") + >>> _retrieve_inspection_files(table) + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Invalid file list entry type, 'banana' + """ + inspection_files = list() + for section in _find_table_section(table, "inspection_files"): + if (not file_type) or (section.tag == file_type): + for entry in section: + if entry.tag == "file": + inspection_files.append(entry.text) + else: + emsg = "Invalid file list entry type, '{}'" + raise CCPPDatatableError(emsg.format(entry.tag)) + return inspection_files + + +def _retrieve_process_list(table): + """Find and return a list of all physics scheme processes in
. + + capgen-ng does not currently record a ``process`` attribute on + scheme entries, so this returns an empty list when no scheme carries + one. The flag is kept for CLI compatibility. + + # Test valid module + >>> table = ET.fromstring(""\ + ""\ + ""\ + "") + >>> _retrieve_process_list(table) + ['four=scheme2', 'three=scheme1'] + + # Test no schemes element + >>> table = ET.fromstring("") + >>> _retrieve_process_list(table) + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Could not find 'schemes' element + """ + result = list() + schemes = table.find("schemes") + if schemes is None: + raise CCPPDatatableError("Could not find 'schemes' element") + for scheme in schemes: + name = scheme.get("name") + proc = scheme.get("process") + if proc: + result.append("{}={}".format(proc, name)) + return sorted(result) + + +def _retrieve_module_list(table): + """Find and return a list of all scheme modules in
. + # Test valid module + >>> table = ET.fromstring(""\ + ""\ + ""\ + "") + >>> _retrieve_module_list(table) + ['partridge', 'turtle_dove'] + + # Test no schemes element + >>> table = ET.fromstring("") + >>> _retrieve_module_list(table) + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Could not find 'schemes' element + + """ + result = set() + schemes = table.find("schemes") + if schemes is None: + raise CCPPDatatableError("Could not find 'schemes' element") + for scheme in schemes: + for phase in scheme: + module = phase.get("module") + if module is not None: + result.add(module) + return sorted(result) + + +def _retrieve_dependencies(table): + """Find and return a sorted, dedup'd list of host and scheme + dependency file paths (collected from the ``dependencies`` attribute + in metadata tables). + # Test valid dependencies + >>> table = ET.fromstring("" \ + "bananaorange" \ + "") + >>> _retrieve_dependencies(table) + ['banana', 'orange'] + + # Test no dependencies + >>> table = ET.fromstring("" \ + "") + >>> _retrieve_dependencies(table) + [] + + # Test missing dependencies tag + >>> table = ET.fromstring("") + >>> _retrieve_dependencies(table) + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Could not find 'dependencies' element + """ + result = set() + depends = table.find("dependencies") + if depends is None: + raise CCPPDatatableError("Could not find 'dependencies' element") + for dependency in depends: + dep_file = dependency.text + if dep_file is not None: + result.add(dep_file) + return sorted(result) + + +def _find_var_dictionary(table, dict_name=None, dict_type=None): + """Find and return a var_dictionary in
. + If not found, return None. + # Test valid table with dict_name provided + >>> table = ET.fromstring(""\ + ""\ + ""\ + ""\ + "") + >>> _find_var_dictionary(table, dict_name='orange').get("name") + 'orange' + + # Test valid table with dict_type provided + >>> _find_var_dictionary(table, dict_type='host').get("name") + 'banana' + + # Test valid table with both dict_type and dict_name provided + >>> _find_var_dictionary(table, dict_type='host', dict_name='banana').get("name") + 'banana' + + # Test no table found (expect None) + >>> _find_var_dictionary(table, dict_name='apple') is None + True + + # Test error handling + >>> _find_var_dictionary(table) + Traceback (most recent call last): + ... + ValueError: At least one of or must contain a string + """ + var_dicts = table.find("var_dictionaries") + target_dict = None + if (dict_name is None) and (dict_type is None): + raise ValueError(("At least one of or must " + "contain a string")) + if var_dicts is None: + return None + for vdict in var_dicts: + if (((dict_name is None) or (vdict.get("name") == dict_name)) and + ((dict_type is None) or (vdict.get("type") == dict_type))): + target_dict = vdict + break + return target_dict + + +def _retrieve_suite_list(table): + """Find and return a list of all suites found in
. + # Test suites are found + >>> table = ET.fromstring(""\ + ""\ + "") + >>> _retrieve_suite_list(table) + ['umbrella', 'galoshes'] + + # Test suites not found + >>> table = ET.fromstring("") + >>> _retrieve_suite_list(table) + [] + """ + result = list() + api_elem = table.find("api") + if api_elem is not None: + suites_elem = api_elem.find("suites") + if suites_elem is not None: + for suite in suites_elem: + result.append(suite.get("name")) + return result + + +def _retrieve_suite_group_names(table, suite_name): + """Find and return a list of the group names for this suite. + # Test suites are found + >>> table = ET.fromstring(""\ + ""\ + ""\ + ""\ + "") + >>> _retrieve_suite_group_names(table, 'umbrella') + ['florence', 'delores', 'edna'] + + # Test non-present suite + >>> _retrieve_suite_group_names(table, 'poncho') + [] + """ + result = list() + api_elem = table.find("api") + if api_elem is not None: + suites_elem = api_elem.find("suites") + if suites_elem is not None: + for suite in suites_elem: + if suite.get("name") == suite_name: + for item in suite: + if item.tag == "group": + result.append(item.get("name")) + return result + + +def _is_variable_protected(table, var_name, var_dict): + """Determine whether variable, , from is protected. + Do this by checking the 'protected' attribute for in + or any of 's ancestors (parent dictionaries). + # Test found variable + >>> table = ET.fromstring(""\ + ""\ + ""\ + ""\ + "") + >>> var_dict = _find_var_dictionary(table, dict_name="banana") + >>> _is_variable_protected(table, "hi", var_dict) + True + + >>> var_dict = _find_var_dictionary(table, dict_type="api") + >>> _is_variable_protected(table, "hello", var_dict) + False + + # Test non-present variable also returns False + >>> _is_variable_protected(table, "hiya", var_dict) + False + """ + protected = False + while (not protected) and (var_dict is not None): + dvars = var_dict.find("variables") + if dvars is not None: + for var in dvars: + if var.get("name") == var_name: + protected = var.get("protected", default="False") == "True" + break + parent = var_dict.get("parent") + if parent is not None: + var_dict = _find_var_dictionary(table, dict_name=parent) + else: + var_dict = None + return protected + + +def _retrieve_variable_list(table, suite_name, + intent_type=None, exclude_protected=True): + """Find and return a list of all the required variables in . + If suite, , is not found in
, return an empty list. + If is present, return only that variable type (input or + output). + If is True, do not include protected variables. + >>> table = ET.fromstring(""\ + ""\ + ""\ + ""\ + ""\ + ""\ + "") + + # Test group variable retrieval + >>> _retrieve_variable_list(table, 'fruit', exclude_protected=False) + ['var3', 'var4', 'var5'] + + >>> _retrieve_variable_list(table, 'fruit') + ['var3'] + + >>> _retrieve_variable_list(table, 'fruit', intent_type='input', exclude_protected=False) + ['var3', 'var5'] + + >>> _retrieve_variable_list(table, 'fruit', intent_type='output', exclude_protected=False) + ['var4', 'var5'] + + >>> _retrieve_variable_list(table, 'fruit', intent_type='input') + ['var3'] + + >>> _retrieve_variable_list(table, 'fruit', intent_type='output') + [] + + # Test host variable retrieval + >>> _retrieve_variable_list(table, 'fruit', intent_type='host', exclude_protected=False) + ['var1', 'var2'] + + >>> _retrieve_variable_list(table, 'fruit', intent_type='host') + ['var1'] + + # Test invalid intent type + >>> _retrieve_variable_list(table, 'fruit', intent_type='banana') + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Invalid intent_type, 'banana' + + """ + var_set = set() + excl_vars = list() + if intent_type == "host": + allowed_intents = list() + elif intent_type is None: + allowed_intents = ['in', 'out', 'inout'] + elif intent_type == "input": + allowed_intents = ['in', 'inout'] + elif intent_type == "output": + allowed_intents = ['out', 'inout'] + else: + emsg = "Invalid intent_type, '{}'" + raise CCPPDatatableError(emsg.format(intent_type)) + if exclude_protected or (intent_type == "host"): + host_dict = _find_var_dictionary(table, dict_type="host") + if host_dict is not None: + hvars = host_dict.find("variables") + if hvars is not None: + for var in hvars: + vname = var.get("name") + if exclude_protected: + exclude = _is_variable_protected(table, vname, + host_dict) + else: + exclude = False + if intent_type == "host": + if not exclude: + var_set.add(vname) + else: + if exclude: + excl_vars.append(vname) + if intent_type != "host": + group_names = _retrieve_suite_group_names(table, suite_name) + for group in group_names: + cl_name = group + "_call_list" + group_dict = _find_var_dictionary(table, dict_name=cl_name, + dict_type="group_call_list") + if group_dict is not None: + gvars = group_dict.find("variables") + if gvars is not None: + for var in gvars: + vname = var.get("name") + vintent = var.get("intent") + if exclude_protected: + exclude = vname in excl_vars + if not exclude: + exclude = _is_variable_protected(table, vname, + group_dict) + else: + exclude = False + if (vintent in allowed_intents) and (not exclude): + var_set.add(vname) + return sorted(var_set) + + +def datatable_report(datatable, action, sep, exclude_protected=False): + """Perform a lookup on and return the result.""" + if not action: + emsg = "datatable_report: An action is required\n" + emsg += _command_line_parser().format_usage() + raise ValueError(emsg) + if not sep: + emsg = "datatable_report: A separator character () is required\n" + emsg += _command_line_parser().format_usage() + raise ValueError(emsg) + table = _read_datatable(datatable) + if action.action_is("capgen_files"): + result = _retrieve_capgen_files(table) + elif action.action_is("host_files"): + result = _retrieve_capgen_files(table, file_type="host_files") + elif action.action_is("suite_files"): + result = _retrieve_capgen_files(table, file_type="suite_files") + elif action.action_is("utility_files"): + result = _retrieve_capgen_files(table, file_type="utilities") + elif action.action_is("inspection_files"): + result = _retrieve_inspection_files(table) + elif action.action_is("process_list"): + result = _retrieve_process_list(table) + elif action.action_is("module_list"): + result = _retrieve_module_list(table) + elif action.action_is("dependencies"): + result = _retrieve_dependencies(table) + elif action.action_is("suite_list"): + result = _retrieve_suite_list(table) + elif action.action_is("required_variables"): + result = _retrieve_variable_list(table, action.value, + exclude_protected=exclude_protected) + elif action.action_is("input_variables"): + result = _retrieve_variable_list(table, action.value, + intent_type="input", + exclude_protected=exclude_protected) + elif action.action_is("output_variables"): + result = _retrieve_variable_list(table, action.value, + intent_type="output", + exclude_protected=exclude_protected) + elif action.action_is("host_variables"): + result = _retrieve_variable_list(table, "host", + exclude_protected=exclude_protected, + intent_type="host") + else: + result = '' + if isinstance(result, list): + result = sep.join(result) + return result + + +def _indent_str(indent): + """Return the line start string for indent level, .""" + return _INDENT_STR * indent + + +def _format_line(line_in, indent, line_wrap, increase_indent=True): + """Format into separate lines in an attempt to not have the + length of any line greater than characters including any + indent (with indent level specified by ). + If is True, increase the indent level for new lines + created by the process. + A value of less one means do not wrap the line. + >>> line = "This is a very long string that should be wrapped hopefully" + >>> _format_line(line, 1, 50) + ' This is a very long string that should be\\n wrapped hopefully\\n' + + >>> _format_line(line, 1, 50, increase_indent=False) + ' This is a very long string that should be\\n wrapped hopefully\\n' + + >>> _format_line(line, 0, 50) + 'This is a very long string that should be wrapped\\n hopefully\\n' + + >>> line = 'short line' + >>> _format_line(line, 0, 2, increase_indent=False) + 'short\\nline\\n' + """ + in_squote = False + in_dquote = False + outline = '' + indent_str = _indent_str(indent) + curr_indent = len(indent_str) + wrap_points = list() + line = line_in.strip() + llen = len(line) + if (line_wrap <= 0) or (llen + curr_indent <= line_wrap): + index = llen + 1 + else: + index = 0 + while index < llen: + inchar = line[index] + if in_squote: + if inchar == "'": + in_squote = False + elif in_dquote: + if inchar == '"': + in_dquote = False + elif inchar == ' ': + wrap_points.append(index + curr_indent) + index += 1 + if (line_wrap <= 0) or (llen + curr_indent <= line_wrap): + this_line = indent_str + line + next_line = "" + else: + good_points = [x for x in wrap_points if x <= line_wrap] + if increase_indent: + indent += 2 + if good_points: + wrap = max(good_points) - curr_indent + this_line = indent_str + line[0:wrap] + next_line = _format_line(line[wrap+1:], indent, line_wrap, + increase_indent=False) + elif wrap_points: + wrap = min(wrap_points) - curr_indent + this_line = indent_str + line[0:wrap] + next_line = _format_line(line[wrap+1:], indent, line_wrap, + increase_indent=False) + else: + this_line = indent_str + line + next_line = "" + outline = this_line + '\n' + next_line + return outline + + +def table_entry_pretty_print(entry, indent, line_wrap=-1): + """Create and return a pretty print string of the contents of . + >>> table = ET.fromstring("") + >>> table_entry_pretty_print(table, 0) + '\\n \\n\\n' + + >>> table_entry_pretty_print(table, 1, line_wrap=20) + ' \\n \\n \\n' + """ + output = "" + outline = "<{}".format(entry.tag) + for name in entry.attrib: + outline += " {}={}".format(name, entry.attrib[name]) + has_children = len(list(entry)) > 0 + has_text = entry.text + if has_children or has_text: + outline += ">" + output += _format_line(outline, indent, line_wrap) + else: + outline += " />" + output += _format_line(outline, indent, line_wrap) + if has_children: + for child in entry: + output += table_entry_pretty_print(child, indent+1, + line_wrap=line_wrap) + if has_text: + output += _format_line(entry.text, indent+1, line_wrap) + if has_children or has_text: + outline = "".format(entry.tag) + output = output.rstrip() + '\n' + _format_line(outline, + indent, line_wrap) + return output + + +def datatable_pretty_print(datatable, indent, line_wrap): + """Create and return a pretty print string of the contents of .""" + indent = 0 + table = _read_datatable(datatable) + report = table_entry_pretty_print(table, indent, line_wrap=line_wrap) + return report + + +### +### Main entry point +### + + +def main(argv: Optional[List[str]] = None) -> int: + global _INDENT_STR + if argv is None: + argv = sys.argv[1:] + pargs = parse_command_line(argv) + if pargs.show: + _INDENT_STR = " " * pargs.indent + report = datatable_pretty_print(pargs.datatable, 0, + line_wrap=pargs.line_wrap) + else: + arg_vars = vars(pargs) + action = None + errmsg = '' + esep = '' + for opt in arg_vars: + if (opt in DatatableReport.valid_actions()) and arg_vars[opt]: + if action: + errmsg += esep + "Duplicate action, '{}'".format(opt) + esep = '\n' + else: + action = DatatableReport(opt, arg_vars[opt]) + if errmsg: + raise ValueError(errmsg) + report = datatable_report(pargs.datatable, action, + pargs.sep, pargs.exclude_protected) + print("{}".format(report.rstrip())) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/capgen-ng/ccpp_validator.py b/capgen-ng/ccpp_validator.py new file mode 100755 index 00000000..89161830 --- /dev/null +++ b/capgen-ng/ccpp_validator.py @@ -0,0 +1,623 @@ +#!/usr/bin/env python3 + +"""ccpp_validator — validate Fortran source files against CCPP scheme metadata. + +For each scheme phase declared in a ``.meta`` file this tool checks that the +corresponding Fortran subroutine: + +1. **Exists** in the Fortran source tree. +2. Has the **same number of dummy arguments** as declared in the metadata. +3. The dummy-argument **names match** the ``local_name`` values in the metadata + (order-insensitive). + +The tool does *not* parse full Fortran type declarations — that level of +verification is intentionally kept out of the code generator path (see design +doc: toolchain structure). + +Usage +----- +:: + + ccpp_validator.py \\ + --scheme-files scheme1.meta,scheme2.meta \\ + --source-files scheme1.F90,scheme2.F90 \\ + [--verbose] + +Exit codes +---------- +0 — all checks passed +1 — one or more validation errors found +2 — internal / usage error +""" + +import argparse +import logging +import os +import re +import sys +from typing import Dict, List, NamedTuple, Optional, Set + +# Ensure the capgen-ng package is importable when invoked directly. +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_PACKAGE_DIR = os.path.dirname(_SCRIPT_DIR) +if _PACKAGE_DIR not in sys.path: + sys.path.insert(0, _PACKAGE_DIR) + +from metadata.parse_tools import CCPPError, init_log, set_log_level +from metadata.variable_resolver import SchemeStore +from metadata.metadata_table import parse_metadata_file + +_LOGGER = init_log('ccpp_validator') + +# --------------------------------------------------------------------------- +# Fortran source parsing helpers +# --------------------------------------------------------------------------- + +# Matches the start of a subroutine definition (case-insensitive, optional +# prefixes recursive/pure/elemental). +_SUB_RE = re.compile( + r'(?i)\s*(?:(?:recursive|pure|elemental|impure)\s+)*' + r'subroutine\s+(\w+)\s*(?:\(([^)]*)\))?' +) +# Matches `end subroutine [name]` (case-insensitive). Bare ``end`` is not +# detected — CCPP scheme code consistently uses the explicit form. +_END_SUB_RE = re.compile(r'(?i)^\s*end\s*subroutine\b') +_CONT_RE = re.compile(r'&\s*(?:!.*)?$') # Fortran line continuation +_COMMENT_RE = re.compile(r'!.*$') +# Fixed-form continuation marker. F77 / fixed-form Fortran requires a +# non-blank character in column 6 of a continuation line; CCPP code +# conventionally uses ``&`` for this and pairs it with a trailing ``&`` +# on the prior line for portability with free-form parsers. Only +# applied when we know we are mid-continuation (the buffer is non-empty). +_LEAD_CONT_RE = re.compile(r'^\s*&\s?') + + +class _SubSig(NamedTuple): + """Parsed signature of one Fortran subroutine. + + Attributes + ---------- + args : list of str + Lowercase dummy-argument names in declaration order. + optional : set of str + Subset of *args* declared with the ``optional`` attribute in the + subroutine body. These args may be absent from the metadata + without producing a validation error — they will simply never be + passed at the cap call site. + """ + args: List[str] + optional: Set[str] + + +def _paren_aware_split(s: str, sep: str) -> List[str]: + """Split *s* on *sep*, ignoring separators inside balanced parentheses. + + Examples + -------- + >>> _paren_aware_split('integer, optional, intent(in)', ',') + ['integer', ' optional', ' intent(in)'] + >>> _paren_aware_split('x, y(:,:), z', ',') + ['x', ' y(:,:)', ' z'] + """ + result: List[str] = [] + depth = 0 + buf = '' + for ch in s: + if ch == '(': + depth += 1 + buf += ch + elif ch == ')': + depth -= 1 + buf += ch + elif ch == sep and depth == 0: + result.append(buf) + buf = '' + else: + buf += ch + if buf: + result.append(buf) + return result + + +def _line_optional_names(line: str) -> List[str]: + """Return lowercase var names from a type-decl line carrying ``optional``. + + Matches Fortran lines of the form + `` [, ...] :: [, ...]`` where one of the + comma-separated attributes (paren-aware) is the bare token + ``optional``. Returns an empty list when ``::`` is absent or when no + ``optional`` attribute is present. + + Examples + -------- + >>> _line_optional_names('integer, optional, intent(in) :: innie') + ['innie'] + >>> _line_optional_names('real, intent(out), optional :: outie') + ['outie'] + >>> _line_optional_names('real(kind=kind_phys), optional :: x, y(:,:)') + ['x', 'y'] + >>> _line_optional_names('integer :: not_optional') + [] + >>> _line_optional_names(' ! a comment, optional :: not_a_decl') + [] + """ + if '::' not in line: + return [] + before, _, after = line.partition('::') + attrs = [a.strip().lower() for a in _paren_aware_split(before, ',')] + if 'optional' not in attrs: + return [] + names: List[str] = [] + for tok in _paren_aware_split(after, ','): + m = re.match(r'\s*(\w+)', tok) + if m: + names.append(m.group(1).lower()) + return names + + +def _join_continuation(lines: List[str]) -> List[str]: + """Join Fortran continuation lines (ending with ``&``) into single logical lines. + + Handles both free-form (``&`` only at the trailing end of the prior + line) and fixed-form / dual-form continuation (``&`` at the trailing + end *and* at column 6 of the next line). In dual-form code the + leading ``&`` is part of the continuation marker, not the continued + expression — it must be stripped before the next line is appended + to the buffer. + + Examples + -------- + >>> _join_continuation([' foo &\\n', ' bar\\n', ' baz\\n']) + [' foo bar', ' baz'] + >>> _join_continuation([' foo &\\n', ' & bar\\n', ' baz\\n']) + [' foo bar', ' baz'] + """ + result = [] + buf = '' + for raw in lines: + line = raw.rstrip('\n').rstrip('\r') + stripped = _COMMENT_RE.sub('', line) + if buf and not stripped.strip(): + # Mid-continuation, and this line is blank or a pure + # comment. Fortran 90+ permits such lines interleaved + # between continuation lines without ending the logical + # line — skip it and keep accumulating. + continue + if buf: + # Mid-continuation: drop a leading ``&`` (fixed-form + # column-6 marker) so it doesn't end up glued into the + # continued expression. No-op on free-form code. + stripped = _LEAD_CONT_RE.sub('', stripped, count=1) + if _CONT_RE.search(stripped): + buf += _CONT_RE.sub('', stripped) + else: + buf += stripped + result.append(buf) + buf = '' + if buf: + result.append(buf) + return result + + +def _parse_subroutines(source: str) -> Dict[str, _SubSig]: + """Extract subroutine signatures from Fortran *source*. + + Returns a mapping ``{subroutine_name_lower: _SubSig}`` where each + :class:`_SubSig` carries the dummy-argument list (in declaration + order) and the subset of those args declared with the ``optional`` + attribute in the subroutine body. + + Only the first definition of each subroutine name is recorded + (Fortran does not allow overloading at the subroutine level). + Subroutine and argument names are lowercased for case-insensitive + comparison. + + Optional-attribute detection scans body lines for type-declaration + lines of the form ``, ..., optional, ... :: [, ]...`` + while a tracking session is open for that subroutine. Optional + declarations inside a *nested* subroutine with the same name as an + outer subroutine are attributed to the inner sub (which is then + discarded by the first-occurrence-wins rule); optional declarations + inside a nested sub with a *different* name are correctly + attributed to that nested sub. + + Examples + -------- + >>> src = 'subroutine foo(a, b, c)\\n integer, intent(in) :: a, b, c\\nend subroutine foo\\n' + >>> result = _parse_subroutines(src) + >>> result['foo'].args + ['a', 'b', 'c'] + >>> sorted(result['foo'].optional) + [] + >>> src2 = 'subroutine bar()\\nend subroutine bar\\n' + >>> _parse_subroutines(src2)['bar'].args + [] + >>> src3 = ('subroutine baz(x, y, z)\\n' + ... ' integer, intent(in) :: x\\n' + ... ' integer, optional, intent(in) :: y\\n' + ... ' integer, intent(out), optional :: z\\n' + ... 'end subroutine baz\\n') + >>> sig = _parse_subroutines(src3)['baz'] + >>> sig.args + ['x', 'y', 'z'] + >>> sorted(sig.optional) + ['y', 'z'] + """ + logical = _join_continuation(source.splitlines(keepends=True)) + args_by_name: Dict[str, List[str]] = {} + optional_by_name: Dict[str, Set[str]] = {} + # Stack of names whose body we are currently scanning. Each entry is + # the recorded name (for which we collect optionals) or ``None`` when + # this is a duplicate-name sub whose body should be skipped for the + # purpose of optional-attribution (its args were already discarded). + stack: List[Optional[str]] = [] + + for line in logical: + m = _SUB_RE.match(line) + if m: + name = m.group(1).lower() + arglist_raw = m.group(2) or '' + args = [a.strip().lower() for a in arglist_raw.split(',') + if a.strip()] + if name not in args_by_name: + args_by_name[name] = args + optional_by_name[name] = set() + stack.append(name) + else: + stack.append(None) # duplicate: ignore + continue + if _END_SUB_RE.match(line): + if stack: + stack.pop() + continue + if stack and stack[-1] is not None: + tracked = stack[-1] + arg_set = set(args_by_name[tracked]) + for n in _line_optional_names(line): + if n in arg_set: + optional_by_name[tracked].add(n) + + return { + name: _SubSig(args=args_by_name[name], + optional=optional_by_name[name]) + for name in args_by_name + } + + +def _load_source_tree(source_files: List[str]) -> Dict[str, _SubSig]: + """Read all Fortran source files and return a merged subroutine dict. + + Parameters + ---------- + source_files : list of str + Paths to ``.F90`` / ``.f90`` files. + + Returns + ------- + dict + Merged ``{subroutine_name_lower: _SubSig}``; first occurrence + wins if the same name appears in multiple files. + """ + merged: Dict[str, _SubSig] = {} + for fpath in source_files: + with open(fpath) as fh: + src = fh.read() + for name, sig in _parse_subroutines(src).items(): + if name not in merged: + merged[name] = sig + return merged + + +# --------------------------------------------------------------------------- +# Validation logic +# --------------------------------------------------------------------------- + +def _validate_scheme( + scheme_name: str, + scheme_store: SchemeStore, + subroutine_tree: Dict[str, _SubSig], + logger: logging.Logger, +) -> List[str]: + """Validate one scheme against *subroutine_tree*. + + Optional Fortran-only args (declared with the ``optional`` attribute + in the body and absent from the scheme metadata) are silently allowed + — they will never be passed at the cap call site, so the host need + not declare or provide them. Non-optional Fortran args missing from + the metadata, and metadata args missing from Fortran, remain hard + errors. + + Parameters + ---------- + scheme_name : str + scheme_store : SchemeStore + subroutine_tree : dict + logger : Logger + + Returns + ------- + list of str + Error messages (empty if all checks passed). + """ + errors: List[str] = [] + for phase in scheme_store.phases_for(scheme_name): + sub_name = '{}_{}'.format(scheme_name, phase).lower() + meta_vars = scheme_store.variables_for(scheme_name, phase) or [] + + logger.debug("Checking %s (phase=%s, sub=%s)", scheme_name, phase, sub_name) + + if sub_name not in subroutine_tree: + errors.append( + "Subroutine '{}' declared in metadata (scheme '{}', phase '{}') " + "not found in any source file.".format(sub_name, scheme_name, phase) + ) + continue + + sig = subroutine_tree[sub_name] + fort_args: List[str] = sig.args + fort_optional: Set[str] = sig.optional + meta_local_names: List[str] = [v.local_name.lower() for v in meta_vars] + + meta_set: Set[str] = set(meta_local_names) + fort_set: Set[str] = set(fort_args) + # Optional Fortran args that are absent from the metadata are + # silently allowed — never passed at the call site. + fort_only_optional: Set[str] = (fort_set - meta_set) & fort_optional + # Effective Fortran arg count for the count-mismatch check + # excludes those silent optional-only-in-Fortran args. + effective_fort_count = len(fort_args) - len(fort_only_optional) + + if len(meta_local_names) != effective_fort_count: + extra = '' + if fort_only_optional: + extra = ' (plus {} optional-only-in-Fortran args silently ' \ + 'allowed: {})'.format( + len(fort_only_optional), + sorted(fort_only_optional), + ) + # Degenerate-parse hint: if the Fortran subroutine was + # found but yielded zero args while metadata declares + # many, the parser almost certainly failed on the + # signature — most often because the file uses a + # continuation style (or other Fortran dialect feature) + # not handled by ``_join_continuation``. Flag it so the + # error trace points at the actual cause instead of a + # spurious "every metadata arg is missing" diff. + if len(fort_args) == 0 and len(meta_local_names) > 0: + extra += ( + " HINT: the Fortran signature parser found the " + "subroutine but extracted zero arguments. This is " + "almost always a parser bug, not a real mismatch — " + "common causes are unsupported continuation styles " + "or unusual signature syntax. Check the .F90 file " + "for the subroutine declaration and report a " + "validator bug if the signature looks normal." + ) + errors.append( + "Argument count mismatch for '{}': " + "metadata declares {} args {}, " + "Fortran declares {} required args.{}".format( + sub_name, + len(meta_local_names), meta_local_names, + effective_fort_count, extra, + ) + ) + + only_meta = meta_set - fort_set + only_fort_required = (fort_set - meta_set) - fort_optional + if only_meta: + errors.append( + "Arguments in metadata but not Fortran for '{}': {}".format( + sub_name, sorted(only_meta) + ) + ) + if only_fort_required: + errors.append( + "Non-optional arguments in Fortran but not metadata for '{}': {}".format( + sub_name, sorted(only_fort_required) + ) + ) + return errors + + +_FORTRAN_EXTENSIONS = ('.F90', '.f90', '.F', '.f') + + +def _fortran_file_for_table(table) -> Optional[str]: + """Return the Fortran source path for a scheme *table* using ``source_path``. + + The convention is that the ``.F90`` file has the same base name as the + ``.meta`` file but lives in the directory given by ``table.source_path`` + (which defaults to the ``.meta`` file's own directory when not set). + + Returns ``None`` if no matching file is found. + + Parameters + ---------- + table : MetadataTable + A scheme table with ``file_path`` and ``source_path`` set. + + Returns + ------- + str or None + Absolute path to the Fortran source file, or ``None``. + + Examples + -------- + >>> from metadata.parse_tools import ParseContext + >>> from metadata.metadata_table import MetadataTable + >>> import tempfile, os + >>> with tempfile.TemporaryDirectory() as d: + ... meta = os.path.join(d, 'foo.meta') + ... fort = os.path.join(d, 'foo.F90') + ... open(fort, 'w').close() + ... ctx = ParseContext(0, meta) + ... t = MetadataTable('foo', 'scheme', meta, ctx) + ... t.apply_table_props({}) + ... os.path.basename(_fortran_file_for_table(t)) + 'foo.F90' + """ + meta_base = os.path.splitext(os.path.basename(table.file_path))[0] + search_dir = table.source_path or os.path.dirname(os.path.abspath(table.file_path)) + for ext in _FORTRAN_EXTENSIONS: + candidate = os.path.join(search_dir, meta_base + ext) + if os.path.isfile(candidate): + return candidate + return None + + +def validate( + scheme_files: List[str], + source_files: Optional[List[str]] = None, + logger: Optional[logging.Logger] = None, +) -> List[str]: + """Validate scheme metadata against Fortran source files. + + When *source_files* is ``None`` or empty, the validator resolves the + Fortran source for each scheme automatically using the ``source_path`` + attribute from the metadata (defaulting to the ``.meta`` file's directory + if ``source_path`` is absent). Pass an explicit list to override. + + Parameters + ---------- + scheme_files : list of str + Paths to scheme ``.meta`` files. + source_files : list of str, optional + Explicit Fortran source files to scan. If omitted, auto-discovered + via ``source_path`` in the metadata. + logger : Logger, optional + + Returns + ------- + list of str + All validation error messages (empty means success). + + Raises + ------ + CCPPError + On metadata parse errors or missing files. + """ + log = logger or _LOGGER + + log.info("Loading scheme metadata from %d file(s)", len(scheme_files)) + all_tables = [] + for fpath in scheme_files: + all_tables.extend(parse_metadata_file(fpath)) + scheme_store = SchemeStore.build_from(all_tables) + log.info("Found %d scheme(s): %s", len(scheme_store.scheme_names()), scheme_store.scheme_names()) + + if source_files: + log.info("Scanning %d explicit Fortran source file(s)", len(source_files)) + resolved_sources = list(source_files) + else: + # Auto-discover Fortran files via source_path in each scheme table. + resolved_sources = [] + for tbl in all_tables: + if not tbl.is_scheme: + continue + fort = _fortran_file_for_table(tbl) + if fort: + resolved_sources.append(fort) + log.debug("Resolved Fortran source for '%s': %s", tbl.table_name, fort) + else: + log.warning( + "No Fortran source found for scheme '%s' (source_path='%s')", + tbl.table_name, tbl.source_path, + ) + subroutine_tree = _load_source_tree(resolved_sources) + log.info("Found %d subroutine definitions", len(subroutine_tree)) + + all_errors: List[str] = [] + for sname in scheme_store.scheme_names(): + errs = _validate_scheme(sname, scheme_store, subroutine_tree, log) + all_errors.extend(errs) + + if all_errors: + log.warning("%d validation error(s) found.", len(all_errors)) + else: + log.info("Validation passed.") + return all_errors + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog='ccpp_validator.py', + description='Validate CCPP scheme Fortran source against metadata', + ) + parser.add_argument( + '--scheme-files', + required=True, + metavar='FILE[,FILE...]', + help='Comma-separated scheme metadata (.meta) files', + ) + parser.add_argument( + '--source-files', + required=True, + metavar='FILE[,FILE...]', + help='Comma-separated Fortran source (.F90) files', + ) + parser.add_argument( + '--verbose', '-v', + action='count', + default=0, + help='Increase verbosity (use twice for DEBUG)', + ) + # legacy-compat: transient migration shim (delete the argument, + # the enable() call below, and the rest of the legacy_compat + # touchpoints when the migration is complete). + parser.add_argument( + '--legacy-mode', + action='store_true', + help=( + "TRANSIENT MIGRATION SHIM. Accept legacy CCPP standard " + "names (currently 'horizontal_loop_extent') in scheme " + "metadata and silently rewrite them to their canonical " + "capgen-ng equivalents ('horizontal_dimension'). Emits a " + "loud warning at startup. Will be removed." + ), + ) + return parser + + +def main(argv: Optional[List[str]] = None) -> int: + parser = _build_parser() + args = parser.parse_args(argv) + + if args.verbose == 0: + set_log_level(_LOGGER, logging.WARNING) + elif args.verbose == 1: + set_log_level(_LOGGER, logging.INFO) + else: + set_log_level(_LOGGER, logging.DEBUG) + + # legacy-compat: transient migration shim. Emit the loud banner + # before any parsing happens so user has fair warning. + if args.legacy_mode: + from metadata import legacy_compat + legacy_compat.enable(_LOGGER) + + scheme_files = [f.strip() for f in args.scheme_files.split(',') if f.strip()] + source_files = [f.strip() for f in args.source_files.split(',') if f.strip()] + + try: + errors = validate(scheme_files, source_files) + except CCPPError as exc: + _LOGGER.error("%s", exc) + return 2 + except OSError as exc: + _LOGGER.error("File error: %s", exc) + return 2 + + if errors: + for err in errors: + print("ERROR:", err, file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/capgen-ng/generator/__init__.py b/capgen-ng/generator/__init__.py new file mode 100644 index 00000000..bb54d8e4 --- /dev/null +++ b/capgen-ng/generator/__init__.py @@ -0,0 +1 @@ +"""Cap code generation for ccpp-capgen-ng.""" diff --git a/capgen-ng/generator/datatable.py b/capgen-ng/generator/datatable.py new file mode 100644 index 00000000..6c26b0d4 --- /dev/null +++ b/capgen-ng/generator/datatable.py @@ -0,0 +1,430 @@ +#!/usr/bin/env python3 + +"""Write ``datatable.xml`` — the generator database consumed by ``ccpp_datafile.py`` +and CMake to discover compilation targets, scheme modules, and suite structure. + +XML layout +---------- + +Two top-level file sections partition capgen-ng's outputs by language: + +* ```` lists *Fortran sources only* — utilities, the host-facing + API, and per-suite cap modules. All paths here end in ``.F90`` and are + intended to be compiled by the host build system. +* ```` lists *non-Fortran* artifacts — generated + ``.meta`` files and expanded SDF XML. These are produced for debugging + and downstream tooling; they are *not* compiled. + +:: + + + + + + /abs/path/ccpp_kinds.F90 + + + /abs/path/ccpp_static_api.F90 + + + /abs/path/ccpp__cap.F90 + ... + + + + + /abs/path/ccpp_.meta + ... + + + /abs/path/ccpp__expanded.xml + ... + + + + + + + + ... + + + + ... + + ... + + + + + + + ... + + + + + + + + + + + ... + + + + + + + + + + + + + + + ... + + + ... + + + + +Compatibility notes +------------------- +The ```` section uses the same phase element tag names as the new +generator's phase vocabulary (``init``, ``run``, ``timestep_init``, +``timestep_final``, ``final``, ``register``). ``ccpp_datafile.py`` iterates +scheme children by tag rather than filtering on specific names, so this is +forward-compatible. +""" + +import os +import xml.etree.ElementTree as ET +from typing import Dict, List, Optional, Set, Tuple + +from generator.suite_resolver import SuiteResolution, iter_phase_calls + +_API_DICT_NAME = 'ccpp_api' + + +def _build_capgen_files( + root: ET.Element, + utility_paths: List[str], + host_file_paths: List[str], + suite_file_paths: List[str], +) -> None: + """Append ```` to *root*. + + ```` lists Fortran sources generated by capgen-ng (utilities, + the host-facing API, and per-suite caps). Non-Fortran inspection + artifacts (e.g. ``.meta`` files, expanded SDFs) live in + ```` and are emitted by :func:`_build_inspection_files`. + + ``host_file_paths`` lists files generated for the host-facing API. In + capgen-ng this is the static API (``ccpp_static_api.F90``); the section is + emitted unconditionally (possibly empty) to keep the schema stable. + """ + capgen_files = ET.SubElement(root, 'capgen_files') + + utilities = ET.SubElement(capgen_files, 'utilities') + for path in utility_paths: + f = ET.SubElement(utilities, 'file') + f.text = path + + host_files = ET.SubElement(capgen_files, 'host_files') + for path in host_file_paths: + f = ET.SubElement(host_files, 'file') + f.text = path + + suite_files = ET.SubElement(capgen_files, 'suite_files') + for path in suite_file_paths: + f = ET.SubElement(suite_files, 'file') + f.text = path + + +def _build_inspection_files( + root: ET.Element, + suite_meta_paths: Optional[List[str]] = None, + expanded_sdf_paths: Optional[List[str]] = None, +) -> None: + """Append ```` to *root*. + + Inspection artifacts are non-Fortran outputs that capgen-ng emits for + debugging and downstream tooling. They are *not* compiled. Each kind + of artifact lives in its own subsection so consumers can pick a specific + file type or take them all together via ``--inspection-files``. + + Subsections written: + + * ```` — generated ``ccpp_.meta`` files (the + suite-data variable metadata table). + * ```` — fully expanded suite-definition XML + (``ccpp__expanded.xml``), with any ```` references + resolved. + + The section is always written (possibly with empty subsections) to keep + the schema stable. + """ + inspection = ET.SubElement(root, 'inspection_files') + + suite_meta_files = ET.SubElement(inspection, 'suite_meta_files') + for path in suite_meta_paths or []: + f = ET.SubElement(suite_meta_files, 'file') + f.text = path + + expanded_sdf_files = ET.SubElement(inspection, 'expanded_sdf_files') + for path in expanded_sdf_paths or []: + f = ET.SubElement(expanded_sdf_files, 'file') + f.text = path + + +def _build_schemes( + root: ET.Element, + suite_resolutions: List[SuiteResolution], + scheme_store, +) -> None: + """Append ```` to *root*. + + Each scheme appears once; its phases are the union of phases seen across + all suites. Duplicate scheme names (scheme used in multiple suites) are + deduplicated — the call_list is the same regardless of which suite uses + the scheme. + """ + schemes_elem = ET.SubElement(root, 'schemes') + seen_scheme_phases: Dict[str, Set[str]] = {} + + for sr in suite_resolutions: + for rg in sr.groups: + for phase_name, items in rg.phase_calls.items(): + for rc in iter_phase_calls(items): + sname = rc.scheme_name + if sname not in seen_scheme_phases: + seen_scheme_phases[sname] = set() + seen_scheme_phases[sname].add(phase_name) + + for sname in sorted(seen_scheme_phases): + scheme_elem = ET.SubElement(schemes_elem, 'scheme') + scheme_elem.set('name', sname) + for phase in sorted(seen_scheme_phases[sname]): + phase_elem = ET.SubElement(scheme_elem, phase) + phase_elem.set('name', sname) + phase_elem.set('subroutine_name', '{}_{}'.format(sname, phase)) + phase_elem.set('module', sname) + call_list = ET.SubElement(phase_elem, 'call_list') + mvars = scheme_store.variables_for(sname, phase) + if mvars: + for mv in mvars: + var = ET.SubElement(call_list, 'var') + var.set('name', mv.standard_name) + var.set('intent', mv.intent or '') + var.set('local_name', mv.local_name) + if mv.diagnostic_name: + var.set('diagnostic_name', mv.diagnostic_name) + if mv.diagnostic_name_fixed: + var.set('diagnostic_name_fixed', + mv.diagnostic_name_fixed) + + +def _build_api( + root: ET.Element, + suite_resolutions: List[SuiteResolution], +) -> None: + """Append ``...`` to *root*.""" + api_elem = ET.SubElement(root, 'api') + suites_elem = ET.SubElement(api_elem, 'suites') + + for sr in suite_resolutions: + suite_elem = ET.SubElement(suites_elem, 'suite') + suite_elem.set('name', sr.suite_name) + for rg in sr.groups: + group_elem = ET.SubElement(suite_elem, 'group') + group_elem.set('name', rg.group_name) + seen: Set[str] = set() + for items in rg.phase_calls.values(): + for rc in iter_phase_calls(items): + if rc.scheme_name not in seen: + seen.add(rc.scheme_name) + sch = ET.SubElement(group_elem, 'scheme') + sch.text = rc.scheme_name + + +def _build_var_dictionaries( + root: ET.Element, + host_dict, + host_name: str, + suite_resolutions: List[SuiteResolution], +) -> None: + """Append ```` to *root*. + + Emits one dictionary entry per scope in the lookup chain + (``host`` → ``api`` → ``suite`` → ``group`` → ``group_call_list``). + The ``parent`` attribute on each entry preserves the inheritance chain + that ``ccpp_datafile.py`` walks when resolving ``protected`` and + ``--exclude-protected``. + + The host dictionary carries one ```` per entry in *host_dict* + (host model + control vars). Each group's call-list dictionary + carries one ```` per (standard_name, intent) tuple seen at any + phase call site in that group. + """ + if host_dict is None: + return + + dicts = ET.SubElement(root, 'var_dictionaries') + + host_d = ET.SubElement(dicts, 'var_dictionary') + host_d.set('name', host_name) + host_d.set('type', 'host') + h_vars = ET.SubElement(host_d, 'variables') + for std_name in sorted(host_dict): + entry = host_dict[std_name] + v = ET.SubElement(h_vars, 'var') + v.set('name', std_name) + if entry.local_name: + v.set('local_name', entry.local_name) + if getattr(entry, 'protected', False): + v.set('protected', 'True') + + api_d = ET.SubElement(dicts, 'var_dictionary') + api_d.set('name', _API_DICT_NAME) + api_d.set('type', 'api') + api_d.set('parent', host_name) + ET.SubElement(api_d, 'variables') + + for sr in suite_resolutions: + suite_d = ET.SubElement(dicts, 'var_dictionary') + suite_d.set('name', sr.suite_name) + suite_d.set('type', 'suite') + suite_d.set('parent', _API_DICT_NAME) + ET.SubElement(suite_d, 'variables') + + for rg in sr.groups: + group_d = ET.SubElement(dicts, 'var_dictionary') + group_d.set('name', rg.group_name) + group_d.set('type', 'group') + group_d.set('parent', sr.suite_name) + ET.SubElement(group_d, 'variables') + + call_d = ET.SubElement(dicts, 'var_dictionary') + call_d.set('name', '{}_call_list'.format(rg.group_name)) + call_d.set('type', 'group_call_list') + call_d.set('parent', rg.group_name) + cl_vars = ET.SubElement(call_d, 'variables') + + seen: Set[Tuple[str, str]] = set() + for items in rg.phase_calls.values(): + for rc in iter_phase_calls(items): + for arg in rc.args: + intent = arg.intent or '' + key = (arg.standard_name, intent) + if key in seen: + continue + seen.add(key) + v = ET.SubElement(cl_vars, 'var') + v.set('name', arg.standard_name) + if intent: + v.set('intent', intent) + if arg.scheme_local_name: + v.set('local_name', arg.scheme_local_name) + + +def write_datatable( + suite_resolutions: List[SuiteResolution], + scheme_store, + utility_paths: List[str], + suite_file_paths: List[str], + output_root: str, + host_file_paths: Optional[List[str]] = None, + dependency_paths: Optional[List[str]] = None, + suite_meta_paths: Optional[List[str]] = None, + expanded_sdf_paths: Optional[List[str]] = None, + host_dict=None, + host_name: str = 'host', +) -> str: + """Write ``datatable.xml`` and return its absolute path. + + Parameters + ---------- + suite_resolutions : list of SuiteResolution + scheme_store : SchemeStore + utility_paths : list of str + Absolute paths to utility Fortran files (e.g. ``ccpp_kinds.F90``). + suite_file_paths : list of str + Absolute paths to generated suite cap files. + output_root : str + Output directory. + host_file_paths : list of str, optional + Absolute paths to host-facing API files (capgen-ng emits + ``ccpp_static_api.F90`` here). The ```` section is + always written (possibly empty). + dependency_paths : list of str, optional + Absolute paths of scheme dependency files (collected from + ``MetadataTable.dependencies``). Written as ```` + children of the ```` element. + suite_meta_paths : list of str, optional + Absolute paths to generated ``ccpp_.meta`` files (inspection + artifacts; written under ````). + expanded_sdf_paths : list of str, optional + Absolute paths to generated ``ccpp__expanded.xml`` files + (inspection artifacts; written under + ````). + + Returns + ------- + str + Absolute path to the written ``datatable.xml``. + + Examples + -------- + >>> import tempfile, os + >>> from generator.datatable import write_datatable + >>> from unittest.mock import MagicMock + >>> sr = MagicMock() + >>> sr.suite_name = 'test' + >>> sr.groups = [] + >>> store = MagicMock() + >>> with tempfile.TemporaryDirectory() as d: + ... path = write_datatable([sr], store, [], [], d) + ... os.path.basename(path) + 'datatable.xml' + """ + os.makedirs(output_root, exist_ok=True) + + root = ET.Element('ccpp_datatable') + root.set('version', '1.0') + + _build_capgen_files( + root, utility_paths, host_file_paths or [], + suite_file_paths, + ) + _build_inspection_files( + root, + suite_meta_paths=suite_meta_paths, + expanded_sdf_paths=expanded_sdf_paths, + ) + _build_schemes(root, suite_resolutions, scheme_store) + _build_api(root, suite_resolutions) + _build_var_dictionaries(root, host_dict, host_name, suite_resolutions) + + deps_elem = ET.SubElement(root, 'dependencies') + for dep in sorted(set(dependency_paths or [])): + d = ET.SubElement(deps_elem, 'dependency') + d.text = dep + + tree = ET.ElementTree(root) + ET.indent(tree, space=' ') + out_path = os.path.join(os.path.abspath(output_root), 'datatable.xml') + tree.write(out_path, encoding='unicode', xml_declaration=True) + + # Ensure file ends with a newline. + with open(out_path, 'a') as fh: + fh.write('\n') + + return out_path diff --git a/capgen-ng/generator/group_cap.py b/capgen-ng/generator/group_cap.py new file mode 100644 index 00000000..3d44de07 --- /dev/null +++ b/capgen-ng/generator/group_cap.py @@ -0,0 +1,1186 @@ +#!/usr/bin/env python3 + +"""Generate group-level cap Fortran source files. + +A group cap module (``ccpp___cap.F90``) contains one subroutine +per phase. Each subroutine: + +* USEs host-model modules for the variables it references. +* Declares control-variable dummy arguments. +* Declares transformation temporaries and optional pointer variables. +* Calls each scheme in the group in suite-XML order. +* Applies pre-call (forward) and post-call (backward) transformations. + +Subcycle loops wrap scheme calls enclosed by ```` elements in the +suite XML. + +Init deduplication (Section 12) is handled by the ``initialized`` guard array +declared in this module: an ``initialized(number_of_instances)`` integer array +is allocated at suite-init time and reset on suite-final. Each group's init +subroutine calls scheme ``_init`` routines only when ``initialized(inst) == 0`` +and then sets the element to 1. +""" + +import os +from typing import Dict, List, Optional, Set, Tuple + +from metadata.parse_tools import FORTRAN_CONDITIONAL_REGEX +from metadata.variable_resolver import HostVarEntry +from generator.suite_types import _ptr_type_name, _ptr_type_for_arg +from generator.suite_resolver import ( + ResolvedArg, + ResolvedCall, + ResolvedGroup, + ResolvedSubcycle, + _root_symbol, + iter_phase_calls, + iter_phase_subcycles, +) + +_INDENT = ' ' +_CONT = ' &' + +# Canonical phase order used for code-emission ordering in the group cap: +# subroutine layout, public declarations, scheme-import ``only:`` lists. +# Group caps do not contain a ``register`` phase (handled at suite-cap level); +# the order matches the user-facing ordering convention. +_GROUP_PHASE_ORDER = ('init', 'timestep_init', 'run', 'timestep_final', 'final') + +_CTRL_STDNAMES_ORDER = ( + 'suite_name', + 'group_name', + 'horizontal_loop_begin', + 'horizontal_loop_end', + 'thread_number', + 'number_of_threads', + 'number_of_physics_threads', + 'ccpp_error_code', + 'ccpp_error_message', + 'instance_number', +) + + +def _ctrl_entries_for_signature(host_dict, exclude=None): + """Return all control HostVarEntry objects in canonical order. + + Parameters + ---------- + host_dict : dict + Flat host+control variable dictionary. + exclude : set of str, optional + Standard names to exclude (e.g. ``{'suite_name'}`` at suite cap level). + """ + if host_dict is None: + return [] + exclude_set = set(exclude or []) + result = [] + seen: Set[str] = set() + for std_name in _CTRL_STDNAMES_ORDER: + if std_name in exclude_set: + continue + entry = host_dict.get(std_name) + if entry is not None and entry.is_control: + result.append(entry) + seen.add(std_name) + for std_name, entry in host_dict.items(): + if entry.is_control and std_name not in seen and std_name not in exclude_set: + result.append(entry) + return result + + +######################################################################## +# Fortran type declaration helpers +######################################################################## + +def _fortran_type_str(type_: str, kind: str) -> str: + """Return the Fortran type clause for a declaration. + + >>> _fortran_type_str('real', 'kind_phys') + 'real(kind=kind_phys)' + >>> _fortran_type_str('real', '') + 'real' + >>> _fortran_type_str('integer', '') + 'integer' + >>> _fortran_type_str('character', 'len=512') + 'character(len=512)' + >>> _fortran_type_str('logical', '') + 'logical' + >>> _fortran_type_str('gfs_statein_type', '') + 'type(gfs_statein_type)' + """ + t = type_.strip() + # DDT types (not intrinsic, not external:...) need type(...) syntax. + _INTRINSICS = frozenset({ + 'real', 'integer', 'character', 'logical', 'complex', 'double precision' + }) + if t.lower() not in _INTRINSICS and not t.lower().startswith('external:'): + if not t.lower().startswith('type('): + t = 'type({})'.format(t) + if kind: + if t.lower().startswith('character'): + return 'character({})'.format(kind) + return '{}(kind={})'.format(t, kind) + return t + + +def _dim_decl(dimensions: List[str]) -> str: + """Return the Fortran dimension attribute string for a declaration. + + Returns ``''`` for scalars, ``', dimension(d1, d2, ...)'`` for arrays. + The dimensions here use the declared dimension standard names as-is. + + >>> _dim_decl([]) + '' + >>> _dim_decl(['horizontal_loop_extent']) + ', dimension(horizontal_loop_extent)' + >>> _dim_decl(['horizontal_loop_extent', 'vertical_layer_dimension']) + ', dimension(horizontal_loop_extent, vertical_layer_dimension)' + """ + if not dimensions: + return '' + return ', dimension({})'.format(', '.join(dimensions)) + + +def _dim_decl_local(dimensions: List[str], host_dict) -> str: + """Return the Fortran dimension attribute using local names from host_dict. + + Each standard name in *dimensions* is resolved to the corresponding local + Fortran variable name. Falls back to the standard name when not found + (should not happen with valid metadata). + + Special case for ``horizontal_dimension`` / ``horizontal_loop_extent``: + the temp must match the chunk slice the scheme actually receives at + the call site (``host_var(lb:ub, …)`` via :func:`_one_dim_part`). + Using the host's local name for ``horizontal_dimension`` (e.g. ``ncols``) + would over-size the temp and break the unit-conversion assignment + ``ps_l = factor * phys_state%ps(col_start:col_end)`` with a Fortran + shape mismatch. Emit ``dimension(:)`` instead, where ```` + and ```` are the host's local names for ``horizontal_loop_begin`` / + ``horizontal_loop_end``. + + >>> _dim_decl_local([], None) + '' + """ + if not dimensions: + return '' + locals_ = [] + for std_name in dimensions: + if std_name in ('horizontal_dimension', 'horizontal_loop_extent'): + lb_entry = host_dict.get('horizontal_loop_begin') if host_dict else None + ub_entry = host_dict.get('horizontal_loop_end') if host_dict else None + lb = lb_entry.local_name if lb_entry else 'horizontal_loop_begin' + ub = ub_entry.local_name if ub_entry else 'horizontal_loop_end' + locals_.append('{}:{}'.format(lb, ub)) + else: + entry = host_dict.get(std_name) if host_dict else None + locals_.append(entry.local_name if entry is not None else std_name) + return ', dimension({})'.format(', '.join(locals_)) + + +def _intent_clause(intent: str) -> str: + """Return the Fortran intent attribute. + + >>> _intent_clause('in') + ', intent(in)' + >>> _intent_clause('out') + ', intent(out)' + >>> _intent_clause('inout') + ', intent(inout)' + """ + return ', intent({})'.format(intent) + + +# Control variables that are output-only (set by the routine, not read). +_CTRL_OUT_STDNAMES = frozenset({'ccpp_error_code', 'ccpp_error_message'}) + + +def _ctrl_intent_for(standard_name: str) -> str: + """Return ``'out'`` for error-reporting control vars, else ``'in'``. + + >>> _ctrl_intent_for('ccpp_error_code') + 'out' + >>> _ctrl_intent_for('ccpp_error_message') + 'out' + >>> _ctrl_intent_for('horizontal_loop_begin') + 'in' + """ + return 'out' if standard_name in _CTRL_OUT_STDNAMES else 'in' + + +def _ctrl_local(host_dict, standard_name: str): + """Return the local Fortran name for a control standard_name, or ``None``.""" + if not host_dict: + return None + entry = host_dict.get(standard_name) + return entry.local_name if entry is not None else None + + +######################################################################## +# USE-statement collection +######################################################################## + +def _active_std_names(active: str) -> Set[str]: + """Return the set of identifier tokens from an active expression string. + + Uses the same tokeniser as ``_translate_active_expr`` so that only word + tokens (potential standard names) are returned, not operators or literals. + + >>> sorted(_active_std_names('my_flag .eqv. .true.')) + ['my_flag'] + >>> sorted(_active_std_names('a .or. b')) + ['a', 'b'] + >>> _active_std_names('') + set() + """ + if not active: + return set() + result: Set[str] = set() + for m in FORTRAN_CONDITIONAL_REGEX.finditer(active): + tok = m.group(0) + # Skip Fortran keywords / literals / operators captured by the regex. + if tok.strip() and tok[0].isalpha() and '_' not in tok[:1]: + # Only word-like tokens could be standard names; filter out + # Fortran logical literals and operators (.true., .false., .not., ...) + result.add(tok) + elif tok[0] == '_' or (tok[0].isalpha() and tok.isidentifier()): + result.add(tok) + return result + + +def _collect_group_uses( + rg: ResolvedGroup, + host_dict, +) -> Dict[str, Set[str]]: + """Collect ``{module: {symbol, ...}}`` across all phases of a group. + + Includes direct argument symbols, dimension helper symbols, and variables + referenced in ``active`` conditional expressions. + + Parameters + ---------- + rg : ResolvedGroup + host_dict : dict + Flat host+control variable dictionary (for dimension look-ups). + + Returns + ------- + dict + """ + uses: Dict[str, Set[str]] = {} + + def _add(mod: Optional[str], sym: str) -> None: + if mod is not None: + uses.setdefault(mod, set()).add(sym) + + for items in rg.phase_calls.values(): + for rc in iter_phase_calls(items): + for arg in rc.args: + # Direct argument symbol. + if arg.source != 'control': + _add(arg.module_name, arg.root_symbol) + # Dimension helper symbols (non-control only). + # USE the access-path root, not entry.local_name — DDT-component + # entries have local_name = component (not a free module symbol). + for dim_std in arg.used_dim_std_names: + entry = host_dict.get(dim_std) + if entry is not None and entry.module_name is not None: + _add(entry.module_name, _root_symbol(entry.access_path)) + # Variables referenced in active expressions (Gap 1). + # Same access-path-root rule applies (DDT-component flags + # reach the cap via the DDT instance, not the component). + for std_name in _active_std_names(arg.active): + entry = host_dict.get(std_name) if host_dict else None + if entry is not None and entry.module_name is not None: + _add(entry.module_name, _root_symbol(entry.access_path)) + # Constituent extra symbols (index_of_X, etc.) live in the + # suite cap module along with the constituent arrays. + if arg.source == 'constituent' and arg.constituent_module_name: + for sym in arg.constituent_extra_symbols: + _add(arg.constituent_module_name, sym) + + # Also add dim_uses already collected during resolution. + for mod, syms in rg.dim_uses.items(): + uses.setdefault(mod, set()).update(syms) + + return uses + + +def _collect_kinds_used(rg: ResolvedGroup) -> List[str]: + """Collect kind parameter *names* referenced by transformation temporaries. + + Mirrors the kind-resolution logic in :func:`_generate_phase_subroutine` + (the only place a group cap emits ``kind=`` directly). Returned + names are sorted alphabetically. Excluded: + + * character ``len=...`` specifiers — not kind parameters. + * bare integer literals (``kind = 8``, ``kind = 4``, ...) — valid + Fortran kind specifiers but not module symbols, so they must not + appear in ``use ccpp_kinds, only: ...``. They flow through to the + temp declaration (``real(kind=8)``) and to numeric-literal suffixes + (``1.0_8``) unchanged; only the USE list needs to filter them out. + """ + kinds: Set[str] = set() + for items in rg.phase_calls.values(): + for rc in iter_phase_calls(items): + for arg in rc.args: + if not arg.temp_name: + continue + kind = arg.kind_scheme or ( + arg.host_entry.kind if arg.host_entry else '' + ) + if not kind: + continue + if kind.startswith('len='): + continue + if kind.isdigit(): + continue + kinds.add(kind) + return sorted(kinds) + + +######################################################################## +# Control variable dummy argument handling +######################################################################## + +def _collect_control_args(rg: ResolvedGroup) -> List[ResolvedArg]: + """Return deduplicated control-variable arguments for the group subroutine. + + Returns one ResolvedArg per unique standard_name, in a consistent order. + """ + seen: Dict[str, ResolvedArg] = {} + for items in rg.phase_calls.values(): + for rc in iter_phase_calls(items): + for arg in rc.args: + if arg.source == 'control' and arg.standard_name not in seen: + seen[arg.standard_name] = arg + return list(seen.values()) + + +def _extra_dim_ctrl_entries( + phase_items, + phase: str, + ctrl_args: List[ResolvedArg], + host_dict, +) -> List[HostVarEntry]: + """Return HostVarEntry objects for control vars needed but not in ctrl_args. + + Covers two cases (Gap 3): + + 1. ``instance_number`` — needed for state-array indexing in init/final/ + timestep_init/timestep_final, and for suite-data access + ``ccpp_suite_data(inst_num)%...`` in any phase that references suite vars. + 2. Any control var appearing only in dimension subscripts (``used_dim_std_names``). + """ + if host_dict is None: + return [] + already = {a.standard_name for a in ctrl_args if a.host_entry is not None} + extras: Dict[str, HostVarEntry] = {} + + has_suite_vars = any( + arg.source == 'suite' + for rc in iter_phase_calls(phase_items) + for arg in rc.args + ) + needs_inst = ( + phase in ('init', 'final', 'timestep_init', 'timestep_final') + or has_suite_vars + ) + if needs_inst: + inst_entry = host_dict.get('instance_number') + if inst_entry is not None and 'instance_number' not in already: + extras['instance_number'] = inst_entry + + for rc in iter_phase_calls(phase_items): + for arg in rc.args: + for dim_std in arg.used_dim_std_names: + if dim_std in already or dim_std in extras: + continue + entry = host_dict.get(dim_std) + if entry is not None and entry.is_control: + extras[dim_std] = entry + + # Subcycle loop bounds resolved from a CCPP standard name in the suite + # XML need the same dummy-arg threading when the resolved entry is a + # control variable (host-module entries are USE'd instead and are + # handled by ``_collect_dim_uses``). Walk all nesting levels — + # each level's bound is independently a candidate dummy arg. + for item in iter_phase_subcycles(phase_items): + std = item.loop_std_name + if not std or std in already or std in extras: + continue + entry = host_dict.get(std) + if entry is not None and entry.is_control: + extras[std] = entry + + return list(extras.values()) + + +######################################################################## +# Fortran source line helpers +######################################################################## + +def _use_statements(uses: Dict[str, Set[str]]) -> List[str]: + """Generate sorted USE statements. + + >>> lines = _use_statements({'mod_a': {'sym1', 'sym2'}, 'mod_b': {'sym3'}}) + >>> lines[0] + ' use mod_a, only: sym1, sym2' + >>> lines[1] + ' use mod_b, only: sym3' + """ + result = [] + for mod in sorted(uses): + syms = ', '.join(sorted(uses[mod])) + result.append('{}use {}, only: {}'.format(_INDENT, mod, syms)) + return result + + +def _collect_scheme_uses(rg: ResolvedGroup) -> List[Tuple[str, str, List[str]]]: + """Return ``[(scheme_name, module_name, [phase_routine, ...]), ...]``. + + Schemes are listed in first-seen order across phases (in canonical phase + iteration order); within each scheme the phase routines are listed in + canonical phase order. Each phase routine is the Fortran subroutine name + ``_``. + + ``module_name`` is the Fortran module that exports those subroutines — + typically equal to the scheme name, but overridden by an explicit + ``module_name`` attribute in the scheme's ``[ccpp-table-properties]``. + """ + seen_schemes: Dict[str, Set[str]] = {} + scheme_modules: Dict[str, str] = {} + order: List[str] = [] + for phase in _GROUP_PHASE_ORDER: + for rc in iter_phase_calls(rg.phase_calls.get(phase, [])): + if rc.scheme_name not in seen_schemes: + seen_schemes[rc.scheme_name] = set() + order.append(rc.scheme_name) + seen_schemes[rc.scheme_name].add(phase) + # rc.scheme_module is empty for old/legacy ResolvedCall objects + # built in tests; fall back to the scheme name so emission still + # works. + scheme_modules[rc.scheme_name] = ( + rc.scheme_module or rc.scheme_name + ) + result: List[Tuple[str, str, List[str]]] = [] + for sname in order: + phases_present = [p for p in _GROUP_PHASE_ORDER if p in seen_schemes[sname]] + syms = ['{}_{}'.format(sname, p) for p in phases_present] + result.append((sname, scheme_modules[sname], syms)) + return result + + +def _scheme_use_statements(rg: ResolvedGroup) -> List[str]: + """Generate ``use , only: _, ...`` lines. + + Schemes are emitted in first-seen order; phase routines within each + ``only:`` clause follow the canonical phase order. When the scheme's + ``[ccpp-table-properties]`` declares a ``module_name`` distinct from + the scheme name, the USE statement targets that module. + """ + return [ + '{}use {}, only: {}'.format(_INDENT, mod, ', '.join(syms)) + for _sname, mod, syms in _collect_scheme_uses(rg) + if syms + ] + + +######################################################################## +# Pre/post-call transformation code +######################################################################## + +def _transform_comment(arg: ResolvedArg, reverse: bool = False) -> str: + """Compose the trailing inline comment for a transform assignment. + + Lists every active transform (unit conversion, kind change, vertical + flip) so a reader can tell at a glance what the generated copy does. + + Identity unit conversions (registered for dimensionally-equivalent + spellings such as ``J kg-1`` ↔ ``m2 s-2``, where the formula is just + ``{var}``) are suppressed: the assignment carries no scaling factor + and labelling it as a "unit conversion" is misleading. Detection is + done by comparing the rendered transform expression against the raw + operand — equality means the formula returned the variable unchanged. + """ + bits: List[str] = [] + if arg.needs_unit_transform or arg.needs_kind_transform: + # Suppress identity conversions (formula '{var}' for equivalent + # units; kinds also matching). + if reverse: + is_identity = (arg.unit_backward == arg.temp_name) + else: + is_identity = (arg.unit_forward == arg.call_expr) + if not is_identity: + if reverse: + bits.append('unit conversion: {} to {}'.format( + arg.kind_scheme or '', arg.kind_host or '', + )) + else: + bits.append('unit conversion: {} to {}'.format( + arg.kind_host or '', arg.kind_scheme or '', + )) + if arg.needs_vert_flip: + bits.append('vertical flip (top_at_one mismatch)') + if not bits: + return '' + return '! ' + '; '.join(bits) + + +def _pre_call_lines(arg: ResolvedArg) -> List[str]: + """Generate pre-call Fortran lines for one argument.""" + lines = [] + if arg.transform_case == 1: + return lines + + indent = _INDENT * 2 + + if arg.transform_case == 2: + # Optional, no transform: pointer assignment. + lines.append('{}if ({}) then'.format(indent, arg.active_local or '.true.')) + lines.append('{} {}%ptr => {}'.format(indent, arg.ptr_name, arg.call_expr)) + lines.append('{}else'.format(indent)) + lines.append('{} nullify({}%ptr)'.format(indent, arg.ptr_name)) + lines.append('{}end if'.format(indent)) + + elif arg.transform_case == 3: + # Transform, not optional. + if arg.unit_forward: + comment = _transform_comment(arg) + sep = ' ' if comment else '' + lines.append('{}{} = {}{}{}'.format( + indent, arg.temp_name, arg.unit_forward, sep, comment + )) + + elif arg.transform_case == 4: + # Transform + optional pointer. + lines.append('{}if ({}) then'.format(indent, arg.active_local or '.true.')) + if arg.unit_forward: + lines.append('{} {} = {}'.format(indent, arg.temp_name, arg.unit_forward)) + lines.append('{} {}%ptr => {}'.format(indent, arg.ptr_name, arg.temp_name)) + lines.append('{}else'.format(indent)) + lines.append('{} nullify({}%ptr)'.format(indent, arg.ptr_name)) + lines.append('{}end if'.format(indent)) + + return lines + + +def _post_call_lines(arg: ResolvedArg) -> List[str]: + """Generate post-call Fortran lines for one argument.""" + lines = [] + if arg.transform_case == 1: + return lines + + indent = _INDENT * 2 + + if arg.transform_case == 2: + lines.append('{}nullify({}%ptr)'.format(indent, arg.ptr_name)) + + elif arg.transform_case == 3: + if arg.unit_backward: + comment = _transform_comment(arg, reverse=True) + sep = ' ' if comment else '' + lines.append('{}{} = {}{}{}'.format( + indent, arg.call_expr, arg.unit_backward, sep, comment + )) + + elif arg.transform_case == 4: + lines.append('{}if ({}) then'.format(indent, arg.active_local or '.true.')) + lines.append('{} nullify({}%ptr)'.format(indent, arg.ptr_name)) + if arg.unit_backward: + lines.append('{} {} = {}'.format(indent, arg.call_expr, arg.unit_backward)) + lines.append('{}end if'.format(indent)) + + return lines + + +def _call_arg_expr(arg: ResolvedArg) -> str: + """Return the Fortran expression to pass for this argument at the call site.""" + if arg.transform_case == 1: + return arg.call_expr + elif arg.transform_case == 2: + return '{}%ptr'.format(arg.ptr_name) + elif arg.transform_case == 3: + return arg.temp_name + else: # 4 + return '{}%ptr'.format(arg.ptr_name) + + +######################################################################## +# Scheme-call code generation helper +######################################################################## + +def _max_subcycle_depth(items) -> int: + """Return the maximum subcycle nesting depth in *items*. + + A flat list of scheme calls has depth 0; a single ```` + wrapping schemes has depth 1; a subcycle wrapping a subcycle has + depth 2; and so on. Used to pre-declare one integer loop counter + per nesting level (``ccpp_loop_counter``, ``ccpp_loop_counter_2``, + ``ccpp_loop_counter_3``, ...). + """ + depth = 0 + for item in items: + if isinstance(item, ResolvedSubcycle): + depth = max(depth, 1 + _max_subcycle_depth(item.calls)) + return depth + + +def _loop_counter_name(depth: int) -> str: + """Return the loop-counter Fortran identifier for *depth* (1-based). + + Depth 1 (outermost / single level) returns ``'ccpp_loop_counter'`` + so existing single-subcycle tests and host expectations are + unchanged. Deeper levels get ``ccpp_loop_counter_``. + """ + if depth <= 1: + return 'ccpp_loop_counter' + return 'ccpp_loop_counter_{}'.format(depth) + + +def _emit_phase_items( + items, indent: str, lines: List[str], depth: int, +) -> None: + """Recursively emit Fortran for a list of :data:`PhaseItem` objects. + + A :class:`ResolvedCall` becomes a single scheme call (with pre/post + transforms). A :class:`ResolvedSubcycle` becomes a ``do`` loop + that wraps recursively-emitted children, with one fresh integer + loop variable per nesting level. + """ + for item in items: + if isinstance(item, ResolvedCall): + _emit_one_call(item, indent, lines) + elif isinstance(item, ResolvedSubcycle): + counter = _loop_counter_name(depth) + lines.append( + '{}do {} = 1, {}'.format(indent, counter, item.loop) + ) + _emit_phase_items( + item.calls, indent + _INDENT, lines, depth=depth + 1, + ) + lines.append('{}end do'.format(indent)) + lines.append('') + + +def _emit_one_call( + rc: ResolvedCall, + indent: str, + lines: List[str], +) -> None: + """Append Fortran lines for a single scheme call (with transforms + errcheck).""" + # Pre-call transformations. + for arg in rc.args: + lines.extend(_pre_call_lines(arg)) + + call_args_exprs = [ + '{}={}'.format(a.scheme_local_name, _call_arg_expr(a)) + for a in rc.args + ] + call_name = '{}_{}'.format(rc.scheme_name, rc.phase) + + if call_args_exprs: + lines.append('{}call {}( &'.format(indent, call_name)) + for i, expr in enumerate(call_args_exprs): + sep = ', &' if i < len(call_args_exprs) - 1 else ')' + lines.append('{} {}{}'.format(indent, expr, sep)) + else: + lines.append('{}call {}()'.format(indent, call_name)) + + errflg_arg = next( + (a for a in rc.args if a.standard_name == 'ccpp_error_code'), None + ) + if errflg_arg is not None: + lines.append('{}if ({} /= 0) return'.format(indent, _call_arg_expr(errflg_arg))) + + for arg in rc.args: + lines.extend(_post_call_lines(arg)) + lines.append('') + + +######################################################################## +# State machine guards +######################################################################## + +def _state_entry_guard( + phase: str, + inst_idx: str, + errflg_local: Optional[str], + errmsg_local: Optional[str], + sub_label: str, + indent: str, +) -> List[str]: + """Return Fortran lines that validate the group state on phase entry. + + Per the design table: + + ===================== ============================================ + Phase Required state + ===================== ============================================ + ``init`` ``UNINITIALIZED`` (idempotent skip if ``INITIALIZED``) + ``timestep_init`` ``== INITIALIZED`` + ``run`` ``== IN_TIMESTEP`` + ``timestep_final`` ``== IN_TIMESTEP`` + ``final`` ``>= INITIALIZED`` + ===================== ============================================ + + Invalid state sets ``errflg = 1``, populates ``errmsg``, and returns. + + If the host did not declare ``ccpp_error_code`` / ``ccpp_error_message`` + (extremely unusual), the guard is omitted — there is no way to report the + error and the routine cannot proceed without somewhere to write it. + """ + if not errflg_local or not errmsg_local: + return [] + + state_var = 'ccpp_group_state({})'.format(inst_idx) + msg = "ccpp_{}: invalid group state".format(sub_label) + + def _err_block(condition: str) -> List[str]: + return [ + '{}if ({}) then'.format(indent, condition), + "{} {} = '{}'".format(indent, errmsg_local, msg), + '{} {} = 1'.format(indent, errflg_local), + '{} return'.format(indent), + '{}end if'.format(indent), + '', + ] + + if phase == 'init': + # Idempotent skip when already INITIALIZED; error if past INITIALIZED + # (e.g. IN_TIMESTEP) — init must come from UNINITIALIZED. + lines: List[str] = [ + '{}if ({} == CCPP_GROUP_INITIALIZED) return'.format(indent, state_var), + '', + ] + lines.extend(_err_block( + '{} /= CCPP_GROUP_UNINITIALIZED'.format(state_var) + )) + return lines + if phase == 'timestep_init': + return _err_block('{} /= CCPP_GROUP_INITIALIZED'.format(state_var)) + if phase == 'run': + return _err_block('{} /= CCPP_GROUP_IN_TIMESTEP'.format(state_var)) + if phase == 'timestep_final': + return _err_block('{} /= CCPP_GROUP_IN_TIMESTEP'.format(state_var)) + if phase == 'final': + return _err_block('{} < CCPP_GROUP_INITIALIZED'.format(state_var)) + return [] + + +######################################################################## +# Subroutine generator +######################################################################## + +def _generate_phase_subroutine( + suite_name: str, + group_name: str, + phase: str, + phase_items, + ctrl_entries, + host_dict, +) -> List[str]: + """Generate one phase subroutine for a group cap. + + For the ``init`` phase the body is wrapped in a state guard + (``ccpp_group_state(1) < CCPP_GROUP_INITIALIZED``) and the state is set + to ``CCPP_GROUP_INITIALIZED`` at the end. + + For the ``final`` phase the state is reset to ``CCPP_GROUP_UNINITIALIZED`` + at the end. + + ``ResolvedSubcycle`` items in *phase_items* generate ``do`` loops (run + phase only). + + Returns a list of Fortran source lines (no trailing newlines). + """ + sub_name = 'ccpp_{}_{}_{}'.format(suite_name, group_name, phase) + lines: List[str] = [] + + # ---- subroutine declaration ------------------------------------------ + ctrl_local_names = [e.local_name for e in ctrl_entries] + + if ctrl_local_names: + lines.append('{}subroutine {}( &'.format(_INDENT, sub_name)) + for i, lname in enumerate(ctrl_local_names): + sep = ', &' if i < len(ctrl_local_names) - 1 else ')' + lines.append('{} {}{}'.format(_INDENT, lname, sep)) + else: + lines.append('{}subroutine {}()'.format(_INDENT, sub_name)) + + # ---- dummy argument declarations ------------------------------------ + if ctrl_local_names: + lines.append('') + for entry in ctrl_entries: + # Character control dummies always use len=* so the host's specific + # length doesn't propagate into the generated signature. + kind = 'len=*' if entry.type.strip().lower() == 'character' else entry.kind + t = _fortran_type_str(entry.type, kind) + intent = _intent_clause(_ctrl_intent_for(entry.standard_name)) + dim = _dim_decl(entry.dimensions) + lines.append( + '{}{}{}{} :: {}'.format(_INDENT * 2, t, intent, dim, entry.local_name) + ) + + # ---- local variable declarations (transformation temps, subcycle counter) + local_decls: List[str] = [] + # Each subcycle nesting level needs its own integer loop variable. + # The outermost (and only one, in the single-level case) is named + # ``ccpp_loop_counter`` to preserve the existing single-level + # convention; deeper levels are ``ccpp_loop_counter_2``, + # ``ccpp_loop_counter_3``, ... so nested loops have distinct vars. + max_depth = _max_subcycle_depth(phase_items) + for d in range(1, max_depth + 1): + name = 'ccpp_loop_counter' if d == 1 else 'ccpp_loop_counter_{}'.format(d) + local_decls.append('{}integer :: {}'.format(_INDENT * 2, name)) + + seen_temp_names: Set[str] = set() + seen_ptr_names: Set[str] = set() + for rc in iter_phase_calls(phase_items): + for arg in rc.args: + if arg.temp_name and arg.temp_name not in seen_temp_names: + seen_temp_names.add(arg.temp_name) + t = _fortran_type_str( + arg.host_entry.type if arg.host_entry else arg.suite_var.type_, + arg.kind_scheme or (arg.host_entry.kind if arg.host_entry else ''), + ) + # Use scheme dimensions (local names) for the temp declaration + # so the temp is sized for the chunk the scheme actually receives + # (Gap 2: avoids emitting standard names in the declaration). + dim = _dim_decl_local(arg.scheme_dimensions, host_dict) + # Transform-case 4 emits ``%ptr => ``, so the temp + # must have the TARGET attribute or pointer assignment is illegal. + target_attr = ', target' if arg.ptr_name else '' + local_decls.append( + '{}{}{}{} :: {}'.format( + _INDENT * 2, t, dim, target_attr, arg.temp_name + ) + ) + if arg.ptr_name and arg.ptr_name not in seen_ptr_names: + seen_ptr_names.add(arg.ptr_name) + type_, kind, rank = _ptr_type_for_arg(arg) + ptr_tname = _ptr_type_name(type_, kind, rank) + local_decls.append( + '{}type({}) :: {}'.format(_INDENT * 2, ptr_tname, arg.ptr_name) + ) + + if local_decls: + lines.append('') + lines.extend(local_decls) + + lines.append('') + + call_indent = _INDENT * 2 + + inst_idx = _instance_idx(host_dict) + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') + sub_label = '{}_{}_{}'.format(suite_name, group_name, phase) + + # ---- initialize error reporting vars ------------------------------- + if errflg_local and errmsg_local: + lines.append("{}{} = ''".format(call_indent, errmsg_local)) + lines.append('{}{} = 0'.format(call_indent, errflg_local)) + lines.append('') + + # ---- phase entry state guards -------------------------------------- + lines.extend( + _state_entry_guard( + phase, inst_idx, errflg_local, errmsg_local, sub_label, call_indent + ) + ) + + # ---- scheme calls --------------------------------------------------- + _emit_phase_items(phase_items, call_indent, lines, depth=1) + + # ---- post-call state transitions ------------------------------------ + if phase == 'init': + lines.append( + '{}ccpp_group_state({}) = CCPP_GROUP_INITIALIZED'.format( + call_indent, inst_idx + ) + ) + lines.append('') + elif phase == 'timestep_init': + lines.append( + '{}ccpp_group_state({}) = CCPP_GROUP_IN_TIMESTEP'.format( + call_indent, inst_idx + ) + ) + lines.append('') + elif phase == 'timestep_final': + lines.append( + '{}ccpp_group_state({}) = CCPP_GROUP_INITIALIZED'.format( + call_indent, inst_idx + ) + ) + lines.append('') + elif phase == 'final': + lines.append( + '{}ccpp_group_state({}) = CCPP_GROUP_UNINITIALIZED'.format( + call_indent, inst_idx + ) + ) + lines.append('') + + lines.append('{}end subroutine {}'.format(_INDENT, sub_name)) + return lines + + +######################################################################## +# Group cap module generator +######################################################################## + +def _instance_idx(host_dict) -> str: + """Return the Fortran index expression for the current model instance. + + Returns the local name of the ``instance_number`` control variable when the + host declares it; otherwise returns the literal ``'1'`` for single-instance + models. + + >>> class _FakeEntry: + ... local_name = 'inst_num' + >>> class _FakeDict(dict): + ... pass + >>> d = _FakeDict({'instance_number': _FakeEntry()}) + >>> _instance_idx(d) + 'inst_num' + >>> _instance_idx({}) + '1' + """ + entry = host_dict.get('instance_number') if host_dict else None + return entry.local_name if entry is not None else '1' + + +def _instance_local(host_dict) -> Optional[str]: + """Return the host's local name for ``instance_number``, or ``None``. + + Companion to :func:`_instance_idx`. Callers use the ``None`` return + to decide whether to inject ``instance_number`` into a subroutine's + signature (i.e. whether the host opted into the multi-instance API). + + >>> class _FakeEntry: + ... local_name = 'inst_num' + >>> _instance_local({'instance_number': _FakeEntry()}) + 'inst_num' + >>> _instance_local({}) is None + True + """ + entry = host_dict.get('instance_number') if host_dict else None + return entry.local_name if entry is not None else None + + +def _generate_state_alloc(suite_name: str, group_name: str) -> List[str]: + """Generate the ``ccpp___state_alloc`` subroutine. + + The subroutine always accepts ``number_of_instances`` as an explicit + ``intent(in)`` integer argument so the caller (the suite cap's init + routine) can supply the count at runtime without the group cap needing to + USE any host module. + + Idempotent: a second call after the array is already allocated is a + no-op. The suite cap calls this once per ``_init`` invocation, + so when multiple instances initialize, only the first allocates and + initialises the state array; subsequent calls return immediately to + avoid clobbering peer-instance state slots. Matches the + ``_suite_state_alloc`` pattern. + """ + sub_name = 'ccpp_{}_{}_{}'.format(suite_name, group_name, 'state_alloc') + i1 = _INDENT + i2 = _INDENT * 2 + lines = [ + '', + '{}subroutine {}(number_of_instances, errmsg, errflg)'.format(i1, sub_name), + '', + '{}integer, intent(in) :: number_of_instances'.format(i2), + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + '{}if (allocated(ccpp_group_state)) return'.format(i2), + '{}allocate(ccpp_group_state(number_of_instances))'.format(i2), + '{}ccpp_group_state(:) = CCPP_GROUP_UNINITIALIZED'.format(i2), + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + return lines + + +def _generate_state_dealloc(suite_name: str, group_name: str) -> List[str]: + """Generate the ``ccpp___state_dealloc`` subroutine.""" + sub_name = 'ccpp_{}_{}_{}'.format(suite_name, group_name, 'state_dealloc') + i1 = _INDENT + i2 = _INDENT * 2 + return [ + '', + '{}subroutine {}(errmsg, errflg)'.format(i1, sub_name), + '', + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + '{}if (allocated(ccpp_group_state)) deallocate(ccpp_group_state)'.format(i2), + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + + +def _generate_group_cap( + suite_name: str, + group_name: str, + rg: ResolvedGroup, + host_dict, +) -> List[str]: + """Generate the full group cap module source lines. + + Parameters + ---------- + suite_name : str + group_name : str + rg : ResolvedGroup + host_dict : dict + Flat host+control dictionary. + + Returns + ------- + list of str (without trailing newlines) + """ + mod_name = 'ccpp_{}_{}_{}'.format(suite_name, group_name, 'cap') + alloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, group_name, 'state_alloc') + dealloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, group_name, 'state_dealloc') + lines: List[str] = [] + + # ---- module header -------------------------------------------------- + lines.append( + '! ccpp_{}_{}_cap.F90 -- generated by ccpp_capgen_ng, do not edit'.format( + suite_name, group_name + ) + ) + lines.append('module {}'.format(mod_name)) + lines.append('') + + # ---- USE statements ------------------------------------------------- + uses = _collect_group_uses(rg, host_dict) + + # Add USE for types module when optional pointer args are present. + ptr_type_names: Set[str] = set() + for items in rg.phase_calls.values(): + for rc in iter_phase_calls(items): + for arg in rc.args: + if arg.ptr_name: + type_, kind, rank = _ptr_type_for_arg(arg) + ptr_type_names.add(_ptr_type_name(type_, kind, rank)) + if ptr_type_names: + types_mod = 'ccpp_{}_types'.format(suite_name) + uses[types_mod] = ptr_type_names + + # USE ccpp_kinds for any kind parameter referenced in transformation + # temporaries declared in this group (e.g. ``real(kind=kind_phys)``). + kind_names = _collect_kinds_used(rg) + if kind_names: + uses['ccpp_kinds'] = set(kind_names) + + use_lines = _use_statements(uses) + use_lines.extend(_scheme_use_statements(rg)) + lines.extend(use_lines) + if use_lines: + lines.append('') + + lines.append('{}implicit none'.format(_INDENT)) + lines.append('{}private'.format(_INDENT)) + + # Public physics subroutines, in canonical phase order. + # Always emit all phases so the suite cap can rely on the state machine + # transitioning through every phase, even when a group has no scheme + # routine for a particular phase. + for phase in _GROUP_PHASE_ORDER: + sub_name = 'ccpp_{}_{}_{}'.format(suite_name, group_name, phase) + lines.append('{}public :: {}'.format(_INDENT, sub_name)) + + # Public state management subroutines. + lines.append('{}public :: {}'.format(_INDENT, alloc_sub)) + lines.append('{}public :: {}'.format(_INDENT, dealloc_sub)) + + # ---- state machine module-level declarations ------------------------- + lines.append('') + lines.append('{}integer, private, parameter :: CCPP_GROUP_UNINITIALIZED = 0'.format(_INDENT)) + lines.append('{}integer, private, parameter :: CCPP_GROUP_INITIALIZED = 1'.format(_INDENT)) + lines.append('{}integer, private, parameter :: CCPP_GROUP_IN_TIMESTEP = 2'.format(_INDENT)) + lines.append('{}integer, private, allocatable :: ccpp_group_state(:)'.format(_INDENT)) + + lines.append('') + lines.append('contains') + lines.append('') + + # ---- subroutines per phase ------------------------------------------ + # All phases share the same uniform control arg signature (excluding + # suite_name and group_name which are consumed at higher dispatch levels). + ctrl_sig_entries = _ctrl_entries_for_signature( + host_dict, exclude={'suite_name', 'group_name'} + ) + for phase in _GROUP_PHASE_ORDER: + phase_items = rg.phase_calls.get(phase, []) + sub_lines = _generate_phase_subroutine( + suite_name, group_name, phase, phase_items, ctrl_sig_entries, host_dict + ) + lines.extend(sub_lines) + lines.append('') + + # ---- state management subroutines ----------------------------------- + lines.extend(_generate_state_alloc(suite_name, group_name)) + lines.append('') + lines.extend(_generate_state_dealloc(suite_name, group_name)) + lines.append('') + + lines.append('end module {}'.format(mod_name)) + return lines + + +def _ctrl_args_for_phase(rg: ResolvedGroup, phase: str) -> List[ResolvedArg]: + """Return control args used in a specific phase, deduplicated.""" + seen: Dict[str, ResolvedArg] = {} + for rc in iter_phase_calls(rg.phase_calls.get(phase, [])): + for arg in rc.args: + if arg.source == 'control' and arg.standard_name not in seen: + seen[arg.standard_name] = arg + return list(seen.values()) + + +######################################################################## +# Public API +######################################################################## + +def write_group_cap( + suite_name: str, + group_name: str, + rg: ResolvedGroup, + host_dict, + output_root: str, +) -> str: + """Write the group cap Fortran module to *output_root*. + + Parameters + ---------- + suite_name : str + group_name : str + rg : ResolvedGroup + Resolved call information for this group. + host_dict : dict + Flat host+control variable dictionary. + output_root : str + Output directory (created if absent). + + Returns + ------- + str + Absolute path of the written file. + """ + os.makedirs(output_root, exist_ok=True) + filename = 'ccpp_{}_{}_cap.F90'.format(suite_name, group_name) + out_path = os.path.join(output_root, filename) + + lines = _generate_group_cap(suite_name, group_name, rg, host_dict) + with open(out_path, 'w', encoding='utf-8') as fh: + fh.write('\n'.join(lines) + '\n') + return out_path diff --git a/capgen-ng/generator/host_constituents.py b/capgen-ng/generator/host_constituents.py new file mode 100644 index 00000000..575bd2e6 --- /dev/null +++ b/capgen-ng/generator/host_constituents.py @@ -0,0 +1,642 @@ +#!/usr/bin/env python3 + +"""Generate ``ccpp_host_constituents.F90`` — the host-wide constituent module. + +In capgen-ng's per-instance design, the constituent state is sized to +``number_of_instances`` (declared by the host's ``type=host`` table). +This module owns: + +* ``ccpp_model_constituents_obj(:)`` — one DDT instance per host + instance, lazily allocated on first ``ccpp_register_constituents``. +* Per-suite buffers ``_dynamic_constituents(:)`` — populated by + ``_register`` from register-phase scheme call results. + Single-dim because registration is identical across instances; the + first instance fills the buffer and subsequent instances skip. +* ``index_of_`` integers (module-level scalars; identical across + instances) — bound by ``ccpp_initialize_constituents`` via + ``%const_index`` queries. +* ``ccpp_model_const_stdnames`` parameter array listing the std names + known to capgen at code-generation time. + +Host-facing API mirrors original capgen but every routine that touches +a specific instance's state takes ``instance_number`` (when the host +declares it). Scheme call sites generated by the resolver access +``ccpp_model_constituents_obj()%vars_layer(…)`` directly, so the +old module-level pointer bindings (``ccpp_constituents`` etc.) have +been removed. +""" + +import os +from typing import List, Optional, Set, Tuple + +from generator.suite_resolver import SuiteResolution + +_INDENT = ' ' + +_HOST_CONST_MOD = 'ccpp_host_constituents' +_CONST_PROP_MOD = 'ccpp_constituent_prop_mod' +_CONST_DDT = 'ccpp_model_constituents_t' +_CONST_PROP_TYPE = 'ccpp_constituent_properties_t' +_CONST_PROP_PTR_TYPE = 'ccpp_constituent_prop_ptr_t' + +# Public name of the host-wide constituent object (now a per-instance +# allocatable array). +_CONST_OBJ = 'ccpp_model_constituents_obj' + + +######################################################################## +# Aggregation helpers +######################################################################## + +def _any_constituent_state(suite_results: List[SuiteResolution]) -> bool: + """Return True iff any suite either uses or registers constituents.""" + return any( + sr.uses_constituents or sr.constituent_register_calls + for sr in suite_results + ) + + +def _all_index_names(suite_results: List[SuiteResolution]) -> List[str]: + """Return sorted unique base std-names that need an ``index_of_``.""" + names: Set[str] = set() + for sr in suite_results: + names.update(sr.constituent_index_names) + return sorted(names) + + +def _suites_with_register_consts( + suite_results: List[SuiteResolution], +) -> List[str]: + return [sr.suite_name for sr in suite_results + if sr.constituent_register_calls] + + +def _dyn_const_array_name(suite_name: str) -> str: + return '{}_dynamic_constituents'.format(suite_name) + + +def _host_lookup(host_dict, std_name: str) -> Tuple[Optional[str], Optional[str]]: + """Return (local_name, module_name) for *std_name* in *host_dict*. + + Returns ``(None, None)`` when the entry is absent or is a control var. + """ + if not host_dict: + return None, None + entry = host_dict.get(std_name) + if entry is None: + return None, None + return entry.local_name, entry.module_name + + +######################################################################## +# Per-routine emitters +######################################################################## + +def _instance_signature( + base_args: List[str], + inst_local: Optional[str], +) -> List[str]: + """Return signature args with *inst_local* inserted before err args.""" + sig = list(base_args) + if inst_local: + sig.append(inst_local) + sig += ['errflg', 'errmsg'] + return sig + + +def _register_constituents_lines( + suite_results: List[SuiteResolution], + host_dict, +) -> List[str]: + """Emit ``ccpp_register_constituents``.""" + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + register_suites = _suites_with_register_consts(suite_results) + inst_local, _ = _host_lookup(host_dict, 'instance_number') + ninst_local, ninst_mod = _host_lookup(host_dict, 'number_of_instances') + inst_idx = inst_local if inst_local else '1' + ninst_arg = ninst_local if ninst_local else '1' + + sig = _instance_signature(['host_constituents'], inst_local) + lines: List[str] = [''] + lines.append('{}subroutine ccpp_register_constituents({})'.format( + i1, ', '.join(sig), + )) + if ninst_local and ninst_mod: + lines.append('{}use {}, only: {}'.format(i2, ninst_mod, ninst_local)) + lines.append('') + lines.append( + '{}type({}), target, intent(in) :: host_constituents(:)'.format( + i2, _CONST_PROP_TYPE, + ) + ) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines.append('{}integer, intent(out) :: errflg'.format(i2)) + lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) + lines.append('') + lines.append('{}integer :: num_consts, index'.format(i2)) + lines.append('{}type({}), pointer :: const_prop => null()'.format( + i2, _CONST_PROP_TYPE, + )) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + # Allocate the object array on first call (idempotent across instances). + lines.append('{}if (.not. allocated({})) then'.format(i2, _CONST_OBJ)) + lines.append('{}allocate({}({}))'.format(i3, _CONST_OBJ, ninst_arg)) + lines.append('{}end if'.format(i2)) + lines.append('') + # Count. + lines.append('{}num_consts = size(host_constituents, 1)'.format(i2)) + for sname in register_suites: + buf = _dyn_const_array_name(sname) + lines.append('{}if (allocated({})) then'.format(i2, buf)) + lines.append('{}num_consts = num_consts + size({}, 1)'.format(i3, buf)) + lines.append('{}end if'.format(i2)) + lines.append('') + lines.append('{}call {}({})%initialize_table(num_consts)'.format( + i2, _CONST_OBJ, inst_idx, + )) + lines.append('') + # Host constituents first. + lines.append('{}do index = 1, size(host_constituents, 1)'.format(i2)) + lines.append('{}const_prop => host_constituents(index)'.format(i3)) + lines.append( + '{}call {}({})%new_field(const_prop, errcode=errflg, errmsg=errmsg)'.format( + i3, _CONST_OBJ, inst_idx, + ) + ) + lines.append('{}nullify(const_prop)'.format(i3)) + lines.append('{}if (errflg /= 0) return'.format(i3)) + lines.append('{}end do'.format(i2)) + for sname in register_suites: + buf = _dyn_const_array_name(sname) + lines.append('') + lines.append("{}! Merge {} dynamic constituents".format(i2, sname)) + lines.append('{}if (allocated({})) then'.format(i2, buf)) + lines.append('{}do index = 1, size({}, 1)'.format(i3, buf)) + lines.append('{}const_prop => {}(index)'.format(i3 + _INDENT, buf)) + lines.append( + '{}call {}({})%new_field(const_prop, errcode=errflg, errmsg=errmsg)'.format( + i3 + _INDENT, _CONST_OBJ, inst_idx, + ) + ) + lines.append('{}nullify(const_prop)'.format(i3 + _INDENT)) + lines.append('{}if (errflg /= 0) return'.format(i3 + _INDENT)) + lines.append('{}end do'.format(i3)) + lines.append('{}end if'.format(i2)) + lines.append('') + lines.append( + '{}call {}({})%lock_table(errcode=errflg, errmsg=errmsg)'.format( + i2, _CONST_OBJ, inst_idx, + ) + ) + lines.append('') + lines.append('{}end subroutine ccpp_register_constituents'.format(i1)) + return lines + + +def _initialize_constituents_lines( + suite_results: List[SuiteResolution], + host_dict, +) -> List[str]: + """Emit ``ccpp_initialize_constituents``.""" + i1 = _INDENT + i2 = _INDENT * 2 + inst_local, _ = _host_lookup(host_dict, 'instance_number') + inst_idx = inst_local if inst_local else '1' + index_names = _all_index_names(suite_results) + + sig = _instance_signature(['ncols', 'num_layers'], inst_local) + lines: List[str] = [''] + lines.append('{}subroutine ccpp_initialize_constituents({})'.format( + i1, ', '.join(sig), + )) + lines.append( + '{}use ccpp_scheme_utils, only: ccpp_initialize_constituent_ptr'.format(i2) + ) + lines.append('') + lines.append('{}integer, intent(in) :: ncols, num_layers'.format(i2)) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines.append('{}integer, intent(out) :: errflg'.format(i2)) + lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) + lines.append('') + lines.append('{}type({}), pointer :: const_obj_ptr => null()'.format( + i2, _CONST_DDT, + )) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + lines.append( + '{}call {}({})%lock_data(ncols, num_layers, errcode=errflg, errmsg=errmsg)'.format( + i2, _CONST_OBJ, inst_idx, + ) + ) + lines.append('{}if (errflg /= 0) return'.format(i2)) + # Cache the singleton pointer in ccpp_scheme_utils (cam-sima compat). + # Only the FIRST call across instances actually sets it (the routine + # is guarded internally); other instances see the first instance's + # object when they call ccpp_constituent_index — a known limitation + # of the framework's scheme_utils for multi-instance hosts. + lines.append('{}const_obj_ptr => {}({})'.format( + i2, _CONST_OBJ, inst_idx, + )) + lines.append('{}call ccpp_initialize_constituent_ptr(const_obj_ptr)'.format(i2)) + lines.append('{}nullify(const_obj_ptr)'.format(i2)) + lines.append('') + for std_name in index_names: + lines.append( + "{}call {}({})%const_index(index_of_{}, '{}', " + "errcode=errflg, errmsg=errmsg)".format( + i2, _CONST_OBJ, inst_idx, std_name, std_name, + ) + ) + lines.append('{}if (errflg /= 0) return'.format(i2)) + # %const_index doesn't error on a miss — it sets the integer to + # int_unassigned and leaves errcode unchanged. Surface that case + # explicitly so the host sees the bad registration at init time + # instead of crashing on a -huge(1) subscript later. + lines.append( + '{}if (index_of_{} == int_unassigned) then'.format( + i2, std_name, + ) + ) + lines.append('{}errflg = 1'.format(i2 + _INDENT)) + lines.append( + "{}errmsg = 'ccpp_initialize_constituents: constituent " + "''{}'' is referenced by a scheme but is not in the " + "registered constituent table; check that some scheme'" + "'s _register routine or the host_constituents argument " + "to ccpp_register_constituents includes it'".format( + i2 + _INDENT, std_name, + ) + ) + lines.append('{}return'.format(i2 + _INDENT)) + lines.append('{}end if'.format(i2)) + lines.append('') + lines.append('{}end subroutine ccpp_initialize_constituents'.format(i1)) + return lines + + +def _is_scheme_constituent_lines(suite_results: List[SuiteResolution]) -> List[str]: + """Emit ``ccpp_is_scheme_constituent`` (no instance_number — module-level).""" + i1 = _INDENT + i2 = _INDENT * 2 + index_names = _all_index_names(suite_results) + lines: List[str] = [''] + lines.append( + '{}subroutine ccpp_is_scheme_constituent(var_name, ' + 'constituent_exists, errflg, errmsg)'.format(i1) + ) + lines.append('') + lines.append('{}character(len=*), intent(in) :: var_name'.format(i2)) + lines.append('{}logical, intent(out) :: constituent_exists'.format(i2)) + lines.append('{}integer, intent(out) :: errflg'.format(i2)) + lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + if index_names: + lines.append( + '{}constituent_exists = any(ccpp_model_const_stdnames == var_name)'.format(i2) + ) + else: + lines.append('{}constituent_exists = .false.'.format(i2)) + lines.append('') + lines.append('{}end subroutine ccpp_is_scheme_constituent'.format(i1)) + return lines + + +def _wrap_method_sub( + sub_name: str, + method: str, + extra_args: List[Tuple[str, str, str]], + inst_local: Optional[str], + errcode_call: bool = True, +) -> List[str]: + """Thin wrapper subroutine calling ``obj(inst_num)%method(...)``.""" + i1 = _INDENT + i2 = _INDENT * 2 + inst_idx = inst_local if inst_local else '1' + sig = [n for n, _, _ in extra_args] + if inst_local: + sig.append(inst_local) + sig += ['errflg', 'errmsg'] + lines: List[str] = [''] + lines.append('{}subroutine {}({})'.format(i1, sub_name, ', '.join(sig))) + lines.append('') + for _, decl, _ in extra_args: + lines.append('{}{}'.format(i2, decl)) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines.append('{}integer, intent(out) :: errflg'.format(i2)) + lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + call_args = [call for _, _, call in extra_args] + if errcode_call: + call_args += ['errcode=errflg', 'errmsg=errmsg'] + lines.append('{}call {}({})%{}({})'.format( + i2, _CONST_OBJ, inst_idx, method, ', '.join(call_args), + )) + lines.append('') + lines.append('{}end subroutine {}'.format(i1, sub_name)) + return lines + + +def _wrap_method_subs(host_dict) -> List[str]: + inst_local, _ = _host_lookup(host_dict, 'instance_number') + lines: List[str] = [] + lines.extend(_wrap_method_sub( + 'ccpp_number_constituents', 'num_constituents', + [ + ('num_flds', 'integer, intent(out) :: num_flds', 'num_flds'), + ('advected', 'logical, optional, intent(in) :: advected', + 'advected=advected'), + ], + inst_local, + )) + lines.extend(_wrap_method_sub( + 'ccpp_gather_constituents', 'copy_in', + [('const_array', + 'real(kind=kind_phys), intent(out) :: const_array(:,:,:)', + 'const_array')], + inst_local, + )) + lines.extend(_wrap_method_sub( + 'ccpp_update_constituents', 'copy_out', + [('const_array', + 'real(kind=kind_phys), intent(in) :: const_array(:,:,:)', + 'const_array')], + inst_local, + )) + lines.extend(_wrap_method_sub( + 'ccpp_const_get_index', 'const_index', + [ + ('stdname', 'character(len=*), intent(in) :: stdname', + 'standard_name=stdname'), + ('const_index', 'integer, intent(out) :: const_index', + 'index=const_index'), + ], + inst_local, + )) + return lines + + +def _accessor_functions(host_dict) -> List[str]: + """Pointer-returning accessor functions, indexed by instance_number.""" + i1 = _INDENT + i2 = _INDENT * 2 + inst_local, _ = _host_lookup(host_dict, 'instance_number') + inst_idx = inst_local if inst_local else '1' + args = '({})'.format(inst_local) if inst_local else '()' + decl = ('integer, intent(in) :: {}'.format(inst_local) + if inst_local else '') + + def _emit(fname, method, ret_decl): + out = [ + '', + '{}function {}{} result(const_ptr)'.format(i1, fname, args), + '{}{}'.format(i2, ret_decl), + ] + if decl: + out.append('{}{}'.format(i2, decl)) + out += [ + '{}const_ptr => {}({})%{}()'.format( + i2, _CONST_OBJ, inst_idx, method, + ), + '{}end function {}'.format(i1, fname), + ] + return out + + lines: List[str] = [] + lines += _emit( + 'ccpp_constituents_array', 'field_data_ptr', + 'real(kind=kind_phys), pointer :: const_ptr(:,:,:)', + ) + lines += _emit( + 'ccpp_advected_constituents_array', 'advected_constituents_ptr', + 'real(kind=kind_phys), pointer :: const_ptr(:,:,:)', + ) + # Properties function returns a different pointer type. + fname = 'ccpp_model_const_properties' + ret_decl = 'type({}), pointer :: const_ptr(:)'.format(_CONST_PROP_PTR_TYPE) + args_props = '({})'.format(inst_local) if inst_local else '()' + lines += [ + '', + '{}function {}{} result(const_ptr)'.format(i1, fname, args_props), + '{}{}'.format(i2, ret_decl), + ] + if decl: + lines.append('{}{}'.format(i2, decl)) + lines += [ + '{}const_ptr => {}({})%constituent_props_ptr()'.format( + i2, _CONST_OBJ, inst_idx, + ), + '{}end function {}'.format(i1, fname), + ] + return lines + + +def _deallocate_lines( + suite_results: List[SuiteResolution], + host_dict, +) -> List[str]: + """Emit ``ccpp_deallocate_dynamic_constituents`` — per-instance with + last-to-leave teardown. + + Each call resets the constituent object for the given instance. When + every instance's object has been reset (none still has its property + table locked), the per-suite dynamic-constituent buffers and the + object array itself are deallocated, and the cached + ``index_of_`` integers are reset. + + Mirrors the per-instance + last-to-leave pattern used by + ``_final`` in the suite cap. + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + register_suites = _suites_with_register_consts(suite_results) + index_names = _all_index_names(suite_results) + inst_local, _ = _host_lookup(host_dict, 'instance_number') + inst_idx = inst_local if inst_local else '1' + + sig_args = [inst_local] if inst_local else [] + + lines: List[str] = [''] + lines.append('{}subroutine ccpp_deallocate_dynamic_constituents({})'.format( + i1, ', '.join(sig_args), + )) + lines.append('') + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines.append('') + lines.append('{}integer :: i'.format(i2)) + lines.append('{}logical :: all_done'.format(i2)) + lines.append('') + lines.append('{}if (.not. allocated({})) return'.format(i2, _CONST_OBJ)) + lines.append('') + # Per-instance reset. + lines.append('{}call {}({})%reset()'.format(i2, _CONST_OBJ, inst_idx)) + lines.append('') + # Last-to-leave: when no instance still has its constituent table + # locked, tear down everything (buffers + object array + indices). + lines.append('{}all_done = .true.'.format(i2)) + lines.append('{}do i = 1, size({}, 1)'.format(i2, _CONST_OBJ)) + lines.append('{}if ({}(i)%const_props_locked()) then'.format( + i3, _CONST_OBJ, + )) + lines.append('{}all_done = .false.'.format(i3 + _INDENT)) + lines.append('{}exit'.format(i3 + _INDENT)) + lines.append('{}end if'.format(i3)) + lines.append('{}end do'.format(i2)) + lines.append('') + lines.append('{}if (all_done) then'.format(i2)) + # The per-suite ``_dynamic_constituents`` buffers are NOT + # deallocated here. They're owned by the suite-cap lifecycle (filled + # by ``_register``, gated by the suite-cap state machine), so + # tearing them down independently would leave the suite_state in a + # state where the next ``ccpp_register`` short-circuits via the + # ``state >= REGISTERED`` guard and the buffer never gets re-filled. + # They are deallocated in ``_final``'s last-to-leave block + # instead. + lines.append('{}deallocate({})'.format(i3, _CONST_OBJ)) + for std_name in index_names: + lines.append('{}index_of_{} = 0'.format(i3, std_name)) + lines.append('{}end if'.format(i2)) + lines.append('') + lines.append('{}end subroutine ccpp_deallocate_dynamic_constituents'.format(i1)) + return lines + + +######################################################################## +# Top-level generator +######################################################################## + +def _generate_host_constituents( + suite_results: List[SuiteResolution], + host_dict=None, +) -> Optional[List[str]]: + """Generate ``ccpp_host_constituents.F90`` source lines, or ``None``.""" + if not _any_constituent_state(suite_results): + return None + + register_suites = _suites_with_register_consts(suite_results) + index_names = _all_index_names(suite_results) + + lines: List[str] = [] + lines.append( + '! ccpp_host_constituents.F90 -- generated by ccpp_capgen_ng, do not edit' + ) + lines.append('module {}'.format(_HOST_CONST_MOD)) + lines.append('') + lines.append('{}use ccpp_kinds, only: kind_phys'.format(_INDENT)) + lines.append('{}use {}, only: &'.format(_INDENT, _CONST_PROP_MOD)) + lines.append('{}{}, &'.format(_INDENT * 2, _CONST_DDT)) + lines.append('{}{}, &'.format(_INDENT * 2, _CONST_PROP_TYPE)) + lines.append('{}{}, &'.format(_INDENT * 2, _CONST_PROP_PTR_TYPE)) + # int_unassigned is the sentinel returned by %const_index when a + # standard name is not in the constituent table. Used by + # ccpp_initialize_constituents to validate that every cached + # index_of_ integer corresponds to an actually-registered + # constituent — surfaces missing-registration bugs at init time + # instead of letting them become silent invalid array accesses + # later in run-phase scheme calls. + lines.append('{}int_unassigned'.format(_INDENT * 2)) + lines.append('') + lines.append('{}implicit none'.format(_INDENT)) + lines.append('{}private'.format(_INDENT)) + lines.append('') + + # Publics: state + routines. + publics = [_CONST_OBJ] + publics += ['index_of_{}'.format(n) for n in index_names] + publics += [ + 'ccpp_register_constituents', + 'ccpp_initialize_constituents', + 'ccpp_is_scheme_constituent', + 'ccpp_number_constituents', + 'ccpp_gather_constituents', + 'ccpp_update_constituents', + 'ccpp_const_get_index', + 'ccpp_constituents_array', + 'ccpp_advected_constituents_array', + 'ccpp_model_const_properties', + 'ccpp_deallocate_dynamic_constituents', + ] + if index_names: + publics.append('ccpp_model_const_stdnames') + publics += [_dyn_const_array_name(s) for s in register_suites] + for p in publics: + lines.append('{}public :: {}'.format(_INDENT, p)) + lines.append('') + + # State declarations. + lines.append( + '{}type({}), target, allocatable :: {}(:)'.format( + _INDENT, _CONST_DDT, _CONST_OBJ, + ) + ) + lines.append('') + for sname in register_suites: + buf = _dyn_const_array_name(sname) + lines.append( + '{}type({}), allocatable, target :: {}(:)'.format( + _INDENT, _CONST_PROP_TYPE, buf, + ) + ) + if register_suites: + lines.append('') + for std_name in index_names: + lines.append('{}integer :: index_of_{} = 0'.format(_INDENT, std_name)) + if index_names: + max_len = max(len(n) for n in index_names) + lines.append('') + lines.append( + '{}character(len={}), parameter :: ccpp_model_const_stdnames({}) = (/ &'.format( + _INDENT, max_len, len(index_names), + ) + ) + for i, std_name in enumerate(index_names): + sep = ', &' if i < len(index_names) - 1 else ' /)' + padding = ' ' * (max_len - len(std_name)) + lines.append("{}'{}{}'{}".format(_INDENT * 2, std_name, padding, sep)) + lines.append('') + lines.append('contains') + + lines.extend(_register_constituents_lines(suite_results, host_dict)) + lines.extend(_initialize_constituents_lines(suite_results, host_dict)) + lines.extend(_is_scheme_constituent_lines(suite_results)) + lines.extend(_wrap_method_subs(host_dict)) + lines.extend(_accessor_functions(host_dict)) + lines.extend(_deallocate_lines(suite_results, host_dict)) + + lines.append('') + lines.append('end module {}'.format(_HOST_CONST_MOD)) + return lines + + +def write_host_constituents( + suite_results: List[SuiteResolution], + outdir: str, + host_dict=None, +) -> Optional[str]: + """Write ``ccpp_host_constituents.F90`` if needed, return its path or ``None``.""" + lines = _generate_host_constituents(suite_results, host_dict) + if lines is None: + return None + if not os.path.isdir(outdir): + os.makedirs(outdir, exist_ok=True) + path = os.path.join(outdir, 'ccpp_host_constituents.F90') + with open(path, 'w') as fh: + fh.write('\n'.join(lines) + '\n') + return os.path.abspath(path) diff --git a/capgen-ng/generator/kinds_writer.py b/capgen-ng/generator/kinds_writer.py new file mode 100644 index 00000000..1cdf28c3 --- /dev/null +++ b/capgen-ng/generator/kinds_writer.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 + +"""Write the ``ccpp_kinds.F90`` module from ``--kind-type`` mappings. + +This is the simplest generated file -- a single Fortran module that re-exports +host-supplied kind parameters as ``integer, parameter, public`` constants. + +Each kind is described by a ``(module, spec)`` pair: the Fortran module that +defines the precision constant (``module``) and the name of that constant +(``spec``). When the spec is a standard ``ISO_FORTRAN_ENV`` name (e.g. +``REAL64``), the user may omit the module on the command line and it defaults +to ``iso_fortran_env``. When the host supplies its own kind module, the +caller passes ``(, )``. + +Example output for ``--kind-type kind_phys=REAL64`` (default module):: + + ! ccpp_kinds.F90 -- generated by ccpp_capgen_ng, do not edit + module ccpp_kinds + use iso_fortran_env, only: REAL64 + + implicit none + private + + integer, parameter, public :: kind_phys = REAL64 + + end module ccpp_kinds + +Example output for ``--kind-type kind_phys=my_host_kinds:kind_r8`` (host module):: + + ! ccpp_kinds.F90 -- generated by ccpp_capgen_ng, do not edit + module ccpp_kinds + use my_host_kinds, only: kind_r8 + + implicit none + private + + integer, parameter, public :: kind_phys = kind_r8 + + end module ccpp_kinds + +``ccpp_kinds.F90`` is always generated and is a dependency of all other +generated Fortran files that reference any kind parameter. +""" + +import os +from typing import Dict, List, Tuple + +from metadata.parse_tools import CCPPError + +_KINDS_FILENAME = 'ccpp_kinds.F90' +_KINDS_MODULE = 'ccpp_kinds' + +# Mapping from kind name to a (module, spec) pair. +KindSpec = Tuple[str, str] +KindMap = Dict[str, KindSpec] + + +######################################################################## +# Public API +######################################################################## + +def write_ccpp_kinds(kind_types: KindMap, output_root: str) -> str: + """Write ``ccpp_kinds.F90`` to *output_root*. + + Parameters + ---------- + kind_types : dict + Mapping ``kind_name -> (module_name, kind_spec)``. Example:: + + {'kind_phys': ('iso_fortran_env', 'REAL64'), + 'kind_dyn': ('my_host_kinds', 'kind_r4')} + + output_root : str + Directory where the file is written (created if absent). + + Returns + ------- + str + Absolute path of the written file. + + Raises + ------ + CCPPError + If *kind_types* is empty. + """ + os.makedirs(output_root, exist_ok=True) + lines = _generate_ccpp_kinds(kind_types) + out_path = os.path.join(output_root, _KINDS_FILENAME) + with open(out_path, 'w', encoding='utf-8') as fh: + fh.write('\n'.join(lines) + '\n') + return out_path + + +######################################################################## +# Internal generator +######################################################################## + +def _generate_ccpp_kinds(kind_types: KindMap) -> List[str]: + """Generate the ccpp_kinds module source as a list of lines. + + Lines do not carry trailing newlines. The caller joins them and + appends a final newline. + + Parameters + ---------- + kind_types : dict + Mapping ``kind_name -> (module_name, kind_spec)``. Keys are sorted + alphabetically in the output; ``use`` lines are grouped by module + and sorted alphabetically by module name. + + Returns + ------- + list of str + + Raises + ------ + CCPPError + If *kind_types* is empty. + + Examples + -------- + >>> lines = _generate_ccpp_kinds({'kind_phys': ('iso_fortran_env', 'REAL64')}) + >>> lines[0] + '! ccpp_kinds.F90 -- generated by ccpp_capgen_ng, do not edit' + >>> 'module ccpp_kinds' in lines + True + >>> any('kind_phys' in l and 'REAL64' in l for l in lines) + True + >>> 'end module ccpp_kinds' in lines + True + + Two distinct kinds, both from ``iso_fortran_env``: + + >>> lines = _generate_ccpp_kinds({ + ... 'kind_phys': ('iso_fortran_env', 'REAL64'), + ... 'kind_dyn': ('iso_fortran_env', 'REAL32'), + ... }) + >>> sum(1 for l in lines if 'parameter' in l and 'public' in l) + 2 + >>> sum(1 for l in lines if 'use iso_fortran_env' in l) + 1 + >>> any('REAL32' in l and 'REAL64' in l for l in lines) + True + + Two kinds sharing the same spec name -- the spec is listed once: + + >>> lines = _generate_ccpp_kinds({ + ... 'a': ('iso_fortran_env', 'REAL64'), + ... 'b': ('iso_fortran_env', 'REAL64'), + ... }) + >>> sum(1 for l in lines if 'REAL64' in l and 'iso_fortran_env' in l) + 1 + + Host-supplied module: + + >>> lines = _generate_ccpp_kinds({ + ... 'kind_phys': ('my_host_kinds', 'kind_r8'), + ... }) + >>> any('use my_host_kinds, only: kind_r8' in l for l in lines) + True + >>> any('kind_phys = kind_r8' in l for l in lines) + True + + Empty mapping is an error: + + >>> _generate_ccpp_kinds({}) + Traceback (most recent call last): + ... + metadata.parse_tools.parse_source.CCPPError: ccpp_kinds requires at least ... + """ + if not kind_types: + raise CCPPError( + "ccpp_kinds requires at least one kind mapping; " + "this is a generator bug -- the caller should inject " + "kind_phys=iso_fortran_env:REAL64 when no --kind-type is given" + ) + + # Group specs by module: {module: sorted_unique_specs} + by_module: Dict[str, List[str]] = {} + for _, (mod, spec) in kind_types.items(): + by_module.setdefault(mod, []) + if spec not in by_module[mod]: + by_module[mod].append(spec) + for mod in by_module: + by_module[mod].sort() + + lines: List[str] = [ + '! ccpp_kinds.F90 -- generated by ccpp_capgen_ng, do not edit', + 'module {}'.format(_KINDS_MODULE), + ] + + # ``use`` statements -- one per module, alphabetized; specs comma-separated. + sorted_modules = sorted(by_module) + use_lines: List[str] = [] + for mod in sorted_modules: + specs = by_module[mod] + use_lines.append(' use {}, only: {}'.format(mod, ', '.join(specs))) + # Column-align the ``only:`` clauses for readability when there is more + # than one ``use`` line. + if len(use_lines) > 1: + max_mod_len = max(len(mod) for mod in sorted_modules) + use_lines = [] + for mod in sorted_modules: + pad = ' ' * (max_mod_len - len(mod)) + specs = by_module[mod] + use_lines.append( + ' use {},{} only: {}'.format(mod, pad, ', '.join(specs)) + ) + lines.extend(use_lines) + + lines += [ + '', + ' implicit none', + ' private', + '', + ] + + # Parameter declarations, column-aligned on '='. + sorted_kinds = sorted(kind_types.items()) + max_len = max(len(k) for k, _ in sorted_kinds) + for kind_name, (_mod, spec) in sorted_kinds: + pad = ' ' * (max_len - len(kind_name)) + lines.append( + ' integer, parameter, public :: {}{} = {}'.format( + kind_name, pad, spec + ) + ) + + lines += [ + '', + 'end module {}'.format(_KINDS_MODULE), + ] + + return lines diff --git a/capgen-ng/generator/static_api.py b/capgen-ng/generator/static_api.py new file mode 100644 index 00000000..b5f1dc8c --- /dev/null +++ b/capgen-ng/generator/static_api.py @@ -0,0 +1,982 @@ +#!/usr/bin/env python3 + +"""Generate the static API module ``ccpp_static_api.F90``. + +The static API module is generated once per build (not per suite) and +provides the canonical public entry points that a host model calls: + + - ``ccpp_register`` + - ``ccpp_init`` + - ``ccpp_physics_init``, ``ccpp_physics_timestep_init``, + ``ccpp_physics_run``, ``ccpp_physics_timestep_final``, + ``ccpp_physics_final`` + - ``ccpp_final`` + +Each entry point dispatches by ``suite_name`` (and ``group_name`` for the +physics-phase calls) to the corresponding suite cap subroutine. + +The module also exposes five suite-introspection routines so a host +can query at runtime what is compiled into the API: + + - ``ccpp_physics_suite_list(suites)`` — names of all compiled-in suites. + - ``ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errflg)`` + — group ("part") names belonging to *suite_name*. + - ``ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errflg)`` + — scheme module names that compose *suite_name* (deduped across phases). + - ``ccpp_physics_suite_variables(suite_name, variable_list, errmsg, + errflg, [input_vars], [output_vars])`` — flat leaf standard names the + host exchanges with the suite. ``input_vars`` selects intent in/inout + (skipping ``protected`` host vars); ``output_vars`` selects intent + out/inout. Both default ``.true.`` (the union, deduplicated). + - ``ccpp_physics_suite_host_data(suite_name, variable_list, errmsg, + errflg, [input_vars], [output_vars])`` — same intent-filter semantics, + but DDT-leaf standard names are collapsed to the standard name of the + top-level DDT instance the host owns. Plain (non-DDT) leaves appear + individually, just like in ``..._suite_variables``. + +The suite-variables / suite-host-data routines exclude control variables +(those passed via the framework signature) — hosts already know their own +control table, and excluding them keeps the lists focused on the data the +host has to read/write to interface with the suite. +""" + +import os +from typing import Dict, List, Optional, Set, Tuple + +from metadata.parse_tools import CCPPError +from metadata.variable_resolver import SchemeStore +from generator.suite_resolver import ( + ResolvedArg, + SuiteResolution, + iter_phase_calls, + iter_phase_subcycles, +) +from metadata.variable_resolver import HostVarEntry +from generator.suite_cap import ( + _all_suite_scheme_names, + _schemes_with_register, + _suite_ctrl_args_for_phase, + _suite_extra_ctrl_entries_for_phase, + _PHYSICS_PHASES, + _CONST_MOD, + _CONST_DDT, + _CONST_PROP_TYPE, +) +from generator.group_cap import ( + _active_std_names, + _ctrl_entries_for_signature, + _ctrl_intent_for, + _ctrl_local, + _fortran_type_str, + _dim_decl, + _instance_local, + _intent_clause, +) + +_INDENT = ' ' + + +######################################################################## +# Helpers +######################################################################## + +def _all_ctrl_args_for_phase( + suite_resolutions: List[SuiteResolution], + phase: str, +) -> List[ResolvedArg]: + """Return the union of direct scheme control args across all suites for *phase*. + + Deduplicated by standard_name, first-seen order. + """ + seen: Dict[str, ResolvedArg] = {} + for sr in suite_resolutions: + for arg in _suite_ctrl_args_for_phase(sr, phase): + if arg.standard_name not in seen: + seen[arg.standard_name] = arg + return list(seen.values()) + + +def _all_extra_ctrl_entries_for_phase( + suite_resolutions: List[SuiteResolution], + phase: str, + ctrl_std_names: Set[str], + host_dict, +) -> List[HostVarEntry]: + """Return extra HostVarEntry objects (state indexing / dim subscripts) needed + by any suite-group for *phase* but not already in *ctrl_std_names*. + """ + if host_dict is None: + return [] + seen = set(ctrl_std_names) + result: Dict[str, HostVarEntry] = {} + for sr in suite_resolutions: + for entry in _suite_extra_ctrl_entries_for_phase(sr, phase, seen, host_dict): + if entry.standard_name not in seen and entry.standard_name not in result: + result[entry.standard_name] = entry + return list(result.values()) + + +######################################################################## +# Helpers — suite-introspection +######################################################################## + +def _emit_var_set_loop( + list_name: str, + items: List[str], + indent: str, + allocate: bool = True, +) -> List[str]: + """Emit ``allocate((N))`` followed by per-element assignments. + + Items are emitted as 1-based Fortran assignments: ``(i) = ''``. + Used by every introspection routine that returns a string list. + + >>> _emit_var_set_loop('x', ['a', 'b'], ' ') + [' allocate(x(2))', " x(1) = 'a'", " x(2) = 'b'"] + >>> _emit_var_set_loop('x', [], ' ') + [' allocate(x(0))'] + """ + lines = [] + if allocate: + lines.append('{}allocate({}({}))'.format(indent, list_name, len(items))) + for i, item in enumerate(items): + lines.append("{}{}({}) = '{}'".format(indent, list_name, i + 1, item)) + return lines + + +def _build_local_to_std_top_level_map(host_dict) -> Dict[str, str]: + """Build local_name → standard_name map for top-level host_dict entries. + + Only entries whose ``access_path`` does not contain ``'%'`` are + included — those are top-level DDT instances or plain (non-DDT) + leaves. Used to collapse a flattened DDT-leaf back to the standard + name of the DDT instance the host owns. + """ + if not host_dict: + return {} + return { + entry.local_name: entry.standard_name + for entry in host_dict.values() + if '%' not in entry.access_path + } + + +def _arg_top_level_name( + arg: ResolvedArg, + local_to_std: Dict[str, str], +) -> str: + """Return the top-level standard name for *arg* (DDT-collapsed view). + + For a plain leaf or a directly-referenced DDT instance, returns + ``arg.standard_name`` unchanged. For a DDT-leaf access path like + ``phys_state(instance_number)%t``, parses the root local name + (``phys_state``), strips any subscript, and looks up the standard + name of the host_dict entry whose ``local_name`` matches. + + Falls back to ``arg.standard_name`` if the lookup fails (which would + indicate inconsistent metadata, but produces a well-defined output). + """ + if arg.host_entry is None: + return arg.standard_name + ap = arg.host_entry.access_path + if '%' not in ap: + return arg.standard_name + root = ap.split('%', 1)[0] + paren = root.find('(') + if paren >= 0: + root = root[:paren] + return local_to_std.get(root, arg.standard_name) + + +def _collect_host_io( + sr: SuiteResolution, + host_dict=None, + collapse_ddts: bool = False, +) -> Tuple[List[str], List[str]]: + """Collect (inputs, outputs) standard names for the introspection routines. + + Walks every phase of every group of *sr*. Includes scheme args from + every ``source`` category EXCEPT ``'suite'`` — suite-owned vars are + internal data flow between schemes (one scheme writes them, another + reads them) and are not part of the host-facing variable list. + + Concretely, the returned lists include: + + * ``source='host'`` — host metadata vars. + * ``source='control'`` — control vars (errmsg, errflg, …) when they + appear as scheme args (not when they're framework-injected dummies). + * ``source='constituent'`` — both auto-resolved base/tendency + constituents and direct framework-array references + (``ccpp_constituents`` / ``ccpp_constituent_tendencies`` / etc.) and + register-phase ``ccpp_constituent_properties_t`` args. + + *inputs* : intent in ``('in', 'inout')`` and not protected (the + protected check is host-only; constituent / control args + have no protected attribute). + *outputs* : intent in ``('out', 'inout')``. + + When *collapse_ddts* is true, DDT-leaf standard names are mapped to + the standard name of the top-level DDT instance via *host_dict*. + If *host_dict* is ``None`` while *collapse_ddts* is true, the lookup + map is empty and every leaf falls back to its own standard name — + correct in scenarios with no DDT instances (e.g. minimal unit tests). + Both lists are returned sorted alphabetically for deterministic output. + + This matches the behavior of the original capgen's + ``ccpp_physics_suite_variables`` so host comparison tests round-trip + cleanly. + """ + local_to_std = _build_local_to_std_top_level_map(host_dict) if collapse_ddts else {} + + def _collapse_std(std_name: str) -> str: + """Map a CCPP standard name to its DDT-collapsed counterpart. + + For free host variables the standard name maps to itself; for a + DDT-component entry the result is the standard name of the + top-level instance. When ``collapse_ddts`` is False or the + std_name isn't in ``host_dict``, returns the input unchanged. + """ + if not collapse_ddts or host_dict is None: + return std_name + entry = host_dict.get(std_name) + if entry is None or '%' not in entry.access_path: + return std_name + root = entry.access_path.split('%', 1)[0] + paren = root.find('(') + if paren >= 0: + root = root[:paren] + return local_to_std.get(root, std_name) + + inputs: Set[str] = set() + outputs: Set[str] = set() + for group in sr.groups: + for items in group.phase_calls.values(): + # Subcycle loop bounds named by a CCPP standard name (e.g. + # ````) are pure + # inputs from the host — the host supplies the value and the + # generated cap reads it as a do-loop bound. Every nesting + # level contributes a (potentially different) bound, so walk + # the full subcycle tree. Without this, the host's + # compile-time bookkeeping (ccpp_physics_suite_variables / + # _suite_host_data) would silently omit required inputs. + for sc in iter_phase_subcycles(items): + if sc.loop_std_name: + inputs.add(_collapse_std(sc.loop_std_name)) + for call in iter_phase_calls(items): + for arg in call.args: + # Suite-owned vars are internal scheme-to-scheme + # plumbing and not part of the host-facing surface. + if arg.source == 'suite': + continue + name = ( + _arg_top_level_name(arg, local_to_std) + if collapse_ddts + else arg.standard_name + ) + if arg.intent in ('in', 'inout'): + # Only host args have a meaningful protected flag. + if not (arg.host_entry and arg.host_entry.protected): + inputs.add(name) + if arg.intent in ('out', 'inout'): + outputs.add(name) + # Framework-constituent dim references (e.g. + # number_of_ccpp_constituents as the trailing dim of + # ccpp_constituents) appear in the inputs list even + # though they don't have a dedicated scheme arg. + # Tracked on a dedicated field — these names are + # NOT in host_dict and must not be USE'd, so they + # don't live on used_dim_std_names. + for dim_std in arg.used_const_dim_std_names: + inputs.add(dim_std) + # Active-expression references (e.g. + # ``active = (flag_indicating_...)`` on a host var) + # are pure inputs: the host provides the flag so the + # suite knows whether the active arg is present. + # Without this, flags that *aren't* used as a direct + # scheme arg silently fall out of the introspection + # input list. + for active_std in _active_std_names(arg.active): + # Skip Fortran literals captured by the + # active-name tokenizer (e.g. ``.true.``). + if host_dict is not None and active_std in host_dict: + inputs.add(_collapse_std(active_std)) + return sorted(inputs), sorted(outputs) + + +######################################################################## +# Entry point generators +######################################################################## + +def _register_subroutine(suite_names: List[str], host_dict=None) -> List[str]: + """Generate ``ccpp_register`` (mandatory entry point). + + Always emitted with the minimal lifecycle signature + ``(suite_name, ccpp_error_code, ccpp_error_message, instance_number)``. + The body dispatches to ``_register`` for every known suite. Each + suite's register routine is responsible for allocating its state array + and DDT instance array, calling its register-phase scheme entrypoints, + and transitioning suite state to ``REGISTERED`` — even if the suite has + no register-providing schemes (the state transition still fires). + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + inst_local = _instance_local(host_dict) + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + suite_name_entry = host_dict.get('suite_name') if host_dict else None + suite_name_local = ( + suite_name_entry.local_name if suite_name_entry else 'suite_name' + ) + + sig_args = [suite_name_local, errflg_local, errmsg_local] + suite_call_args = [] + if inst_local: + sig_args.append(inst_local) + suite_call_args.append(inst_local) + suite_call_args += [errmsg_local, errflg_local] + + lines: List[str] = [''] + lines.append('{}subroutine ccpp_register({})'.format(i1, ', '.join(sig_args))) + lines.append('') + lines.append('{}character(len=*), intent(in) :: {}'.format(i2, suite_name_local)) + lines.append('{}integer, intent(out) :: {}'.format(i2, errflg_local)) + lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines += [ + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + '{}select case(trim({}))'.format(i2, suite_name_local), + ] + for sname in suite_names: + lines.append("{}case('{}')".format(i2, sname)) + lines.append('{}call {}_register({})'.format( + i3, sname, ', '.join(suite_call_args) + )) + lines += [ + '{}case default'.format(i2), + '{}{} = 1'.format(i3, errflg_local), + "{}{} = 'ccpp_register: unknown suite: ' // trim({})".format( + i3, errmsg_local, suite_name_local + ), + '{}end select'.format(i2), + '', + '{}end subroutine ccpp_register'.format(i1), + ] + return lines + + +def _init_subroutine(suite_names: List[str], host_dict=None) -> List[str]: + """Generate ``ccpp_init`` (minimal lifecycle signature). + + Signature: ``(suite_name, ccpp_error_code, ccpp_error_message, instance_number)``. + Forwards ``instance_number`` (when host-declared) to ``_init``. + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + inst_local = _instance_local(host_dict) + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + suite_name_entry = host_dict.get('suite_name') if host_dict else None + suite_name_local = ( + suite_name_entry.local_name if suite_name_entry else 'suite_name' + ) + + sig_args = [suite_name_local, errflg_local, errmsg_local] + suite_call_args: List[str] = [] + if inst_local: + sig_args.append(inst_local) + suite_call_args.append(inst_local) + suite_call_args += [errmsg_local, errflg_local] + + lines: List[str] = [''] + lines.append('{}subroutine ccpp_init({})'.format(i1, ', '.join(sig_args))) + lines.append('') + lines.append('{}character(len=*), intent(in) :: {}'.format(i2, suite_name_local)) + lines.append('{}integer, intent(out) :: {}'.format(i2, errflg_local)) + lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines += [ + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + '{}select case(trim({}))'.format(i2, suite_name_local), + ] + for sname in suite_names: + lines.append("{}case('{}')".format(i2, sname)) + lines.append('{}call {}_init({})'.format( + i3, sname, ', '.join(suite_call_args) + )) + lines += [ + '{}case default'.format(i2), + '{}{} = 1'.format(i3, errflg_local), + "{}{} = 'ccpp_init: unknown suite: ' // trim({})".format( + i3, errmsg_local, suite_name_local + ), + '{}end select'.format(i2), + '', + '{}end subroutine ccpp_init'.format(i1), + ] + return lines + + +def _final_subroutine(suite_names: List[str], host_dict=None) -> List[str]: + """Generate ``ccpp_final`` (minimal lifecycle signature). + + Signature: ``(suite_name, ccpp_error_code, ccpp_error_message, instance_number)``. + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + inst_local = _instance_local(host_dict) + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + suite_name_entry = host_dict.get('suite_name') if host_dict else None + suite_name_local = ( + suite_name_entry.local_name if suite_name_entry else 'suite_name' + ) + + sig_args = [suite_name_local, errflg_local, errmsg_local] + suite_call_args: List[str] = [] + if inst_local: + sig_args.append(inst_local) + suite_call_args.append(inst_local) + suite_call_args += [errmsg_local, errflg_local] + + lines: List[str] = [''] + lines.append('{}subroutine ccpp_final({})'.format(i1, ', '.join(sig_args))) + lines.append('') + lines.append('{}character(len=*), intent(in) :: {}'.format(i2, suite_name_local)) + lines.append('{}integer, intent(out) :: {}'.format(i2, errflg_local)) + lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines += [ + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + '{}select case(trim({}))'.format(i2, suite_name_local), + ] + for sname in suite_names: + lines.append("{}case('{}')".format(i2, sname)) + lines.append('{}call {}_final({})'.format( + i3, sname, ', '.join(suite_call_args) + )) + lines += [ + '{}case default'.format(i2), + '{}{} = 1'.format(i3, errflg_local), + "{}{} = 'ccpp_final: unknown suite: ' // trim({})".format( + i3, errmsg_local, suite_name_local + ), + '{}end select'.format(i2), + '', + '{}end subroutine ccpp_final'.format(i1), + ] + return lines + + +def _physics_subroutine( + phase: str, + suite_names: List[str], + suite_resolutions: List[SuiteResolution], + host_dict=None, +) -> List[str]: + """Generate one ``ccpp_physics_`` dispatch subroutine. + + The formal argument list is derived entirely from the host's ``type=control`` + metadata table. ``suite_name`` drives the top-level dispatch; ``group_name`` + (if present in the control table) is forwarded to the suite cap dispatch. + """ + sub_name = 'ccpp_physics_{}'.format(phase) + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + # Full control arg list (all control vars from host_dict). + ctrl_entries = _ctrl_entries_for_signature(host_dict) + ctrl_local_names = [e.local_name for e in ctrl_entries] + + # Args forwarded to suite cap (all ctrl vars except suite_name). + suite_ctrl_entries = _ctrl_entries_for_signature(host_dict, exclude={'suite_name'}) + suite_ctrl_local = [e.local_name for e in suite_ctrl_entries] + + # Local name for suite_name (used in select case dispatch). + suite_name_entry = next( + (e for e in ctrl_entries if e.standard_name == 'suite_name'), None + ) + suite_name_local = suite_name_entry.local_name if suite_name_entry else 'suite_name' + + lines = [''] + + # Subroutine signature. + if ctrl_local_names: + lines.append('{}subroutine {}( &'.format(i1, sub_name)) + for i, lname in enumerate(ctrl_local_names): + sep = ', &' if i < len(ctrl_local_names) - 1 else ')' + lines.append('{} {}{}'.format(i1, lname, sep)) + else: + lines.append('{}subroutine {}()'.format(i1, sub_name)) + + # Dummy declarations. + lines.append('') + for entry in ctrl_entries: + # Character control dummies always use len=* so the host's specific + # length doesn't propagate into the generated signature. + kind = 'len=*' if entry.type.strip().lower() == 'character' else entry.kind + t = _fortran_type_str(entry.type, kind) + dim = _dim_decl(entry.dimensions) + intent = _intent_clause(_ctrl_intent_for(entry.standard_name)) + lines.append( + '{}{}{}{} :: {}'.format(i2, t, intent, dim, entry.local_name) + ) + + lines.append('') + + # Initialize error reporting vars before any work. + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') + if errflg_local and errmsg_local: + lines.append("{}{} = ''".format(i2, errmsg_local)) + lines.append('{}{} = 0'.format(i2, errflg_local)) + lines.append('') + + # select case(suite_name) dispatch. + lines.append('{}select case(trim({}))'.format(i2, suite_name_local)) + for sname in suite_names: + lines.append("{}case('{}')".format(i2, sname)) + cap_sub = '{}_physics_{}'.format(sname, phase) + if suite_ctrl_local: + lines.append('{}call {}( &'.format(i3, cap_sub)) + for i, carg in enumerate(suite_ctrl_local): + sep = ', &' if i < len(suite_ctrl_local) - 1 else ')' + lines.append('{} {}{}'.format(i3, carg, sep)) + else: + lines.append('{}call {}()'.format(i3, cap_sub)) + lines.append('{}end select'.format(i2)) + + lines.append('') + lines.append('{}end subroutine {}'.format(i1, sub_name)) + return lines + + +######################################################################## +# Suite-introspection subroutines +######################################################################## + +def _suite_list_subroutine(suite_names: List[str]) -> List[str]: + """Generate ``ccpp_physics_suite_list(suites)``. + + Allocates ``suites`` to the number of compiled-in suites and assigns + each entry to the suite name as a literal string. + """ + i1 = _INDENT + i2 = _INDENT * 2 + + lines: List[str] = [''] + lines.append('{}subroutine ccpp_physics_suite_list(suites)'.format(i1)) + lines.append('') + lines.append( + '{}character(len=*), allocatable, intent(out) :: suites(:)'.format(i2) + ) + lines.append('') + lines.extend(_emit_var_set_loop('suites', suite_names, i2)) + lines.append('') + lines.append('{}end subroutine ccpp_physics_suite_list'.format(i1)) + return lines + + +def _suite_part_list_subroutine( + suite_names: List[str], + suite_resolutions: List[SuiteResolution], +) -> List[str]: + """Generate ``ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errflg)``. + + Dispatches by ``suite_name`` and returns the group ("part") names of + that suite in declaration order. ``case default`` sets ``errflg=1`` + and writes a message naming the unknown suite. + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + lines: List[str] = [''] + lines.append( + '{}subroutine ccpp_physics_suite_part_list( &'.format(i1) + ) + lines.append('{} suite_name, part_list, errmsg, errflg)'.format(i1)) + lines.append('') + lines.append( + '{}character(len=*), intent(in) :: suite_name'.format(i2) + ) + lines.append( + '{}character(len=*), allocatable, intent(out) :: part_list(:)'.format(i2) + ) + lines.append( + '{}character(len=*), intent(out) :: errmsg'.format(i2) + ) + lines.append( + '{}integer, intent(out) :: errflg'.format(i2) + ) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + lines.append('{}select case (trim(suite_name))'.format(i2)) + for sname, sr in zip(suite_names, suite_resolutions): + groups = [g.group_name for g in sr.groups] + lines.append("{}case ('{}')".format(i2, sname)) + lines.extend(_emit_var_set_loop('part_list', groups, i3)) + lines.append('{}case default'.format(i2)) + lines.append('{}errflg = 1'.format(i3)) + lines.append( + "{}errmsg = 'ccpp_physics_suite_part_list: unknown suite: ' " + "// trim(suite_name)".format(i3) + ) + lines.append('{}end select'.format(i2)) + lines.append('') + lines.append('{}end subroutine ccpp_physics_suite_part_list'.format(i1)) + return lines + + +def _suite_schemes_subroutine( + suite_names: List[str], + suite_resolutions: List[SuiteResolution], +) -> List[str]: + """Generate ``ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errflg)``. + + Returns the unique scheme names that compose *suite_name*, deduped + across all phases and groups, sorted alphabetically. + """ + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + lines: List[str] = [''] + lines.append( + '{}subroutine ccpp_physics_suite_schemes( &'.format(i1) + ) + lines.append('{} suite_name, scheme_list, errmsg, errflg)'.format(i1)) + lines.append('') + lines.append( + '{}character(len=*), intent(in) :: suite_name'.format(i2) + ) + lines.append( + '{}character(len=*), allocatable, intent(out) :: scheme_list(:)'.format(i2) + ) + lines.append( + '{}character(len=*), intent(out) :: errmsg'.format(i2) + ) + lines.append( + '{}integer, intent(out) :: errflg'.format(i2) + ) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + lines.append('{}select case (trim(suite_name))'.format(i2)) + for sname, sr in zip(suite_names, suite_resolutions): + schemes = sorted({ + call.scheme_name + for group in sr.groups + for items in group.phase_calls.values() + for call in iter_phase_calls(items) + }) + lines.append("{}case ('{}')".format(i2, sname)) + lines.extend(_emit_var_set_loop('scheme_list', schemes, i3)) + lines.append('{}case default'.format(i2)) + lines.append('{}errflg = 1'.format(i3)) + lines.append( + "{}errmsg = 'ccpp_physics_suite_schemes: unknown suite: ' " + "// trim(suite_name)".format(i3) + ) + lines.append('{}end select'.format(i2)) + lines.append('') + lines.append('{}end subroutine ccpp_physics_suite_schemes'.format(i1)) + return lines + + +def _suite_io_subroutine( + suite_names: List[str], + suite_resolutions: List[SuiteResolution], + host_dict=None, + collapse_ddts: bool = False, +) -> List[str]: + """Generate ``ccpp_physics_suite_variables`` or ``ccpp_physics_suite_host_data``. + + The two routines share the same Fortran shape — ``select case`` on + ``suite_name`` with one branch per suite and three nested branches + keyed on ``input_vars`` / ``output_vars`` — so they are emitted by + one helper. *collapse_ddts* controls both the routine name and + whether DDT-leaves get mapped to their top-level DDT instance. + + Optional ``input_vars``/``output_vars`` default to ``.true.``; + when both are ``.true.`` the union of inputs and outputs is returned + (deduplicated). When both are ``.false.`` an empty list is returned. + """ + sub_name = ( + 'ccpp_physics_suite_host_data' + if collapse_ddts + else 'ccpp_physics_suite_variables' + ) + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + i4 = _INDENT * 4 + + lines: List[str] = [''] + lines.append('{}subroutine {}( &'.format(i1, sub_name)) + lines.append( + '{} suite_name, variable_list, errmsg, errflg, &'.format(i1) + ) + lines.append('{} input_vars, output_vars)'.format(i1)) + lines.append('') + lines.append( + '{}character(len=*), intent(in) :: suite_name'.format(i2) + ) + lines.append( + '{}character(len=*), allocatable, intent(out) :: variable_list(:)'.format(i2) + ) + lines.append( + '{}character(len=*), intent(out) :: errmsg'.format(i2) + ) + lines.append( + '{}integer, intent(out) :: errflg'.format(i2) + ) + lines.append( + '{}logical, optional, intent(in) :: input_vars'.format(i2) + ) + lines.append( + '{}logical, optional, intent(in) :: output_vars'.format(i2) + ) + lines.append('') + lines.append('{}logical :: input_vars_use'.format(i2)) + lines.append('{}logical :: output_vars_use'.format(i2)) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + lines.append('{}if (present(input_vars)) then'.format(i2)) + lines.append('{}input_vars_use = input_vars'.format(i3)) + lines.append('{}else'.format(i2)) + lines.append('{}input_vars_use = .true.'.format(i3)) + lines.append('{}end if'.format(i2)) + lines.append('{}if (present(output_vars)) then'.format(i2)) + lines.append('{}output_vars_use = output_vars'.format(i3)) + lines.append('{}else'.format(i2)) + lines.append('{}output_vars_use = .true.'.format(i3)) + lines.append('{}end if'.format(i2)) + lines.append('') + lines.append('{}select case (trim(suite_name))'.format(i2)) + for sname, sr in zip(suite_names, suite_resolutions): + inputs, outputs = _collect_host_io(sr, host_dict, collapse_ddts) + union = sorted(set(inputs) | set(outputs)) + lines.append("{}case ('{}')".format(i2, sname)) + lines.append('{}if (input_vars_use .and. output_vars_use) then'.format(i3)) + lines.extend(_emit_var_set_loop('variable_list', union, i4)) + lines.append('{}else if (input_vars_use) then'.format(i3)) + lines.extend(_emit_var_set_loop('variable_list', inputs, i4)) + lines.append('{}else if (output_vars_use) then'.format(i3)) + lines.extend(_emit_var_set_loop('variable_list', outputs, i4)) + lines.append('{}else'.format(i3)) + lines.append('{}allocate(variable_list(0))'.format(i4)) + lines.append('{}end if'.format(i3)) + lines.append('{}case default'.format(i2)) + lines.append('{}errflg = 1'.format(i3)) + lines.append( + "{}errmsg = '{}: unknown suite: ' " + "// trim(suite_name)".format(i3, sub_name) + ) + lines.append('{}end select'.format(i2)) + lines.append('') + lines.append('{}end subroutine {}'.format(i1, sub_name)) + return lines + + +######################################################################## +# Module generator +######################################################################## + +def _generate_static_api( + suite_names: List[str], + suite_resolutions: List[SuiteResolution], + host_dict=None, + scheme_store: Optional[SchemeStore] = None, +) -> List[str]: + """Generate the full ``ccpp_static_api.F90`` module source lines. + + Parameters + ---------- + suite_names : list of str + Suite names in order. + suite_resolutions : list of SuiteResolution + Parallel to suite_names. + host_dict : dict, optional + Flat host+control dictionary. When provided, ``number_of_instances`` + is threaded through ``ccpp_init`` for multi-instance support. + scheme_store : SchemeStore, optional + When provided, used to determine which suites have at least one + scheme with a ``register`` phase. Only those suites contribute a + ``case`` arm to ``ccpp_register``, and ``ccpp_register`` itself is + only emitted if any suite qualifies. When omitted, no suite is + treated as having a register phase. + + Returns + ------- + list of str (no trailing newlines) + """ + if len(suite_names) != len(suite_resolutions): + raise CCPPError( + 'suite_names and suite_resolutions must have the same length' + ) + + lines: List[str] = [] + lines.append( + '! ccpp_static_api.F90 -- generated by ccpp_capgen_ng, do not edit' + ) + lines.append('module ccpp_static_api') + lines.append('') + + # USE each suite cap module. ``_register`` is now mandatory and + # always emitted in the suite cap, so always import it here too. + for sname in suite_names: + suite_cap_mod = 'ccpp_{}_cap'.format(sname) + suite_subs = [] + suite_subs.append('{}_register'.format(sname)) + suite_subs.append('{}_init'.format(sname)) + for phase in _PHYSICS_PHASES: + suite_subs.append('{}_physics_{}'.format(sname, phase)) + suite_subs.append('{}_final'.format(sname)) + syms = ', '.join(suite_subs) + lines.append('{}use {}, only: {}'.format(_INDENT, suite_cap_mod, syms)) + + # Re-export the host-facing constituent API + the constituent object so + # host code can do ``use ccpp_static_api, only: ...`` for *everything* + # it needs from CCPP. Mirrors original capgen, which put all of these + # on the generated host cap module. Only emitted when any suite uses + # constituent state (the ccpp_host_constituents module is only emitted + # in that case too). + uses_consts = any( + sr.uses_constituents or sr.constituent_register_calls + for sr in suite_resolutions + ) + constituent_pub_syms = [ + 'ccpp_model_constituents_obj', + 'ccpp_register_constituents', + 'ccpp_initialize_constituents', + 'ccpp_is_scheme_constituent', + 'ccpp_number_constituents', + 'ccpp_gather_constituents', + 'ccpp_update_constituents', + 'ccpp_const_get_index', + 'ccpp_constituents_array', + 'ccpp_advected_constituents_array', + 'ccpp_model_const_properties', + 'ccpp_deallocate_dynamic_constituents', + ] + if uses_consts: + lines.append('{}use ccpp_host_constituents, only: &'.format(_INDENT)) + for i, sym in enumerate(constituent_pub_syms): + sep = ', &' if i < len(constituent_pub_syms) - 1 else '' + lines.append('{}{}{}'.format(_INDENT * 2, sym, sep)) + + lines.append('') + lines.append('{}implicit none'.format(_INDENT)) + lines.append('{}private'.format(_INDENT)) + lines.append('') + + # Public declarations. ccpp_register is mandatory. + pub_subs = [] + pub_subs.append('ccpp_register') + pub_subs.append('ccpp_init') + for phase in _PHYSICS_PHASES: + pub_subs.append('ccpp_physics_{}'.format(phase)) + pub_subs.append('ccpp_final') + pub_subs.append('ccpp_physics_suite_list') + pub_subs.append('ccpp_physics_suite_part_list') + pub_subs.append('ccpp_physics_suite_schemes') + pub_subs.append('ccpp_physics_suite_variables') + pub_subs.append('ccpp_physics_suite_host_data') + if uses_consts: + pub_subs.extend(constituent_pub_syms) + for sub in pub_subs: + lines.append('{}public :: {}'.format(_INDENT, sub)) + + lines.append('') + lines.append('contains') + + # Subroutines. + lines.extend(_register_subroutine(suite_names, host_dict)) + lines.extend(_init_subroutine(suite_names, host_dict)) + for phase in _PHYSICS_PHASES: + lines.extend(_physics_subroutine(phase, suite_names, suite_resolutions, host_dict)) + lines.extend(_final_subroutine(suite_names, host_dict)) + # Introspection routines (do not advance state, no scheme calls). + lines.extend(_suite_list_subroutine(suite_names)) + lines.extend(_suite_part_list_subroutine(suite_names, suite_resolutions)) + lines.extend(_suite_schemes_subroutine(suite_names, suite_resolutions)) + lines.extend(_suite_io_subroutine( + suite_names, suite_resolutions, host_dict, collapse_ddts=False, + )) + lines.extend(_suite_io_subroutine( + suite_names, suite_resolutions, host_dict, collapse_ddts=True, + )) + + lines.append('') + lines.append('end module ccpp_static_api') + return lines + + +######################################################################## +# Public API +######################################################################## + +def write_static_api( + suite_names: List[str], + suite_resolutions: List[SuiteResolution], + output_root: str, + host_dict=None, + scheme_store: Optional[SchemeStore] = None, +) -> str: + """Write ``ccpp_static_api.F90`` to *output_root*. + + Parameters + ---------- + suite_names : list of str + suite_resolutions : list of SuiteResolution + Parallel to suite_names. + output_root : str + Output directory (created if absent). + host_dict : dict, optional + Flat host+control dictionary for multi-instance support. + scheme_store : SchemeStore, optional + Used to detect which suites have at least one ``register``-providing + scheme; only those drive emission of ``ccpp_register`` and the + constituent module USE. When omitted, ``ccpp_register`` is omitted + entirely. + + Returns + ------- + str + Absolute path of the written file. + """ + os.makedirs(output_root, exist_ok=True) + filename = 'ccpp_static_api.F90' + out_path = os.path.join(output_root, filename) + + lines = _generate_static_api(suite_names, suite_resolutions, host_dict, scheme_store) + with open(out_path, 'w', encoding='utf-8') as fh: + fh.write('\n'.join(lines) + '\n') + return out_path diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py new file mode 100644 index 00000000..f8c178f0 --- /dev/null +++ b/capgen-ng/generator/suite_cap.py @@ -0,0 +1,1056 @@ +#!/usr/bin/env python3 + +"""Generate the suite-level cap module ``ccpp__cap.F90``. + +The suite cap: + +* Imports all group cap modules and the constituent property module. +* Exposes eight public entry points: + + - ``_register`` — calls each scheme's ``_register`` to populate + the host-owned ``ccpp_model_constituents_t`` object. + - ``_init`` / ``_final`` — framework setup / teardown. + - ``_physics_init``, ``_physics_timestep_init``, + ``_physics_run``, ``_physics_timestep_final``, + ``_physics_final`` — dispatch by ``group_name`` to the + appropriate group cap subroutine. + +The static API (``ccpp_static_api.F90``) dispatches by ``suite_name`` to +these subroutines. +""" + +import os +from typing import Dict, List, Set + +from metadata.variable_resolver import HostVarEntry, SchemeStore +from generator.suite_resolver import ( + ResolvedArg, + ResolvedGroup, + SuiteResolution, + iter_phase_calls, +) +from generator.group_cap import ( + _ctrl_args_for_phase, + _ctrl_intent_for, + _ctrl_local, + _extra_dim_ctrl_entries, + _ctrl_entries_for_signature, + _fortran_type_str, + _dim_decl, + _instance_idx, + _instance_local, + _intent_clause, +) + +_INDENT = ' ' + +# Canonical set of physics phases, always dispatched by the suite cap. +_PHYSICS_PHASES = ('init', 'timestep_init', 'run', 'timestep_final', 'final') + +# Constituent type / module constants. +_CONST_MOD = 'ccpp_constituent_prop_mod' +_CONST_DDT = 'ccpp_model_constituents_t' +_CONST_PROP_TYPE = 'ccpp_constituent_properties_t' +_CONST_PROP_PTR_TYPE = 'ccpp_constituent_prop_ptr_t' +_CONST_OBJ_STDNAME = 'ccpp_model_constituents_object' + +# Framework-provided constituent symbol names — emitted as suite-cap +# module variables when the suite references constituent state. +_CONST_BASE_ARRAY = 'ccpp_constituents' +_CONST_TEND_ARRAY = 'ccpp_constituent_tendencies' +_CONST_PROPS = 'ccpp_constituent_properties' +_CONST_NUM = 'number_of_ccpp_constituents' + + +######################################################################## +# Helpers +######################################################################## + +def _all_suite_scheme_names(suite_res: SuiteResolution) -> List[str]: + """Return deduplicated scheme names from all groups and phases. + + The order is first-seen across groups (alphabetical by group, then by + order within each group's phase call list). + + >>> from generator.suite_resolver import SuiteResolution, ResolvedGroup, ResolvedCall + >>> rg = ResolvedGroup('grp', phase_calls={'run': [ResolvedCall('sch_a', 'run'), ResolvedCall('sch_b', 'run')]}) + >>> sr = SuiteResolution('s', groups=[rg]) + >>> _all_suite_scheme_names(sr) + ['sch_a', 'sch_b'] + """ + seen: Set[str] = set() + names: List[str] = [] + for rg in suite_res.groups: + for items in rg.phase_calls.values(): + for rc in iter_phase_calls(items): + if rc.scheme_name not in seen: + seen.add(rc.scheme_name) + names.append(rc.scheme_name) + return names + + +def _schemes_with_register( + scheme_names: List[str], + scheme_store: SchemeStore, +) -> List[str]: + """Return those scheme names that have a ``register`` phase. + + >>> from unittest.mock import MagicMock + >>> store = MagicMock() + >>> store.phases_for.side_effect = lambda n: ['register', 'run'] if n == 'sch_a' else ['run'] + >>> _schemes_with_register(['sch_a', 'sch_b'], store) + ['sch_a'] + """ + return [n for n in scheme_names if 'register' in scheme_store.phases_for(n)] + + +def _suite_ctrl_args_for_phase( + suite_res: SuiteResolution, + phase: str, +) -> List[ResolvedArg]: + """Return the union of control args across all groups for *phase*. + + The result is deduplicated by standard_name and preserves first-seen order. + """ + seen: Dict[str, ResolvedArg] = {} + for rg in suite_res.groups: + for arg in _ctrl_args_for_phase(rg, phase): + if arg.standard_name not in seen: + seen[arg.standard_name] = arg + return list(seen.values()) + + +def _group_ctrl_arg_names(rg: ResolvedGroup, phase: str, host_dict=None) -> List[str]: + """Return the local_name list for the control args of a group phase. + + These are the keyword names passed when calling the group cap subroutine. + Includes extra control vars needed for state indexing and dimension subscripts + (instance_number for suite-var access, control vars used only in dim subscripts). + """ + ctrl_args = _ctrl_args_for_phase(rg, phase) + names = [ + a.host_entry.local_name + for a in ctrl_args + if a.host_entry is not None + ] + phase_items = rg.phase_calls.get(phase, []) + for entry in _extra_dim_ctrl_entries(phase_items, phase, ctrl_args, host_dict): + if entry.local_name not in names: + names.append(entry.local_name) + return names + + +def _suite_extra_ctrl_entries_for_phase( + suite_res: SuiteResolution, + phase: str, + ctrl_std_names: Set[str], + host_dict, +) -> List[HostVarEntry]: + """Return extra HostVarEntry objects needed by any group for *phase* but not + already represented in *ctrl_std_names* (the direct scheme control args). + + This covers the same cases as ``_extra_dim_ctrl_entries`` but aggregated + across all groups so the suite-level dispatch subroutine has them in its + signature and can pass them down. + """ + if host_dict is None: + return [] + seen = set(ctrl_std_names) + result: Dict[str, HostVarEntry] = {} + for rg in suite_res.groups: + phase_items = rg.phase_calls.get(phase, []) + ctrl_args = _ctrl_args_for_phase(rg, phase) + for entry in _extra_dim_ctrl_entries(phase_items, phase, ctrl_args, host_dict): + if entry.standard_name not in seen and entry.standard_name not in result: + result[entry.standard_name] = entry + return list(result.values()) + + +######################################################################## +# Subroutine generators +######################################################################## + +def _register_calls(suite_res: SuiteResolution): + """Yield (group_name, ResolvedCall) for every register-phase scheme call. + + Groups are visited in suite-XML order; within each group the calls follow + the resolver's ordering (which mirrors the suite XML). + """ + for rg in suite_res.groups: + for rc in iter_phase_calls(rg.phase_calls.get('register', [])): + yield rg.group_name, rc + + +def _register_uses( + suite_res: SuiteResolution, + suite_name: str, + host_dict=None, +) -> Dict[str, Set[str]]: + """Collect ``{module: {symbol}}`` requirements for register-phase scheme calls. + + Includes: + - host modules for any host-owned register args, + - ``ccpp__data`` for any suite-owned register args, + - one entry per scheme module for its ``_register`` symbol, + - the constituent property type and the per-suite dynamic-constituent + buffer (owned by ``ccpp_host_constituents``) when any register call + produces constituents. + """ + uses: Dict[str, Set[str]] = {} + seen_schemes: Set[str] = set() + for _gname, rc in _register_calls(suite_res): + if rc.scheme_name not in seen_schemes: + seen_schemes.add(rc.scheme_name) + # Module is metadata-declared (``module_name`` in table props) + # when present; otherwise falls back to the scheme name. + scheme_module = rc.scheme_module or rc.scheme_name + uses.setdefault(scheme_module, set()).add( + '{}_register'.format(rc.scheme_name) + ) + for arg in rc.args: + if arg.is_constituent_arg: + continue # local temp, not a USE'd var + mod = arg.module_name + if mod is not None: + uses.setdefault(mod, set()).add(arg.root_symbol) + # Per-suite dynamic-constituent buffer is owned by ccpp_host_constituents + # and written into here. Pull in the constituent property type plus the + # buffer symbol. + if suite_res.constituent_register_calls: + uses.setdefault(_CONST_MOD, set()).add(_CONST_PROP_TYPE) + buf = '{}_dynamic_constituents'.format(suite_name) + uses.setdefault('ccpp_host_constituents', set()).add(buf) + return uses + + +def _add_call_uses(uses: Dict[str, Set[str]], rc) -> None: + """Merge USE-statement requirements for a single :class:`ResolvedCall`. + + Adds: + * the scheme module → ``_`` symbol so the call + site can resolve; + * each non-control arg's host/suite module → ``arg.root_symbol`` + (the top-level token of its access path) so the value is in + scope. + + Mutates *uses* in place. Used by ``_init`` and + ``_final`` to integrate the suite-level / + scheme calls into the USE block. + """ + scheme_module = rc.scheme_module or rc.scheme_name + uses.setdefault(scheme_module, set()).add( + '{}_{}'.format(rc.scheme_name, rc.phase) + ) + for arg in rc.args: + mod = arg.module_name + if mod is not None: + uses.setdefault(mod, set()).add(arg.root_symbol) + + +def _emit_register_call(rc, indent: str, errflg_local: str, lines: List[str]) -> None: + """Emit one scheme ``_register`` call with keyword args + error guard. + + Register-phase calls are kept simple: no transformations (transform code + paths are physics-phase only), keyword-arg style for clarity. + """ + sub = '{}_register'.format(rc.scheme_name) + if not rc.args: + lines.append('{}call {}()'.format(indent, sub)) + else: + lines.append('{}call {}( &'.format(indent, sub)) + for i, arg in enumerate(rc.args): + sep = ', &' if i < len(rc.args) - 1 else ')' + lines.append('{} {}={}{}'.format( + indent, arg.scheme_local_name, arg.call_expr, sep + )) + if errflg_local: + lines.append('{}if ({} /= 0) return'.format(indent, errflg_local)) + + +def _register_lines( + suite_name: str, + suite_res: SuiteResolution, + host_dict=None, +) -> List[str]: + """Generate the ``_register`` subroutine lines. + + Mandatory entry point: emitted unconditionally. Allocates the suite + state array and the suite-owned DDT array on first call, dispatches each + register-phase scheme call across all groups in suite-XML order, and + transitions the suite state for this instance to ``CCPP_SUITE_REGISTERED``. + + Minimal signature: ``(instance_number, errmsg, errflg)`` (instance_number + is included only when the host declares it). + """ + sub_name = '{}_register'.format(suite_name) + i1 = _INDENT + i2 = _INDENT * 2 + + inst_local = _instance_local(host_dict) + inst_idx = _instance_idx(host_dict) + + ninstances_entry = host_dict.get('number_of_instances') if host_dict else None + ninstances_local = ninstances_entry.local_name if ninstances_entry else None + ninstances_arg = ninstances_local if ninstances_local else '1' + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + + sig_args: List[str] = [] + if inst_local: + sig_args.append(inst_local) + sig_args += [errmsg_local, errflg_local] + + lines: List[str] = [] + lines.append('') + lines.append('{}subroutine {}({})'.format(i1, sub_name, ', '.join(sig_args))) + + # USE statements: scheme modules + host/suite-data modules referenced by + # register-phase scheme args. + reg_uses = _register_uses(suite_res, suite_name, host_dict) + if ninstances_local and ninstances_entry is not None and ninstances_entry.module_name: + reg_uses.setdefault(ninstances_entry.module_name, set()).add( + ninstances_local + ) + for mod in sorted(reg_uses): + syms = ', '.join(sorted(reg_uses[mod])) + lines.append('{}use {}, only: {}'.format(i2, mod, syms)) + + lines.append('') + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines += [ + '{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local), + '{}integer, intent(out) :: {}'.format(i2, errflg_local), + ] + + # Constituent merge: declare a per-scheme array temporary and a counter. + has_consts = bool(suite_res.constituent_register_calls) + if has_consts: + lines.append('') + lines.append( + '{}type({}), allocatable :: scheme_consts(:)'.format( + i2, _CONST_PROP_TYPE + ) + ) + lines.append('{}integer :: num_consts, i'.format(i2)) + + lines += [ + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + ] + + # Allocate state and DDT array on first call (idempotent). + suite_alloc_sub = '{}_suite_state_alloc'.format(suite_name) + lines.append('{}call {}({}, {}, {})'.format( + i2, suite_alloc_sub, ninstances_arg, errmsg_local, errflg_local + )) + lines.append('{}if ({} /= 0) return'.format(i2, errflg_local)) + lines.append('') + + # Per-instance idempotent skip: already registered or further along. + lines.append( + '{}if (ccpp_suite_state({}) >= CCPP_SUITE_REGISTERED) return'.format( + i2, inst_idx + ) + ) + lines.append('') + + if has_consts: + # Pack constituent-producing schemes' arrays into the per-suite + # buffer in ccpp_host_constituents. The actual merge into each + # instance's ``ccpp_model_constituents_obj(inst)`` happens later + # when the host calls ``ccpp_register_constituents`` per instance. + # + # The buffer itself is shared across instances (registration is + # identical per instance) — gated by ``.not. allocated`` so that + # only the first instance to enter does the two-pass count+pack. + # Subsequent instances reuse the same buffer. The state-array + # transition still runs per instance (after this block). + const_scheme_names = {sn for sn, _ in suite_res.constituent_register_calls} + buf = '{}_dynamic_constituents'.format(suite_name) + + lines.append( + '{}if (.not. allocated({})) then'.format(i2, buf) + ) + lines.append('{}num_consts = 0'.format(i2 + _INDENT)) + lines.append('{}! First pass: count constituents'.format(i2 + _INDENT)) + for _gname, rc in _register_calls(suite_res): + if rc.scheme_name in const_scheme_names: + _emit_register_call(rc, i2 + _INDENT, errflg_local, lines) + lines.append( + '{}num_consts = num_consts + size(scheme_consts, 1)'.format( + i2 + _INDENT, + ) + ) + lines.append('{}deallocate(scheme_consts)'.format(i2 + _INDENT)) + lines.append('') + lines.append('{}allocate({}(num_consts))'.format(i2 + _INDENT, buf)) + lines.append('{}num_consts = 0'.format(i2 + _INDENT)) + lines.append('') + lines.append('{}! Second pass: copy into per-suite buffer'.format(i2 + _INDENT)) + for _gname, rc in _register_calls(suite_res): + if rc.scheme_name in const_scheme_names: + _emit_register_call(rc, i2 + _INDENT, errflg_local, lines) + lines.append('{}do i = 1, size(scheme_consts, 1)'.format(i2 + _INDENT)) + lines.append( + '{}{}(num_consts + i) = scheme_consts(i)'.format( + i2 + _INDENT * 2, buf, + ) + ) + lines.append('{}end do'.format(i2 + _INDENT)) + lines.append( + '{}num_consts = num_consts + size(scheme_consts, 1)'.format( + i2 + _INDENT, + ) + ) + lines.append('{}deallocate(scheme_consts)'.format(i2 + _INDENT)) + lines.append('{}end if'.format(i2)) + lines.append('') + # Emit any non-constituent register calls in addition (always, per instance). + for _gname, rc in _register_calls(suite_res): + if rc.scheme_name not in const_scheme_names: + _emit_register_call(rc, i2, errflg_local, lines) + else: + # No constituent merge — emit register calls in suite-XML order. + for _gname, rc in _register_calls(suite_res): + _emit_register_call(rc, i2, errflg_local, lines) + + lines.append('') + lines.append( + '{}ccpp_suite_state({}) = CCPP_SUITE_REGISTERED'.format(i2, inst_idx) + ) + lines.append('') + lines.append('{}end subroutine {}'.format(i1, sub_name)) + return lines + + +def _init_lines( + suite_name: str, + suite_res: SuiteResolution, + host_dict=None, +) -> List[str]: + """Generate the ``_init`` framework-setup subroutine lines. + + Per-instance lifecycle: every call passes ``instance_number`` (when the host + declares it). Requires the suite to be in ``CCPP_SUITE_REGISTERED`` (i.e. + ``ccpp_register`` was called). The body: + + 1. Verifies state is ``REGISTERED``; idempotent skip if already + ``FRAMEWORK_INITIALIZED``; error otherwise. + 2. Calls each group ``state_alloc`` routine (idempotent first-call alloc). + 3. Calls the suite-data ``init_fields`` routine which allocates inner + allocatable suite-data fields using suite-owned dim values that may have + been written during the register phase. + 4. Sets ``ccpp_suite_state(instance_number) = CCPP_SUITE_FRAMEWORK_INITIALIZED``. + + Minimal signature: ``(instance_number, errmsg, errflg)``. + """ + sub_name = '{}_init'.format(suite_name) + i1 = _INDENT + i2 = _INDENT * 2 + + ninstances_entry = host_dict.get('number_of_instances') if host_dict else None + ninstances_local = ninstances_entry.local_name if ninstances_entry else None + ninstances_arg = ninstances_local if ninstances_local else '1' + + inst_local = _instance_local(host_dict) + inst_idx = _instance_idx(host_dict) + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + + sig_args: List[str] = [] + if inst_local: + sig_args.append(inst_local) + sig_args += [errmsg_local, errflg_local] + + lines: List[str] = [''] + lines.append('{}subroutine {}({})'.format(i1, sub_name, ', '.join(sig_args))) + + # USE: number_of_instances (from host module) for group state alloc; + # suite_data init_fields routine when this suite owns any vars; + # constituent object (from host module) for pointer binding; + # suite-level scheme module + per-arg host modules. + extra_uses: Dict[str, Set[str]] = {} + if ninstances_local and ninstances_entry is not None and ninstances_entry.module_name: + extra_uses.setdefault(ninstances_entry.module_name, set()).add( + ninstances_local + ) + if suite_res.suite_vars: + data_mod = 'ccpp_{}_data'.format(suite_name) + init_fields = 'ccpp_{}_suite_data_init_fields'.format(suite_name) + extra_uses.setdefault(data_mod, set()).add(init_fields) + if suite_res.suite_init_call is not None: + _add_call_uses(extra_uses, suite_res.suite_init_call) + for mod in sorted(extra_uses): + syms = ', '.join(sorted(extra_uses[mod])) + lines.append('{}use {}, only: {}'.format(i2, mod, syms)) + + lines.append('') + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines += [ + '{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local), + '{}integer, intent(out) :: {}'.format(i2, errflg_local), + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + ] + + # State guard: must be in REGISTERED state (or already INITIALIZED — idempotent). + sub_label = '{}_init'.format(suite_name) + lines += [ + '{}if (.not. allocated(ccpp_suite_state)) then'.format(i2), + "{} {} = '{}: ccpp_register has not been called'".format( + i2, errmsg_local, sub_label + ), + '{} {} = 1'.format(i2, errflg_local), + '{} return'.format(i2), + '{}end if'.format(i2), + '{}if (ccpp_suite_state({}) == CCPP_SUITE_FRAMEWORK_INITIALIZED) return'.format( + i2, inst_idx + ), + '{}if (ccpp_suite_state({}) /= CCPP_SUITE_REGISTERED) then'.format( + i2, inst_idx + ), + "{} {} = '{}: invalid suite state (expected REGISTERED)'".format( + i2, errmsg_local, sub_label + ), + '{} {} = 1'.format(i2, errflg_local), + '{} return'.format(i2), + '{}end if'.format(i2), + '', + ] + + # Group state allocators (idempotent). + for rg in suite_res.groups: + alloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, rg.group_name, 'state_alloc') + lines.append('{}call {}({}, {}, {})'.format( + i2, alloc_sub, ninstances_arg, errmsg_local, errflg_local + )) + lines.append('{}if ({} /= 0) return'.format(i2, errflg_local)) + + # Allocate inner suite-data allocatable fields for this instance. + if suite_res.suite_vars: + init_fields = 'ccpp_{}_suite_data_init_fields'.format(suite_name) + lines.append('{}call {}({}, {}, {})'.format( + i2, init_fields, inst_idx, errmsg_local, errflg_local + )) + lines.append('{}if ({} /= 0) return'.format(i2, errflg_local)) + + # Constituent state binding is owned by the host_constituents module + # under option A — the host calls ccpp_initialize_constituents separately + # to bind ccpp_constituents/ccpp_constituent_tendencies and populate the + # index_of_ integers. + + # Suite-level scheme call (if declared in the SDF). Runs once + # per ``_init`` invocation, after all group state allocators + # have populated their state arrays, and before the suite-state + # transition to FRAMEWORK_INITIALIZED. Errflg check follows the call. + if suite_res.suite_init_call is not None: + lines.append('') + from generator.group_cap import _emit_one_call + _emit_one_call(suite_res.suite_init_call, i2, lines) + + lines += [ + '', + '{}ccpp_suite_state({}) = CCPP_SUITE_FRAMEWORK_INITIALIZED'.format( + i2, inst_idx + ), + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + return lines + + +def _final_lines( + suite_name: str, + suite_res: SuiteResolution, + host_dict=None, +) -> List[str]: + """Generate the ``_final`` framework-teardown subroutine lines. + + Per-instance lifecycle: + + 1. Errors if ``ccpp_suite_state`` is not allocated. + 2. Per-instance idempotent skip: returns immediately if this instance's slot + is already ``CCPP_SUITE_UNREGISTERED``. + 3. Calls suite-data ``final_fields`` for this instance to deallocate inner + allocatable suite-data fields (when this suite owns any). + 4. Sets ``ccpp_suite_state(instance_number) = CCPP_SUITE_UNREGISTERED``. + 5. Last-to-leave dealloc: when every slot is ``UNREGISTERED`` after the + flip, calls each group ``state_dealloc`` and the suite ``state_dealloc`` + (which also tears down the suite_data DDT array). + + Minimal signature: ``(instance_number, errmsg, errflg)``. + """ + sub_name = '{}_final'.format(suite_name) + i1 = _INDENT + i2 = _INDENT * 2 + + inst_local = _instance_local(host_dict) + inst_idx = _instance_idx(host_dict) + + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' + + sig_args: List[str] = [] + if inst_local: + sig_args.append(inst_local) + sig_args += [errmsg_local, errflg_local] + + lines: List[str] = [''] + lines.append('{}subroutine {}({})'.format(i1, sub_name, ', '.join(sig_args))) + + final_uses: Dict[str, Set[str]] = {} + if suite_res.suite_vars: + data_mod = 'ccpp_{}_data'.format(suite_name) + final_fields = 'ccpp_{}_suite_data_final_fields'.format(suite_name) + final_uses.setdefault(data_mod, set()).add(final_fields) + + # If we registered constituents, the per-suite buffer (owned by + # ccpp_host_constituents) is torn down here in the last-to-leave block. + if suite_res.constituent_register_calls: + buf = '{}_dynamic_constituents'.format(suite_name) + final_uses.setdefault('ccpp_host_constituents', set()).add(buf) + + # Suite-level scheme call (if declared in the SDF) — pull + # in the scheme module and any host modules its args reference. + if suite_res.suite_final_call is not None: + _add_call_uses(final_uses, suite_res.suite_final_call) + + for mod in sorted(final_uses): + syms = ', '.join(sorted(final_uses[mod])) + lines.append('{}use {}, only: {}'.format(i2, mod, syms)) + + lines.append('') + if inst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + lines += [ + '{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local), + '{}integer, intent(out) :: {}'.format(i2, errflg_local), + '', + "{}{} = ''".format(i2, errmsg_local), + '{}{} = 0'.format(i2, errflg_local), + '', + '{}if (.not. allocated(ccpp_suite_state)) then'.format(i2), + "{} {} = '{}_final: ccpp_register has not been called'".format( + i2, errmsg_local, suite_name + ), + '{} {} = 1'.format(i2, errflg_local), + '{} return'.format(i2), + '{}end if'.format(i2), + '', + '{}if (ccpp_suite_state({}) == CCPP_SUITE_UNREGISTERED) return'.format( + i2, inst_idx + ), + '', + ] + + # Deallocate inner suite-data fields if this instance was past REGISTERED. + if suite_res.suite_vars: + final_fields = 'ccpp_{}_suite_data_final_fields'.format(suite_name) + lines.append( + '{}if (ccpp_suite_state({}) == CCPP_SUITE_FRAMEWORK_INITIALIZED) then'.format( + i2, inst_idx + ) + ) + lines.append('{} call {}({}, {}, {})'.format( + i2, final_fields, inst_idx, errmsg_local, errflg_local + )) + lines.append('{} if ({} /= 0) return'.format(i2, errflg_local)) + lines.append('{}end if'.format(i2)) + lines.append('') + + # Suite-level scheme call (if declared in the SDF). Runs + # once per ``_final`` invocation, before the suite-state + # transition to UNREGISTERED. Errflg check follows the call. + if suite_res.suite_final_call is not None: + from generator.group_cap import _emit_one_call + _emit_one_call(suite_res.suite_final_call, i2, lines) + + lines.append( + '{}ccpp_suite_state({}) = CCPP_SUITE_UNREGISTERED'.format(i2, inst_idx) + ) + lines.append('') + lines.append( + '{}if (all(ccpp_suite_state == CCPP_SUITE_UNREGISTERED)) then'.format(i2) + ) + for rg in suite_res.groups: + dealloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, rg.group_name, 'state_dealloc') + lines.append('{} call {}({}, {})'.format( + i2, dealloc_sub, errmsg_local, errflg_local + )) + lines.append('{} if ({} /= 0) return'.format(i2, errflg_local)) + suite_dealloc_sub = '{}_suite_state_dealloc'.format(suite_name) + lines.append('{} call {}({}, {})'.format( + i2, suite_dealloc_sub, errmsg_local, errflg_local + )) + lines.append('{} if ({} /= 0) return'.format(i2, errflg_local)) + # Constituent OBJ teardown lives in ccpp_deallocate_dynamic_constituents + # (the host calls it per instance + last-to-leave dealloc). The + # per-suite ``_dynamic_constituents`` buffer, however, is + # tied to THIS suite's lifecycle — populated by ``_register`` + # under the suite-cap state guard — so it must be deallocated here + # in the last-to-leave block, not in the constituent-deallocate + # routine. Otherwise the next ``ccpp_register`` short-circuits on + # the state guard without re-filling the buffer. + if suite_res.constituent_register_calls: + buf = '{}_dynamic_constituents'.format(suite_name) + lines.append('{} if (allocated({})) deallocate({})'.format( + i2, buf, buf, + )) + lines += [ + '{}end if'.format(i2), + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + return lines + + +def _physics_dispatch_lines( + suite_name: str, + phase: str, + suite_res: SuiteResolution, + host_dict=None, +) -> List[str]: + """Generate a ``_physics_`` dispatch subroutine. + + The subroutine signature is derived entirely from the host's ``type=control`` + metadata (all control variables except ``suite_name``, which is consumed at the + static API dispatch level). When ``group_name`` is in the control table the + body uses a ``select case`` dispatch; otherwise all groups are called + unconditionally. + """ + sub_name = '{}_physics_{}'.format(suite_name, phase) + i1 = _INDENT + i2 = _INDENT * 2 + i3 = _INDENT * 3 + + # Suite-level signature: all ctrl vars excluding suite_name. + ctrl_entries = _ctrl_entries_for_signature(host_dict, exclude={'suite_name'}) + ctrl_local_names = [e.local_name for e in ctrl_entries] + + # Determine if group_name is in the control table. + group_name_entry = next( + (e for e in ctrl_entries if e.standard_name == 'group_name'), None + ) + has_group_name = group_name_entry is not None + + # Group-level args passed when calling group cap subroutines. + group_ctrl_entries = _ctrl_entries_for_signature( + host_dict, exclude={'suite_name', 'group_name'} + ) + group_ctrl_local = [e.local_name for e in group_ctrl_entries] + + lines: List[str] = [] + lines.append('') + + # Subroutine signature. + if ctrl_local_names: + lines.append('{}subroutine {}( &'.format(i1, sub_name)) + for i, lname in enumerate(ctrl_local_names): + sep = ', &' if i < len(ctrl_local_names) - 1 else ')' + lines.append('{} {}{}'.format(i1, lname, sep)) + else: + lines.append('{}subroutine {}()'.format(i1, sub_name)) + + # Dummy argument declarations. + lines.append('') + for entry in ctrl_entries: + # Character control dummies always use len=* so the host's specific + # length doesn't propagate into the generated signature. + kind = 'len=*' if entry.type.strip().lower() == 'character' else entry.kind + t = _fortran_type_str(entry.type, kind) + dim = _dim_decl(entry.dimensions) + intent = _intent_clause(_ctrl_intent_for(entry.standard_name)) + lines.append( + '{}{}{}{} :: {}'.format(i2, t, intent, dim, entry.local_name) + ) + + lines.append('') + + # Initialize error reporting vars before any work, then guard on the + # per-instance suite state. + errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') + errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') + if errflg_local and errmsg_local: + lines.append("{}{} = ''".format(i2, errmsg_local)) + lines.append('{}{} = 0'.format(i2, errflg_local)) + lines.append('') + + inst_idx = _instance_idx(host_dict) + sub_label = '{}_physics_{}'.format(suite_name, phase) + lines += [ + '{}if (.not. allocated(ccpp_suite_state)) then'.format(i2), + "{} {} = '{}: ccpp_register has not been called'".format( + i2, errmsg_local, sub_label + ), + '{} {} = 1'.format(i2, errflg_local), + '{} return'.format(i2), + '{}end if'.format(i2), + '{}if (ccpp_suite_state({}) /= CCPP_SUITE_FRAMEWORK_INITIALIZED) then'.format( + i2, inst_idx + ), + "{} {} = '{}: invalid suite state'".format( + i2, errmsg_local, sub_label + ), + '{} {} = 1'.format(i2, errflg_local), + '{} return'.format(i2), + '{}end if'.format(i2), + '', + ] + + def _emit_group_call(rg, indent): + # Group phase subroutines are always emitted (so the per-group state + # machine transitions through every phase), so we always dispatch. + cap_sub = 'ccpp_{}_{}_{}'.format(suite_name, rg.group_name, phase) + if group_ctrl_local: + lines.append('{}call {}( &'.format(indent, cap_sub)) + for idx, lname in enumerate(group_ctrl_local): + sep = ', &' if idx < len(group_ctrl_local) - 1 else ')' + lines.append('{} {}{}'.format(indent, lname, sep)) + else: + lines.append('{}call {}()'.format(indent, cap_sub)) + + if has_group_name: + grp_local = group_name_entry.local_name + lines.append('{}select case(trim({}))'.format(i2, grp_local)) + # '' or 'all' → call all groups. + lines.append("{}case('', 'all')".format(i2)) + for rg in suite_res.groups: + _emit_group_call(rg, i3) + # Individual group cases. + for rg in suite_res.groups: + lines.append("{}case('{}')".format(i2, rg.group_name)) + _emit_group_call(rg, i3) + lines.append('{}end select'.format(i2)) + else: + # No group_name control var: call all groups unconditionally. + for rg in suite_res.groups: + _emit_group_call(rg, i2) + + lines.append('') + lines.append('{}end subroutine {}'.format(i1, sub_name)) + return lines + + +def _suite_state_alloc_lines( + suite_name: str, + has_suite_vars: bool, +) -> List[str]: + """Generate the ``_suite_state_alloc`` subroutine. + + Idempotent allocator for the per-instance suite state array and the + suite-owned DDT array. Inner allocatable fields inside the DDT are NOT + allocated here — that happens in ``ccpp__suite_data_init_fields``, + called from ``_init`` after register-phase scheme calls have set + any suite-owned scalar dimensions. + """ + sub_name = '{}_suite_state_alloc'.format(suite_name) + data_alloc = 'ccpp_{}_suite_data_alloc'.format(suite_name) + data_mod = 'ccpp_{}_data'.format(suite_name) + i1 = _INDENT + i2 = _INDENT * 2 + lines = [ + '', + '{}subroutine {}(number_of_instances, errmsg, errflg)'.format(i1, sub_name), + ] + if has_suite_vars: + lines.append('{}use {}, only: {}'.format(i2, data_mod, data_alloc)) + lines += [ + '', + '{}integer, intent(in) :: number_of_instances'.format(i2), + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + '{}if (allocated(ccpp_suite_state)) return'.format(i2), + '{}allocate(ccpp_suite_state(number_of_instances))'.format(i2), + '{}ccpp_suite_state(:) = CCPP_SUITE_UNREGISTERED'.format(i2), + ] + if has_suite_vars: + lines += [ + '{}call {}(number_of_instances, errmsg, errflg)'.format(i2, data_alloc), + '{}if (errflg /= 0) return'.format(i2), + ] + lines += [ + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + return lines + + +def _suite_state_dealloc_lines( + suite_name: str, + has_suite_vars: bool, +) -> List[str]: + """Generate the ``_suite_state_dealloc`` subroutine.""" + sub_name = '{}_suite_state_dealloc'.format(suite_name) + data_dealloc = 'ccpp_{}_suite_data_dealloc'.format(suite_name) + data_mod = 'ccpp_{}_data'.format(suite_name) + i1 = _INDENT + i2 = _INDENT * 2 + lines = [ + '', + '{}subroutine {}(errmsg, errflg)'.format(i1, sub_name), + ] + if has_suite_vars: + lines.append('{}use {}, only: {}'.format(i2, data_mod, data_dealloc)) + lines += [ + '', + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + ] + if has_suite_vars: + lines += [ + '{}call {}(errmsg, errflg)'.format(i2, data_dealloc), + '{}if (errflg /= 0) return'.format(i2), + ] + lines += [ + '{}if (allocated(ccpp_suite_state)) deallocate(ccpp_suite_state)'.format(i2), + '', + '{}end subroutine {}'.format(i1, sub_name), + ] + return lines + + +######################################################################## +# Module generator +######################################################################## + +def _generate_suite_cap( + suite_name: str, + suite_res: SuiteResolution, + scheme_store: SchemeStore, + host_dict=None, +) -> List[str]: + """Generate the full ``ccpp__cap.F90`` module source lines. + + Parameters + ---------- + suite_name : str + suite_res : SuiteResolution + scheme_store : SchemeStore + host_dict : dict, optional + Flat host+control variable dictionary. When provided, ``number_of_instances`` + and ``instance_number`` are used for multi-instance state array sizing and + indexing. + + Returns + ------- + list of str (no trailing newlines) + """ + mod_name = 'ccpp_{}_cap'.format(suite_name) + lines: List[str] = [] + + # Module header. + lines.append( + '! ccpp_{}_cap.F90 -- generated by ccpp_capgen_ng, do not edit'.format( + suite_name + ) + ) + lines.append('module {}'.format(mod_name)) + lines.append('') + + # USE statements: one per group cap (all phase + state subroutines). + for rg in suite_res.groups: + group_cap_mod = 'ccpp_{}_{}_{}'.format(suite_name, rg.group_name, 'cap') + syms_list = [ + 'ccpp_{}_{}_{}'.format(suite_name, rg.group_name, p) + for p in _PHYSICS_PHASES + ] + syms_list.append('ccpp_{}_{}_{}'.format(suite_name, rg.group_name, 'state_alloc')) + syms_list.append('ccpp_{}_{}_{}'.format(suite_name, rg.group_name, 'state_dealloc')) + lines.append('{}use {}, only: {}'.format( + _INDENT, group_cap_mod, ', '.join(syms_list) + )) + + lines.append('') + lines.append('{}implicit none'.format(_INDENT)) + lines.append('{}private'.format(_INDENT)) + lines.append('') + + # Public declarations: all framework lifecycle and physics phase entry + # points are always emitted. ``ccpp_register`` is mandatory in the new + # design — even an empty register phase fires the state transition. + pub_subs = [] + pub_subs.append('{}_register'.format(suite_name)) + pub_subs.append('{}_init'.format(suite_name)) + for phase in _PHYSICS_PHASES: + pub_subs.append('{}_physics_{}'.format(suite_name, phase)) + pub_subs.append('{}_final'.format(suite_name)) + pub_subs.append('{}_suite_state_alloc'.format(suite_name)) + pub_subs.append('{}_suite_state_dealloc'.format(suite_name)) + + for sub in pub_subs: + lines.append('{}public :: {}'.format(_INDENT, sub)) + + lines.append('') + lines.append('{}integer, private, parameter :: CCPP_SUITE_UNREGISTERED = 0'.format(_INDENT)) + lines.append('{}integer, private, parameter :: CCPP_SUITE_REGISTERED = 1'.format(_INDENT)) + lines.append('{}integer, private, parameter :: CCPP_SUITE_FRAMEWORK_INITIALIZED = 2'.format(_INDENT)) + lines.append('{}integer, private, allocatable :: ccpp_suite_state(:)'.format(_INDENT)) + lines.append('') + lines.append('contains') + + # Subroutines. Order: register, init, physics_*, final, state_alloc/dealloc. + lines.extend(_register_lines(suite_name, suite_res, host_dict)) + lines.extend(_init_lines(suite_name, suite_res, host_dict)) + for phase in _PHYSICS_PHASES: + lines.extend(_physics_dispatch_lines(suite_name, phase, suite_res, host_dict)) + lines.extend(_final_lines(suite_name, suite_res, host_dict)) + + has_suite_vars = bool(suite_res.suite_vars) + lines.extend(_suite_state_alloc_lines(suite_name, has_suite_vars)) + lines.extend(_suite_state_dealloc_lines(suite_name, has_suite_vars)) + + lines.append('') + lines.append('end module {}'.format(mod_name)) + return lines + + +######################################################################## +# Public API +######################################################################## + +def write_suite_cap( + suite_name: str, + suite_res: SuiteResolution, + scheme_store: SchemeStore, + output_root: str, + host_dict=None, +) -> str: + """Write ``ccpp__cap.F90`` to *output_root*. + + Parameters + ---------- + suite_name : str + suite_res : SuiteResolution + scheme_store : SchemeStore + output_root : str + Output directory (created if absent). + host_dict : dict, optional + Flat host+control dictionary for multi-instance support. + + Returns + ------- + str + Absolute path of the written file. + """ + os.makedirs(output_root, exist_ok=True) + filename = 'ccpp_{}_cap.F90'.format(suite_name) + out_path = os.path.join(output_root, filename) + + lines = _generate_suite_cap(suite_name, suite_res, scheme_store, host_dict) + with open(out_path, 'w', encoding='utf-8') as fh: + fh.write('\n'.join(lines) + '\n') + return out_path diff --git a/capgen-ng/generator/suite_data.py b/capgen-ng/generator/suite_data.py new file mode 100644 index 00000000..4b7601c9 --- /dev/null +++ b/capgen-ng/generator/suite_data.py @@ -0,0 +1,446 @@ +#!/usr/bin/env python3 + +"""Generate the suite data module ``ccpp__data.F90``.""" + +import os +from typing import Dict, List, Optional + +from metadata.parse_tools import CCPPError +from generator.suite_resolver import SuiteVar + +_INDENT = ' ' + +_INTRINSICS = frozenset({ + 'real', 'integer', 'character', 'logical', 'complex', 'double precision' +}) + + +def _type_str(type_: str, kind: str) -> str: + """Return the Fortran type clause for a SuiteVar field. + + >>> _type_str('real', 'kind_phys') + 'real(kind=kind_phys)' + >>> _type_str('real', '') + 'real' + >>> _type_str('my_ddt', '') + 'type(my_ddt)' + >>> _type_str('character', 'len=512') + 'character(len=512)' + """ + t = type_.strip() + if t.lower() not in _INTRINSICS and not t.lower().startswith('external:'): + if not t.lower().startswith('type('): + t = 'type({})'.format(t) + if kind: + if t.lower().startswith('character'): + return 'character({})'.format(kind) + return '{}(kind={})'.format(t, kind) + return t + + +def _collect_dim_uses( + suite_vars: Dict[str, SuiteVar], + host_dict, +) -> Dict[str, List[str]]: + """Return {module_name: [local_name, ...]} for dimension variables of suite vars. + + Host-module variables (``is_control=False``) are included keyed by their + declaring module. Control variables and suite-owned dimensions are + excluded — suite-owned dim scalars (set during ``_register``) are accessed + via ``ccpp_suite_data(i)%`` directly within the same module so no + USE statement is needed for them. Control variables cannot be dimensions + of suite-owned data because they are not available at ``init_fields`` + time when allocations happen. + + Raises CCPPError if a dimension is found in host_dict but is a control + variable. Suite-owned dimensions (not in host_dict) are silently skipped. + """ + uses: Dict[str, List] = {} + seen: set = set() + suite_var_std_names = set(suite_vars.keys()) + for sv in sorted(suite_vars.values(), key=lambda v: v.standard_name): + for dim_std in sv.dimensions: + if dim_std in seen: + continue + seen.add(dim_std) + # Suite-owned dim → no USE needed (same module access). + if dim_std in suite_var_std_names: + continue + if host_dict is None: + raise CCPPError( + "Suite-owned variable '{}' has dimension '{}' but no host " + "metadata was provided to resolve it".format( + sv.standard_name, dim_std + ) + ) + entry = host_dict.get(dim_std) + if entry is None: + raise CCPPError( + "Suite-owned variable '{}' dimension '{}' not found in " + "host metadata or in suite-owned variables".format( + sv.standard_name, dim_std + ) + ) + if entry.is_control: + raise CCPPError( + "Suite-owned variable '{}' dimension '{}' is a control " + "variable; suite data must use host-module or " + "suite-owned dimensions".format(sv.standard_name, dim_std) + ) + mod = entry.module_name + if mod not in uses: + uses[mod] = [] + if entry.local_name not in uses[mod]: + uses[mod].append(entry.local_name) + return uses + + +def _dim_local_expr(dim_std: str, suite_vars: Dict[str, SuiteVar], host_dict) -> str: + """Return the Fortran expression to use as one allocation dimension token. + + Suite-owned scalars: ``ccpp_suite_data(i)%`` (in the alloc loop + context where ``i`` is the instance index variable). Host-owned: just the + local name. The caller substitutes ``i`` for the instance index variable. + """ + if dim_std in suite_vars: + return 'ccpp_suite_data(i)%{}'.format(suite_vars[dim_std].local_name) + entry = host_dict.get(dim_std) if host_dict else None + if entry is None: + raise CCPPError( + "Cannot resolve suite-owned dimension '{}' from host or " + "suite vars".format(dim_std) + ) + return entry.local_name + + +def _collect_ddt_uses( + suite_vars: Dict[str, SuiteVar], + ddt_module_map: Optional[Dict[str, str]], +) -> Dict[str, List[str]]: + """Return ``{module_name: [ddt_type, ...]}`` for DDT types referenced + by suite-owned variables. + + A type is treated as a DDT when it is neither a Fortran intrinsic nor + an ``external:module:typename`` reference. The DDT type → module + mapping must be supplied by the caller (typically built via + :func:`metadata.variable_resolver.build_ddt_module_map`). + + Raises CCPPError if a DDT-typed suite variable references a type that + is missing from *ddt_module_map*. + """ + uses: Dict[str, List[str]] = {} + seen: set = set() + for sv in sorted(suite_vars.values(), key=lambda v: v.standard_name): + t = sv.type_.strip() + tlow = t.lower() + if tlow in _INTRINSICS or tlow.startswith('external:'): + continue + if tlow.startswith('type('): + t = t[t.index('(') + 1:t.rindex(')')].strip() + if t in seen: + continue + seen.add(t) + if ddt_module_map is None or t not in ddt_module_map: + raise CCPPError( + "Suite-owned variable '{}' has DDT type '{}' but its " + "defining Fortran module is unknown; the DDT must appear " + "in a metadata file alongside a scheme/host/control " + "table".format(sv.standard_name, t) + ) + mod = ddt_module_map[t] + uses.setdefault(mod, []) + if t not in uses[mod]: + uses[mod].append(t) + return uses + + +def _generate_suite_data( + suite_name: str, + suite_vars: Dict[str, SuiteVar], + host_dict=None, + ddt_module_map: Optional[Dict[str, str]] = None, +) -> List[str]: + """Generate the ``ccpp__data.F90`` module source lines. + + >>> lines = _generate_suite_data('mysuite', {}) + >>> 'module ccpp_mysuite_data' in lines + True + >>> any('ccpp_mysuite_data_t' in l for l in lines) + True + """ + mod_name = 'ccpp_{}_data'.format(suite_name) + type_name = 'ccpp_{}_data_t'.format(suite_name) + alloc_sub = 'ccpp_{}_suite_data_alloc'.format(suite_name) + dealloc_sub = 'ccpp_{}_suite_data_dealloc'.format(suite_name) + init_fields_sub = 'ccpp_{}_suite_data_init_fields'.format(suite_name) + final_fields_sub = 'ccpp_{}_suite_data_final_fields'.format(suite_name) + i1 = _INDENT + i2 = _INDENT * 2 + lines: List[str] = [] + + lines.append( + '! ccpp_{}_data.F90 -- generated by ccpp_capgen_ng, do not edit'.format( + suite_name + ) + ) + lines.append('module {}'.format(mod_name)) + lines.append('') + + # USE ccpp_kinds for any kind parameters referenced in suite-var + # declarations (e.g. ``real(kind=kind_phys)``). + kind_names = sorted({ + sv.kind for sv in suite_vars.values() + if sv.kind and not sv.kind.startswith('len=') + }) + if kind_names: + lines.append( + '{}use ccpp_kinds, only: {}'.format(i1, ', '.join(kind_names)) + ) + + # USE the defining module of each DDT type referenced by suite-owned + # variables so that ``type() :: `` declarations are valid. + ddt_uses = _collect_ddt_uses(suite_vars, ddt_module_map) + for mod in sorted(ddt_uses): + types = sorted(ddt_uses[mod]) + lines.append( + '{}use {}, only: {}'.format(i1, mod, ', '.join(types)) + ) + if kind_names or ddt_uses: + lines.append('') + + lines.append('{}implicit none'.format(i1)) + lines.append('{}private'.format(i1)) + lines.append('') + + # DDT type — allocatable fields use deferred-shape colons. + # + # ``TARGET`` is not a valid component attribute in Fortran; instead + # the module-level ``ccpp_suite_data(:)`` array below carries + # ``TARGET`` so that every ``ccpp_suite_data(i)%component(...)`` + # subobject is a valid pointer-assignment target. This is what + # the group cap needs to do ``ptr%ptr => ccpp_suite_data(i)%fld(...)`` + # for optional-arg passing and transformation temporaries. + lines.append('{}type, public :: {}'.format(i1, type_name)) + if suite_vars: + for sv in sorted(suite_vars.values(), key=lambda v: v.standard_name): + t = _type_str(sv.type_, sv.kind) + if sv.dimensions: + rank = len(sv.dimensions) + deferred = '({})'.format(','.join([':'] * rank)) + lines.append( + '{}{}, allocatable :: {}{}'.format( + i2, t, sv.local_name, deferred, + ) + ) + else: + lines.append('{}{} :: {}'.format(i2, t, sv.local_name)) + else: + lines.append('{}! (no suite-owned variables)'.format(i2)) + lines.append('{}end type {}'.format(i1, type_name)) + lines.append('') + + # Module-level allocatable instance array (one per model instance). + # ``TARGET`` makes every subobject (component access, array section, + # nested DDT field, ...) a valid pointer-assignment target. Without + # it Fortran rejects ``ptr => ccpp_suite_data(i)%fld(...)`` with + # "Pointer assignment target is neither TARGET nor POINTER". + lines.append( + '{}type({}), allocatable, target, public :: ccpp_suite_data(:)'.format( + i1, type_name, + ) + ) + + # Alloc/dealloc subroutines are only generated when host_dict is provided + # (the integration path). Unit tests that call without host_dict get the + # type definition and allocatable array but no subroutines. + if suite_vars and host_dict is not None: + lines.append('') + lines.append('{}public :: {}'.format(i1, alloc_sub)) + lines.append('{}public :: {}'.format(i1, dealloc_sub)) + lines.append('{}public :: {}'.format(i1, init_fields_sub)) + lines.append('{}public :: {}'.format(i1, final_fields_sub)) + lines.append('') + lines.append('contains') + + sorted_svs = sorted(suite_vars.values(), key=lambda v: v.standard_name) + + # ---- suite_data_alloc: only allocate the DDT array --------------- + # Inner allocatable fields are NOT allocated here because their + # dimensions may depend on suite-owned scalars set during the + # register phase. Inner allocations live in init_fields, called + # from _init after all _register calls have run. + lines.append('') + lines.append( + '{}subroutine {}(number_of_instances, errmsg, errflg)'.format(i1, alloc_sub) + ) + lines += [ + '', + '{}integer, intent(in) :: number_of_instances'.format(i2), + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + '{}if (allocated(ccpp_suite_data)) return'.format(i2), + '{}allocate(ccpp_suite_data(number_of_instances))'.format(i2), + '', + '{}end subroutine {}'.format(i1, alloc_sub), + ] + + # ---- suite_data_dealloc: only deallocate the DDT array ----------- + lines.append('') + lines.append( + '{}subroutine {}(errmsg, errflg)'.format(i1, dealloc_sub) + ) + lines += [ + '', + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + '{}if (.not. allocated(ccpp_suite_data)) return'.format(i2), + '{}deallocate(ccpp_suite_data)'.format(i2), + '', + '{}end subroutine {}'.format(i1, dealloc_sub), + ] + + # ---- suite_data_init_fields: allocate inner fields per instance -- + dim_uses = _collect_dim_uses(suite_vars, host_dict) + lines.append('') + lines.append( + '{}subroutine {}(i, errmsg, errflg)'.format(i1, init_fields_sub) + ) + for mod in sorted(dim_uses): + syms = ', '.join(sorted(dim_uses[mod])) + lines.append('{}use {}, only: {}'.format(i2, mod, syms)) + lines += [ + '', + '{}integer, intent(in) :: i'.format(i2), + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + ] + for sv in sorted_svs: + if sv.dimensions: + dim_exprs = [ + _dim_local_expr(d, suite_vars, host_dict) + for d in sv.dimensions + ] + lines.append( + '{}allocate(ccpp_suite_data(i)%{}({}))'.format( + i2, sv.local_name, ', '.join(dim_exprs) + ) + ) + lines += [ + '', + '{}end subroutine {}'.format(i1, init_fields_sub), + ] + + # ---- suite_data_final_fields: deallocate inner fields per inst --- + lines.append('') + lines.append( + '{}subroutine {}(i, errmsg, errflg)'.format(i1, final_fields_sub) + ) + lines += [ + '', + '{}integer, intent(in) :: i'.format(i2), + '{}character(len=*), intent(out) :: errmsg'.format(i2), + '{}integer, intent(out) :: errflg'.format(i2), + '', + "{}errmsg = ''".format(i2), + '{}errflg = 0'.format(i2), + ] + for sv in sorted_svs: + if sv.dimensions: + lines.append( + '{}if (allocated(ccpp_suite_data(i)%{})) ' + 'deallocate(ccpp_suite_data(i)%{})'.format( + i2, sv.local_name, sv.local_name + ) + ) + lines += [ + '', + '{}end subroutine {}'.format(i1, final_fields_sub), + ] + + lines.append('') + lines.append('end module {}'.format(mod_name)) + return lines + + +def write_suite_data( + suite_name: str, + suite_vars: Dict[str, SuiteVar], + output_root: str, + host_dict=None, + ddt_module_map: Optional[Dict[str, str]] = None, +) -> str: + """Write ``ccpp__data.F90`` to *output_root*.""" + os.makedirs(output_root, exist_ok=True) + filename = 'ccpp_{}_data.F90'.format(suite_name) + out_path = os.path.join(output_root, filename) + lines = _generate_suite_data(suite_name, suite_vars, host_dict, + ddt_module_map=ddt_module_map) + with open(out_path, 'w', encoding='utf-8') as fh: + fh.write('\n'.join(lines) + '\n') + return out_path + + +def _generate_suite_meta( + suite_name: str, + suite_vars: Dict[str, SuiteVar], +) -> List[str]: + """Generate metadata lines for ``ccpp_.meta``. + + The file documents all suite-owned variables in the standard ``.meta`` + format so that downstream tools can inspect what each suite provides. + + >>> lines = _generate_suite_meta('mysuite', {}) + >>> lines[0].startswith('!') + True + >>> any('ccpp_mysuite_data' in l for l in lines) + True + """ + mod_name = 'ccpp_{}_data'.format(suite_name) + i1 = _INDENT + lines: List[str] = [] + lines.append( + '! ccpp_{}.meta -- generated by ccpp_capgen_ng, do not edit'.format(suite_name) + ) + lines.append('[ccpp-table-properties]') + lines.append('{}name = {}'.format(i1, mod_name)) + lines.append('{}type = suite'.format(i1)) + lines.append('') + lines.append('[ccpp-arg-table]') + lines.append('{}name = {}'.format(i1, mod_name)) + lines.append('{}type = suite'.format(i1)) + for sv in sorted(suite_vars.values(), key=lambda v: v.standard_name): + lines.append('') + lines.append('[ {} ]'.format(sv.local_name)) + lines.append('{}standard_name = {}'.format(i1, sv.standard_name)) + lines.append('{}long_name = {}'.format(i1, sv.standard_name)) + lines.append('{}units = {}'.format(i1, sv.units)) + dim_str = '({})'.format(', '.join(sv.dimensions)) if sv.dimensions else '()' + lines.append('{}dimensions = {}'.format(i1, dim_str)) + lines.append('{}type = {}'.format(i1, sv.type_)) + if sv.kind: + lines.append('{}kind = {}'.format(i1, sv.kind)) + return lines + + +def write_suite_meta( + suite_name: str, + suite_vars: Dict[str, SuiteVar], + output_root: str, +) -> str: + """Write ``ccpp_.meta`` to *output_root* and return its path.""" + os.makedirs(output_root, exist_ok=True) + filename = 'ccpp_{}.meta'.format(suite_name) + out_path = os.path.join(output_root, filename) + lines = _generate_suite_meta(suite_name, suite_vars) + with open(out_path, 'w', encoding='utf-8') as fh: + fh.write('\n'.join(lines) + '\n') + return out_path diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py new file mode 100644 index 00000000..185eb854 --- /dev/null +++ b/capgen-ng/generator/suite_resolver.py @@ -0,0 +1,2039 @@ +#!/usr/bin/env python3 + +"""Variable matching and call-site resolution for the cap code generator. + +Resolves every scheme argument against the flat host/control dictionary built by +:func:`metadata.variable_resolver.build_flat_host_dict`, discovers suite-owned +(interstitial) variables, detects unit/kind transformations, and builds the +complete call-site information needed by :mod:`generator.group_cap`. + +Variable matching rules (Section 8.4 of the redesign spec) +----------------------------------------------------------- +For each standard name requested by a scheme argument: + +1. **Found in host/control dict** → direct reference; check units/kind for + transformation. +2. **Not found, first use is ``intent(out)``** → suite-owned variable; add to + suite data, generate declaration in ``ccpp__data.F90``. +3. **Not found, first use is ``intent(in)`` or ``intent(inout)``** → code + generation error: variable used before it is provided. +4. **Already in suite data (from a prior scheme)** → reference suite data path; + check units/kind for transformation. + +Dimension indexing rules (Section 9.2) +--------------------------------------- +Each entry in the ``dimensions`` list of a host/suite variable is either a +bare standard name (``'vertical_layer_dimension'``) or an explicit +``lower:upper`` range (``'ccpp_constant_one:horizontal_dimension'``, +``'bot_idx:vertical_interface_dimension'``). Bare names are normalised to +``ccpp_constant_one:`` before processing. + +After normalisation the upper-bound standard name drives dispatch: + +- ``instance_dimension`` / ``number_of_instances`` → ```` + (scalar extraction; the instance subscript is already in the access path for + DDT fields, but needed here for the DDT instance variable itself when it is + passed directly). +- ``horizontal_dimension`` / ``horizontal_loop_extent`` → + ``:`` (all phases). The lower bound must resolve to + ``1`` (i.e. be ``ccpp_constant_one`` or the integer literal ``1``). +- Everything else → ``:`` where both bounds are + resolved from *host_dict* or as integer literals. + +Transform cases (Section 10.3) +------------------------------- +Four cases, determined by ``optional`` and whether units/kind differ: + +====== ================ ============ +Case optional? transform? +====== ================ ============ +1 no no +2 yes no +3 no yes +4 yes yes +====== ================ ============ +""" + +import re +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Set, Tuple, Union + +from metadata.parse_tools import CCPPError, FORTRAN_CONDITIONAL_REGEX +from metadata.variable_resolver import HostVarEntry, _INSTANCE_DIMS + +# Dimension standard names that map to horizontal loop bounds. +_HORIZ_LOOP_DIMS: frozenset = frozenset({ + 'horizontal_dimension', + 'horizontal_loop_extent', +}) + +# Standard names for horizontal loop bounds and full horizontal dimension. +_HORIZ_BEGIN_STD = 'horizontal_loop_begin' +_HORIZ_END_STD = 'horizontal_loop_end' +_HORIZ_DIM_STD = 'horizontal_dimension' +_INSTANCE_NUM_STD = 'instance_number' + +# Vertical-dimension standard names (used by the vertical-flip transform +# when host and scheme metadata disagree on the ``top_at_one`` attribute). +_VDIM_STDS: frozenset = frozenset({ + 'vertical_layer_dimension', + 'vertical_interface_dimension', +}) + +# Physics scheme phases that operate on the per-call horizontal slice and +# therefore receive (ub - lb + 1) when a scheme asks for a scalar +# horizontal_dimension. Register is excluded: it runs at suite-cap level +# with the minimal framework signature (no loop bounds available). +_PHYSICS_PHASES: frozenset = frozenset({ + 'init', 'timestep_init', 'run', 'timestep_final', 'final', +}) + +# Framework constant whose Fortran value is always 1. Used as the implicit +# lower bound when a dimension string carries no explicit range, and is the +# only non-integer lower bound accepted for horizontal dimensions. +_CCPP_CONSTANT_ONE = 'ccpp_constant_one' + +# Type marker for register-phase constituent registration args. A scheme +# arg with this type, ``intent=out``, in the ``register`` phase is recognised +# as the per-scheme constituent array for the two-pass merge into the host's +# ``ccpp_model_constituents_object``. +_CONST_PROP_TYPE = 'ccpp_constituent_properties_t' + +# Standard name the host model uses to expose its ``ccpp_model_constituents_t`` +# object via the ``type=host`` table (opt-in, only required when at least +# one register-phase scheme produces constituents). +_CONST_OBJ_STDNAME = 'ccpp_model_constituents_object' + +# Framework-provided constituent standard names. The suite cap owns +# these symbols (allocates/binds them at init time); schemes reference +# them like any other variable and the resolver routes them to a +# synthetic source category ``'constituent'``. +_CONST_BASE_ARRAY_STD = 'ccpp_constituents' +_CONST_TEND_ARRAY_STD = 'ccpp_constituent_tendencies' +_CONST_PROPS_ARRAY_STD = 'ccpp_constituent_properties' +_CONST_NUM_STD = 'number_of_ccpp_constituents' +_TEND_PREFIX = 'tendency_of_' +_INDEX_PREFIX = 'index_of_' + +# Std names directly satisfied by host-constituents-module-owned symbols. +_FRAMEWORK_CONST_STDS = frozenset({ + _CONST_BASE_ARRAY_STD, + _CONST_TEND_ARRAY_STD, + _CONST_PROPS_ARRAY_STD, + _CONST_NUM_STD, +}) + +# Per-instance constituent object name in ccpp_host_constituents. Schemes +# access constituent state through ``(inst_num)%``. +_CONST_OBJ_VAR = 'ccpp_model_constituents_obj' + +# Mapping from framework-named std_name → DDT member. Used to translate +# scheme args declaring one of these framework names into the matching +# per-instance access expression. +_FRAMEWORK_NAME_TO_MEMBER = { + _CONST_BASE_ARRAY_STD: 'vars_layer', + _CONST_TEND_ARRAY_STD: 'vars_layer_tend', + _CONST_PROPS_ARRAY_STD: 'const_metadata', + _CONST_NUM_STD: 'num_layer_vars', +} + + +# Single host-wide module that owns the constituent object, the +# framework-shared pointers, the per-suite dynamic-constituent buffers, +# and the host-facing constituent API. All suite caps USE this module +# for their constituent symbol references. +_HOST_CONST_MOD = 'ccpp_host_constituents' + + +def _constituent_module_name(suite_name: str) -> str: + """Return the module name that owns the host-wide constituent state. + + Constant across suites: in capgen-ng (option A, matching original + capgen) the constituent object is host-wide, not suite-local. + """ + return _HOST_CONST_MOD + + +######################################################################## +# Unit conversion look-up +######################################################################## + +def _normalize_unit_string(unit: str) -> str: + """Canonicalise a unit string so that bare positive exponents carry an + explicit ``+`` sign. + + The CF / udunits conventions allow either ``m2`` or ``m+2`` to denote + "metres squared". The two forms are equivalent, but downstream code + treats unit strings as opaque tokens and compares them with ``==``. + Without normalisation, a host declaring ``m2 s-2`` and a scheme + declaring ``m+2 s-2`` would be flagged as a unit mismatch. + + Normalisation rule: a letter immediately followed by an unsigned + positive integer is rewritten as ``letter+integer``. Existing + ``letter+N`` and ``letter-N`` forms are left unchanged. + """ + return re.sub(r'([A-Za-z])(\d+)', r'\1+\2', unit) + + +def _unit_to_id(unit: str) -> str: + """Convert a unit string to the Python identifier fragment used in + :mod:`metadata.unit_conversion`. + + The input is first normalised by :func:`_normalize_unit_string` so + that bare and explicit positive exponents collapse to the same form. + + Rules (after normalisation): + + * Spaces → ``_`` + * ``letter-N`` → ``letter_minus_N`` + * ``letter+N`` → ``letter_plus_N`` + """ + result = _normalize_unit_string(unit).replace(' ', '_') + result = re.sub(r'([A-Za-z])([+])(\d+)', r'\1_plus_\3', result) + result = re.sub(r'([A-Za-z])(-)(\d+)', r'\1_minus_\3', result) + return result + + +def find_unit_conversion(from_unit: str, to_unit: str): + """Return the conversion formula callable, or ``None`` if unavailable. + + The formula callable takes no arguments and returns a format string + where ``{var}`` is the Fortran expression to convert and ``{kind}`` + is the kind suffix (``_kind_phys`` or ``''``). + + Input unit strings are normalised by :func:`_normalize_unit_string` + before the equality check and the function-name lookup, so equivalent + forms (``m2`` and ``m+2``) compare equal and resolve to the same + conversion entry. + """ + from_norm = _normalize_unit_string(from_unit) + to_norm = _normalize_unit_string(to_unit) + if from_norm == to_norm: + return None + from metadata import unit_conversion as _uc + fn_name = '{}__to__{}'.format(_unit_to_id(from_norm), _unit_to_id(to_norm)) + return getattr(_uc, fn_name, None) + + +def _apply_transform_formula(formula_fn, var_expr: str, kind: str) -> str: + """Apply a unit-conversion formula callable. + + Parameters + ---------- + formula_fn : callable + Returned by :func:`find_unit_conversion`. + var_expr : str + Fortran expression for the source variable. + kind : str + Kind parameter name (e.g. ``'kind_phys'``), or ``''``. + + Returns + ------- + str + Fortran expression for the converted value. + """ + kind_suffix = '_{}'.format(kind) if kind else '' + return formula_fn().format(kind=kind_suffix, var=var_expr) + + +######################################################################## +# Dimension subscript helpers +######################################################################## + +def _format_available_std_names( + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, + near: Optional[str] = None, +) -> str: + """Build a sorted listing of every standard name the resolver can see. + + Used in error messages when a lookup fails. Each line is + `` []`` where source is ``control``, + ``host: ``, or ``suite: ``. The final list + is sorted alphabetically (case-insensitive); when *near* is a + misspelled or mis-cased candidate, close matches surface first + under a separate "did you mean" header so the user spots the + typo quickly. + """ + rows: List[Tuple[str, str]] = [] + for std, entry in host_dict.items(): + if entry.is_control: + rows.append((std, 'control')) + elif entry.module_name: + rows.append((std, 'host: {}'.format(entry.module_name))) + else: + rows.append((std, 'host')) + if suite_vars: + for std, sv in suite_vars.items(): + rows.append((std, 'suite: {}'.format(sv.suite_module_name))) + rows.sort(key=lambda t: t[0]) + + if not rows: + return '\n (host_dict and suite_vars are both empty)' + + width = max(len(s) for s, _ in rows) + fmt = ' {{:<{}}} [{{}}]'.format(width) + + sections: List[str] = [] + if near: + import difflib + candidates = difflib.get_close_matches( + near, [s for s, _ in rows], n=5, cutoff=0.6, + ) + if not candidates: + # Try a case-insensitive direct hit (the most common cause: + # mixed-case in metadata vs lower-cased standard name). + low = near.lower() + candidates = [s for s, _ in rows if s == low] + if candidates: + sections.append('Did you mean (close matches to {!r}):'.format(near)) + sections.extend( + fmt.format(s, src) + for s, src in rows if s in candidates + ) + sections.append('') + + sections.append('Available standard names ({} entries):'.format(len(rows))) + sections.extend(fmt.format(s, src) for s, src in rows) + return '\n' + '\n'.join(sections) + + +def _resolve_single_bound( + bound: str, + host_dict: Dict[str, HostVarEntry], + used: Set[str], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, +) -> Optional[str]: + """Resolve one dimension bound token to a Fortran expression. + + Recognises, in order: + + 1. ``ccpp_constant_one`` — the framework constant equal to ``1``. + 2. Any integer literal — returned as a string unchanged. + 3. A standard name present in *host_dict* — returns the local Fortran name + and records the standard name in *used*. + 4. A standard name present in *suite_vars* — returns the suite data access + path (e.g. ``ccpp_suite_data(inst)%dim_inter``) and records the standard + name in *used*. Suite-owned scalars set during ``_register`` are read + here as dimension bounds in later phases. + + Returns ``None`` when the bound cannot be resolved. + """ + if bound == _CCPP_CONSTANT_ONE: + return '1' + try: + return str(int(bound)) + except ValueError: + pass + entry = host_dict.get(bound) + if entry is not None: + used.add(bound) + return entry.local_name + if suite_vars: + sv = suite_vars.get(bound) + if sv is not None: + used.add(bound) + return sv.access_path + return None + + +def _build_call_subscript( + dimensions: List[str], + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, + flip_vertical: bool = False, +) -> Tuple[str, Set[str]]: + """Build the Fortran subscript string for a variable's dimension list. + + Returns the subscript string (empty for scalars or ``'(s1, s2, ...)'`` for + arrays) and the set of dimension standard names that were resolved via + *host_dict* (needed for USE-statement generation). + + Parameters + ---------- + dimensions : list of str + Ordered dimension standard names from the host/suite variable entry. + phase : str + Current scheme phase (affects horizontal subscripting). + host_dict : dict + Flat host+control variable dictionary. + flip_vertical : bool + When ``True``, every vertical-dimension entry is emitted with + reverse stride (``::-1`` instead of ``:``). + Used by the vertical-flip transform when host and scheme disagree + on ``top_at_one``. + + Returns + ------- + tuple (subscript_str, used_std_names) + + Raises + ------ + CCPPError + If a dimension standard name cannot be resolved. + """ + if not dimensions: + return '', set() + + parts: List[str] = [] + used: Set[str] = set() + for dim in dimensions: + part, u = _one_dim_part(dim, phase, host_dict, suite_vars=suite_vars, + flip_vertical=flip_vertical) + parts.append(part) + used.update(u) + return '({})'.format(', '.join(parts)), used + + +def _one_dim_part( + dim: str, + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, + flip_vertical: bool = False, +) -> Tuple[str, Set[str]]: + """Return the Fortran subscript expression for one dimension entry. + + A dimension entry is either a bare standard name (``'vertical_layer_dimension'``) + or an explicit lower:upper range (``'ccpp_constant_one:horizontal_dimension'``, + ``'bot_idx:vertical_interface_dimension'``). Bare names are normalised + internally to ``ccpp_constant_one:`` before processing. + + Rules applied after normalisation: + + * Upper bound in :data:`_INSTANCE_DIMS` → scalar ``instance_number``. + * Upper bound in :data:`_HORIZ_LOOP_DIMS` → ``lb:ub`` (loop bounds). + Lower bound **must** resolve to ``'1'`` (i.e. be ``ccpp_constant_one`` + or the integer literal ``1``); any other value is an error. + * Everything else → resolve both bounds from *host_dict* or as integer + literals and return ``lower_expr:upper_expr``. + + When *flip_vertical* is True and the dimension is a vertical-axis + dimension (its upper bound is in :data:`_VDIM_STDS`), the bounds are + emitted in reverse-stride form ``::-1`` so the + array section reads (and writes) the vertical axis bottom-to-top + instead of top-to-bottom. This is how the host-side access expression + is rendered when host metadata declares ``top_at_one = .true.`` but + the scheme expects bottom-at-one (or vice versa). + + Returns ``(expr, used_std_names)`` where *expr* is the subscript token + and *used_std_names* is the set of standard names consumed from *host_dict*. + """ + used: Set[str] = set() + + # Normalise to range: bare name → ccpp_constant_one:name + if ':' not in dim: + lower_str = _CCPP_CONSTANT_ONE + upper_str = dim + else: + lower_str, upper_str = dim.split(':', 1) + lower_str = lower_str.strip() + upper_str = upper_str.strip() + + # Instance dimension: scalar subscript regardless of lower bound + if upper_str in _INSTANCE_DIMS: + inst_entry = host_dict.get(_INSTANCE_NUM_STD) + if inst_entry is None: + raise CCPPError( + "Host metadata references instance dimension '{}' but the " + "host's type=control table does not declare " + "'instance_number'. Declare 'instance_number' and " + "'number_of_instances' (paired) for a multi-instance API, " + "or remove the instance dimension from the affected " + "metadata for a single-instance host.".format(upper_str) + ) + used.add(_INSTANCE_NUM_STD) + return inst_entry.local_name, used + + # Horizontal dimension: validate lower, return loop bounds + if upper_str in _HORIZ_LOOP_DIMS: + lower_expr = _resolve_single_bound(lower_str, host_dict, set()) + if lower_expr != '1': + raise CCPPError( + "Lower bound '{}' for horizontal dimension '{}' must be " + "1 or ccpp_constant_one".format(lower_str, upper_str) + ) + lb = host_dict.get(_HORIZ_BEGIN_STD) + ub = host_dict.get(_HORIZ_END_STD) + if lb is None or ub is None: + raise CCPPError( + "Dimension '{}' requires '{}' and '{}' in the host " + "metadata but they were not found".format( + dim, _HORIZ_BEGIN_STD, _HORIZ_END_STD + ) + ) + used.update({_HORIZ_BEGIN_STD, _HORIZ_END_STD, upper_str}) + return '{}:{}'.format(lb.local_name, ub.local_name), used + + # General range: resolve both bounds independently + lower_expr = _resolve_single_bound(lower_str, host_dict, used, + suite_vars=suite_vars) + if lower_expr is None: + raise CCPPError( + "Dimension lower bound '{}' in '{}' is not in the " + "host metadata or suite-owned variables.{}".format( + lower_str, dim, + _format_available_std_names(host_dict, suite_vars, near=lower_str), + ) + ) + upper_expr = _resolve_single_bound(upper_str, host_dict, used, + suite_vars=suite_vars) + if upper_expr is None: + if upper_str.startswith('vertical_'): + raise CCPPError( + "Vertical dimension '{}' is not in the host metadata.{}".format( + upper_str, + _format_available_std_names(host_dict, suite_vars, near=upper_str), + ) + ) + raise CCPPError( + "Dimension '{}' is not in the host metadata or suite-owned " + "variables.{}".format( + dim, + _format_available_std_names(host_dict, suite_vars, near=upper_str), + ) + ) + if flip_vertical and upper_str in _VDIM_STDS: + # Reverse-stride form for the vertical axis (top_at_one mismatch). + return '{}:{}:-1'.format(upper_expr, lower_expr), used + return '{}:{}'.format(lower_expr, upper_expr), used + + +def _build_merged_subscript( + host_dims: List[str], + local_subscript: List[str], + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, + flip_vertical: bool = False, +) -> Tuple[str, Set[str]]: + """Build a call subscript merging a local-name template with host dimensions. + + Walk *local_subscript* tokens left to right: + + * ``':'`` — dimension placeholder: consume the next entry from *host_dims* + and emit the appropriate range expression (``lb:ub``, ``1:nlev``, etc.) + via :func:`_one_dim_part`. + * an integer literal — emitted verbatim. + * any other token — explicit index using a CCPP standard name: resolved + case-insensitively (standard names are case-insensitive) to the + corresponding local Fortran name from *host_dict* (or *suite_vars*). + The resolved standard name is added to the returned ``used`` set so the + group cap emits a ``use , only: `` for it. An + unresolved non-literal token is a metadata error: subscript indices in + a sliced ``local_name`` must be standard names with a defining source. + + When *local_subscript* is empty this is equivalent to calling + :func:`_build_call_subscript` directly. + + Example + ------- + local_subscript = [':', ':', 'index_of_water_vapor'] + host_dims = ['horizontal_dimension', 'vertical_layer_dimension'] + phase = 'run' + → '(lb:ub, 1:nlev, wv_idx)' + """ + if not local_subscript: + return _build_call_subscript(host_dims, phase, host_dict, + suite_vars=suite_vars, + flip_vertical=flip_vertical) + + dim_iter = iter(host_dims) + parts: List[str] = [] + used: Set[str] = set() + + for token in local_subscript: + token = token.strip() + if token == ':': + dim = next(dim_iter) + part, u = _one_dim_part(dim, phase, host_dict, + suite_vars=suite_vars, + flip_vertical=flip_vertical) + parts.append(part) + used.update(u) + elif token.isdigit(): + parts.append(token) + else: + key = token.lower() + entry = host_dict.get(key) + if entry is not None: + parts.append(entry.local_name) + used.add(key) + elif suite_vars and key in suite_vars: + parts.append(suite_vars[key].access_path) + used.add(key) + else: + raise CCPPError( + "Subscript index '{}' in a sliced local_name is not a " + "known CCPP standard name in the host or suite metadata; " + "subscript indices must be standard names with a " + "defining source so the cap can resolve and import " + "the corresponding local variable".format(token) + ) + + return '({})'.format(', '.join(parts)), used + + +def _dim_has_vertical(dim: str) -> bool: + """Return True if a dimension entry's upper bound is a vertical-axis + standard name. + + Accepts both the bare form (``'vertical_layer_dimension'``) and the + explicit ``lower:upper`` form (``'ccpp_constant_one:vertical_layer_dimension'``, + ``'bot_idx:vertical_interface_dimension'``). + """ + upper = dim.split(':', 1)[-1].strip() if ':' in dim else dim.strip() + return upper in _VDIM_STDS + + +def _substitute_instance_idx( + expr: str, host_dict: Dict[str, HostVarEntry], +) -> str: + """Resolve the DDT-instance template ``(instance_number)`` in an + access expression. + + :func:`metadata.variable_resolver._instance_subscript` bakes the + literal string ``(instance_number)`` into the access path of every + HostVarEntry derived from a DDT-instance array. That string is a + *standard-name placeholder*; at codegen time it must be substituted + with the host's actual Fortran local name for ``instance_number``. + When the host has not declared the instance pair (single-instance + API), substitute ``(1)`` so the access path is still well-formed + against length-1 internal arrays. + """ + if '(instance_number)' not in expr: + return expr + inst_entry = host_dict.get(_INSTANCE_NUM_STD) + if inst_entry is None: + return expr.replace('(instance_number)', '(1)') + return expr.replace( + '(instance_number)', '({})'.format(inst_entry.local_name) + ) + + +def _translate_active_expr(active: str, host_dict: Dict[str, HostVarEntry]) -> str: + """Translate standard names in an ``active`` expression to local Fortran. + + Standard-name identifiers are replaced with the host entry's full + Fortran access path (with any ``(instance_number)`` DDT-instance + template resolved to the host's actual local name). For free host + variables this collapses to ``entry.local_name``; for DDT-component + entries the substitution yields the fully qualified access path + (e.g. ``instance_data(instance)%opt_array_flag``). + """ + if not active: + return '' + + def _replace(m: re.Match) -> str: + word = m.group(0) + entry = host_dict.get(word) + if entry is None: + return word + return _substitute_instance_idx(entry.access_path, host_dict) + + return FORTRAN_CONDITIONAL_REGEX.sub(_replace, active) + + +def _root_symbol(access_path: str) -> str: + """Return the root Fortran symbol from an access path. + + This is the part before any ``%`` or ``(``, which is the name that + appears in the ``use module, only: `` statement. + """ + return re.split(r'[%(]', access_path)[0] + + +######################################################################## +# Data classes +######################################################################## + +@dataclass +class SuiteVar: + """A suite-owned variable discovered during variable resolution. + + Suite-owned variables are not provided by the host model; they are + first written by a scheme with ``intent(out)`` and then read by + subsequent schemes. They are declared in the generated + ``ccpp__data.F90`` module. + + Attributes + ---------- + standard_name : str + local_name : str + Scheme's local variable name that first produces this variable. + type_ : str + Fortran type string from the scheme metadata. + kind : str + Optional kind parameter. + units : str + dimensions : list of str + source_scheme : str + Name of the scheme that first declared it (intent out). + source_phase : str + """ + standard_name: str + local_name: str + type_: str + kind: str + units: str + dimensions: List[str] + source_scheme: str + source_phase: str + suite_module_name: str = '' + inst_access: str = '(1)' + allocatable: bool = False + + @property + def access_path(self) -> str: + """Fortran access expression in the suite data module.""" + return 'ccpp_suite_data{}%{}'.format(self.inst_access, self.local_name) + + @property + def module_name(self) -> str: + return self.suite_module_name if self.suite_module_name else 'ccpp_suite_data' + + +@dataclass +class ResolvedArg: + """One resolved argument at a scheme call site. + + Attributes + ---------- + standard_name : str + scheme_local_name : str + Keyword name for the Fortran call (from the scheme metadata ``[ name ]`` + header). + intent : str + ``'in'``, ``'out'``, or ``'inout'``. + is_optional : bool + active : str + Active condition in standard names (empty if always active). + active_local : str + Active condition translated to local Fortran names. + source : str + ``'host'``, ``'control'``, or ``'suite'``. + host_entry : HostVarEntry or None + The resolved host/control entry (``None`` for suite-owned vars + that have already been declared before this call). + suite_var : SuiteVar or None + The suite data entry (``None`` for host/control vars). + base_expr : str + Fortran access path (without dimension subscripts). + subscript : str + Dimension subscript string, e.g. ``'(lb:ub, 1:nlev)'`` or ``''``. + call_expr : str + Full call-site expression: ``base_expr + subscript``. + used_dim_std_names : set of str + Standard names of host/control/suite dimension variables + referenced in the subscript. Used to drive USE statements and + dummy-arg injection in the group cap. + used_const_dim_std_names : set of str + Standard names of *framework-constituent* dimension references + (notably ``number_of_ccpp_constituents``) referenced in the + subscript. These do not produce USE statements (the value is + reached via the per-instance constituent object), but they DO + appear in the host-facing introspection inputs list — original + capgen reports framework-constituent dim names there. + needs_unit_transform : bool + needs_kind_transform : bool + unit_forward : str + Fortran expression: host/suite → scheme (for pre-call, intent in/inout). + Empty if no transformation needed. + unit_backward : str + Fortran expression: scheme → host/suite (for post-call, intent out/inout). + Empty if no transformation needed. + kind_scheme : str + Kind declared in the scheme metadata. + kind_host : str + Kind of the host/suite variable. + temp_name : str + Name for the transformation temporary (``local_name + '_l'``). + ptr_name : str + Name for the optional pointer (``local_name + '_p'``). + transform_case : int + 1 = direct, 2 = pointer only, 3 = transform only, 4 = pointer+transform. + """ + standard_name: str + scheme_local_name: str + intent: str + is_optional: bool + active: str + active_local: str + source: str + host_entry: Optional[HostVarEntry] + suite_var: Optional['SuiteVar'] + base_expr: str + subscript: str + call_expr: str + used_dim_std_names: Set[str] + needs_unit_transform: bool + needs_kind_transform: bool + unit_forward: str + unit_backward: str + kind_scheme: str + kind_host: str + temp_name: str + ptr_name: str + transform_case: int + scheme_dimensions: List[str] + # ``needs_vert_flip`` is True when host and scheme metadata disagree on + # ``top_at_one``: the host-side access expression carries a reverse-stride + # subscript on the vertical axis and the transform pipeline copies through + # a temp local just like a unit conversion does. Composes with unit/kind + # transforms when present. + needs_vert_flip: bool = False + is_constituent_arg: bool = False + # ``is_constituent`` is True if the scheme metadata flagged this variable + # with any of ``constituent``, ``advected``, or ``molar_mass`` (a non-default + # value). Distinct from :attr:`is_constituent_arg`, which marks the + # ``ccpp_constituent_properties_t`` register-phase array argument. + is_constituent: bool = False + # For ``source == 'constituent'`` args: module that owns the constituent + # symbols this arg references (typically the suite cap module). ``None`` + # for non-constituent args. + constituent_module_name: Optional[str] = None + # Extra symbols (beyond :attr:`root_symbol`) that the group cap must USE + # from :attr:`constituent_module_name`. Typically ``index_of_`` + # integers referenced inside the constituent array subscript. + constituent_extra_symbols: Set[str] = field(default_factory=set) + # Framework-constituent dim std names referenced in the subscript + # (e.g. ``number_of_ccpp_constituents`` as the trailing axis of + # ``ccpp_constituents``). These are not USE'd from any module — the + # value is reached via the per-instance constituent object — but + # they're surfaced as inputs by the introspection routines in + # :mod:`generator.static_api`. Replaces the older trick of stuffing + # them into :attr:`used_dim_std_names`. + used_const_dim_std_names: Set[str] = field(default_factory=set) + + @property + def needs_transform(self) -> bool: + return (self.needs_unit_transform or self.needs_kind_transform + or self.needs_vert_flip) + + @property + def module_name(self) -> Optional[str]: + """Module to USE for this argument (``None`` for control vars).""" + if self.source == 'constituent': + return self.constituent_module_name + if self.host_entry is not None: + return self.host_entry.module_name + if self.suite_var is not None: + return self.suite_var.module_name + return None + + @property + def root_symbol(self) -> str: + """Root Fortran symbol name for the USE statement.""" + return _root_symbol(self.base_expr) + + +@dataclass +class ResolvedCall: + """All resolved arguments for one scheme phase call.""" + scheme_name: str + phase: str + args: List[ResolvedArg] = field(default_factory=list) + # Fortran module that exports the scheme's subroutines. Defaults to + # ``scheme_name`` for the common-case where the .meta table name and + # the Fortran module name match; overridden by the ``module_name`` + # attribute in ``[ccpp-table-properties]`` when the two differ. + scheme_module: str = '' + + @property + def used_modules(self) -> Dict[str, Set[str]]: + """Return ``{module_name: {symbol, ...}}`` for USE-statement building.""" + result: Dict[str, Set[str]] = {} + for arg in self.args: + mod = arg.module_name + if mod is not None: + sym = arg.root_symbol + result.setdefault(mod, set()).add(sym) + for dim_std in arg.used_dim_std_names: + pass # dim vars added separately by the group cap writer + return result + + +def _resolve_subcycle_loop_bound( + loop_str: Optional[str], + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, +) -> Tuple[str, str]: + """Resolve a subcycle ``loop=`` attribute into Fortran source. + + Returns ``(fortran_expr, std_name)`` where: + + * *fortran_expr* is the value to splice into ``do ccpp_loop_counter + = 1, ``. For an absent or literal-integer value + this is the literal itself; for a CCPP standard name it is the + host's (or suite's) Fortran local name. + * *std_name* is the resolved CCPP standard name (lower-cased) when + the loop bound was a symbol, otherwise the empty string. Used by + the group cap to (a) emit ``use , only: `` for + host-owned bounds and (b) inject the bound as a dummy argument + when it's a control variable. + + Raises ``CCPPError`` if the loop bound is a non-integer token that + doesn't resolve against the host/control dictionary or the suite's + interstitial variables. + """ + if loop_str is None: + return '1', '' + raw = loop_str.strip() + if not raw: + return '1', '' + # Integer literal — pass through verbatim (also handles negative + # numbers for completeness, though those aren't physically meaningful). + try: + int(raw) + return raw, '' + except ValueError: + pass + # Treat as a CCPP standard name. Standard names are lower-cased at + # parse time; the XML attribute may carry a mixed-case spelling, so + # normalise before lookup. + key = raw.lower() + entry = host_dict.get(key) + if entry is not None: + # Use the full access_path — for a free module variable this is + # just the local name, but for a DDT-component the access path is + # ``%`` (or + # ``(instance_number)%`` when the parent is + # in an instance-dimensioned array; resolve that template here). + return _substitute_instance_idx(entry.access_path, host_dict), key + if suite_vars and key in suite_vars: + sv = suite_vars[key] + return sv.access_path, key + raise CCPPError( + "Subcycle loop=\"{}\" is not an integer literal and does not " + "resolve to a CCPP standard name in the host/control metadata or " + "as a suite-owned variable; declare it (typically in the " + "type=control or type=host table) before using it as a subcycle " + "loop bound".format(loop_str) + ) + + +@dataclass +class ResolvedSubcycle: + """A subcycle ``do`` loop wrapping one or more scheme run calls. + + Only appears in ``phase_calls['run']``; non-run phases are always flat + (subcycle boundaries are not meaningful for init/final). + + Attributes + ---------- + loop : str + Loop-count Fortran expression to splice into ``do + ccpp_loop_counter = 1, ``. An integer literal when the + XML attribute was a literal; otherwise the host's local Fortran + name resolved from the CCPP standard name in the XML. + loop_std_name : str + The resolved CCPP standard name (lower-cased) when *loop* came + from a symbol; empty string when *loop* is an integer literal. + Drives USE-statement emission and control-variable dummy-arg + injection in the group cap. + calls : list of :data:`PhaseItem` + Items wrapped by this loop. Element types: :class:`ResolvedCall` + for scheme calls; :class:`ResolvedSubcycle` for nested loops + (yes, this is recursive — SDFs may declare arbitrary subcycle + nesting and the resolver preserves that structure for the cap + emitter to render as nested ``do`` loops). + """ + loop: str + # Use a forward-ref string for ``PhaseItem`` because the alias is + # defined just below this class. Runtime type-checking still works. + calls: List['PhaseItem'] = field(default_factory=list) + loop_std_name: str = '' + + +# Type alias for the contents of a phase's call list (and a subcycle's +# inner items). PhaseItem is itself the union of plain scheme calls and +# nested subcycles, so a phase / subcycle can carry arbitrary nesting. +PhaseItem = Union[ResolvedCall, ResolvedSubcycle] + + +def iter_phase_calls(items: List[PhaseItem]): + """Yield every :class:`ResolvedCall` in *items*, recursing into + nested :class:`ResolvedSubcycle` items. Subcycles themselves are + not yielded — only the leaf scheme calls.""" + for item in items: + if isinstance(item, ResolvedCall): + yield item + elif isinstance(item, ResolvedSubcycle): + # Recurse so nested ``ResolvedSubcycle`` items unwrap too. + yield from iter_phase_calls(item.calls) + + +def iter_phase_subcycles(items: List[PhaseItem]): + """Yield every :class:`ResolvedSubcycle` in *items*, including nested + subcycles. Used by the group cap to emit nested ``do`` loops and + by ``_collect_host_io`` to find every loop bound.""" + for item in items: + if isinstance(item, ResolvedSubcycle): + yield item + yield from iter_phase_subcycles(item.calls) + + +@dataclass +class ResolvedGroup: + """Resolution results for one suite group.""" + group_name: str + # One list of PhaseItem objects per phase. Non-run phases contain only + # ResolvedCall; the run phase may also contain ResolvedSubcycle items. + phase_calls: Dict[str, List[PhaseItem]] = field(default_factory=dict) + # Module → symbols referenced by dimension lookups in this group. + dim_uses: Dict[str, Set[str]] = field(default_factory=dict) + + +@dataclass +class SuiteResolution: + """Complete resolution result for one suite. + + Attributes + ---------- + constituent_register_calls : list of (scheme_name, scheme_local_name) + For each register-phase scheme arg whose ``type`` is + ``ccpp_constituent_properties_t`` (intent=out, allocatable), records + the (scheme, scheme arg local name) pair. The suite cap uses this + list to emit two-pass merge logic that populates the host's + ``ccpp_model_constituents_object`` with the per-scheme constituent + arrays. Empty when no register-phase scheme produces constituents. + constituent_index_names : list of str + Sorted list of base-constituent standard names that need an + ``index_of_`` integer emitted in the suite cap. Collected by + scanning every ``source='constituent'`` ResolvedArg's + ``constituent_extra_symbols`` for ``index_of_*`` tokens. Used by + :mod:`generator.suite_cap` to emit the index declarations and the + ``ccpp_model_constituents_object%const_index`` population calls + in ``_init``. + uses_constituents : bool + True iff any scheme arg in this suite has ``source='constituent'`` + (excluding the legacy register-phase ``is_constituent_arg``). + Drives suite-cap emission of the ccpp_constituents / + ccpp_constituent_tendencies pointers and related state. + suite_init_call : ResolvedCall or None + Resolved call for the suite-level ```` scheme (if any). + The scheme's ``init`` phase is invoked once per ``_init`` + call (per instance, per suite), after the group ``state_alloc`` + loop and before the state transition to + ``CCPP_SUITE_FRAMEWORK_INITIALIZED``. + suite_final_call : ResolvedCall or None + Resolved call for the suite-level ```` scheme (if any). + The scheme's ``final`` phase is invoked once per ``_final`` + call (per instance, per suite), before the state transition to + ``CCPP_SUITE_UNREGISTERED``. + """ + suite_name: str + groups: List[ResolvedGroup] = field(default_factory=list) + suite_vars: Dict[str, SuiteVar] = field(default_factory=dict) + uses_instance_dimension: bool = False + constituent_register_calls: List[Tuple[str, str]] = field(default_factory=list) + constituent_index_names: List[str] = field(default_factory=list) + uses_constituents: bool = False + suite_init_call: Optional[ResolvedCall] = None + suite_final_call: Optional[ResolvedCall] = None + + +######################################################################## +# Argument resolution helpers +######################################################################## + +def _local_name_conflict( + name: str, + existing_names: Set[str], +) -> str: + """Return *name* with a numeric suffix if it already exists in *existing_names*.""" + if name not in existing_names: + return name + # Split on last '_' to find the suffix ('_l' or '_p'). + if '_' in name: + base, suffix = name.rsplit('_', 1) + suffix = '_' + suffix + else: + base, suffix = name, '' + n = 2 + while True: + candidate = '{}_{}{}' .format(base, n, suffix) + if candidate not in existing_names: + return candidate + n += 1 + + +def _resolve_one_arg( + scheme_var, # MetaVar from scheme metadata + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Dict[str, SuiteVar], + scheme_name: str, + used_local_names: Set[str], + suite_name: str = '', +) -> ResolvedArg: + """Resolve one scheme argument against host/control/suite dictionaries. + + Implements the four variable-matching cases from Section 8.4. + + Parameters + ---------- + scheme_var : MetaVar + The variable entry from the scheme's phase metadata section. + phase : str + The current scheme phase (affects subscripting and error messages). + host_dict : dict + Flat host+control variable dict. + suite_vars : dict + Accumulated suite-owned variables (mutated if Case 2 applies). + scheme_name : str + Name of the enclosing scheme (for error messages). + used_local_names : set of str + Already-used local variable names in this group cap function (for + conflict resolution of temp/pointer names). + + Returns + ------- + ResolvedArg + + Raises + ------ + CCPPError + Case 3: variable not found and intent is not ``'out'``. + """ + std_name = scheme_var.standard_name + intent = scheme_var.intent or 'in' + local = scheme_var.local_name + optional = scheme_var.optional + + # ---- detect constituent register args (special-cased) --------------- + # Schemes that register dynamic constituents declare an intent=out + # ``ccpp_constituent_properties_t`` allocatable array. Those arguments + # are NOT promoted to suite-owned data — they are local temporaries in + # the suite cap's ``_register`` subroutine, used to count and + # populate the host's ``ccpp_model_constituents_object`` via the + # two-pass merge pattern. + is_constituent = ( + phase == 'register' + and intent == 'out' + and scheme_var.type.strip() == _CONST_PROP_TYPE + ) + if is_constituent: + return ResolvedArg( + standard_name=std_name, + scheme_local_name=local, + intent=intent, + is_optional=optional, + active='', + active_local='', + source='constituent', + host_entry=None, + suite_var=None, + base_expr='scheme_consts', + subscript='', + call_expr='scheme_consts', + used_dim_std_names=set(), + needs_unit_transform=False, + needs_kind_transform=False, + unit_forward='', + unit_backward='', + kind_scheme=scheme_var.kind, + kind_host='', + temp_name='', + ptr_name='', + transform_case=1, + scheme_dimensions=list(scheme_var.dimensions), + is_constituent_arg=True, + ) + + # ---- detect constituent-sourced scheme args (framework auto-provision) + # Returns a synthesised ``source='constituent'`` ResolvedArg for: + # * scheme args declaring a framework-known std name + # (ccpp_constituents, ccpp_constituent_tendencies, + # number_of_ccpp_constituents, ccpp_constituent_properties); + # * scheme args flagged ``is_constituent`` with intent=in/inout + # (routed to ccpp_constituents(, index_of_)); + # * scheme args flagged ``is_constituent`` with intent=out and + # standard name starting with ``tendency_of_`` (routed to + # ccpp_constituent_tendencies(, index_of_)). + # Returns ``None`` if the arg is not constituent-related. + const_arg = _resolve_constituent_arg( + scheme_var, phase, host_dict, suite_vars, scheme_name, suite_name, + ) + if const_arg is not None: + return const_arg + + # ---- determine source ----------------------------------------------- + host_entry: Optional[HostVarEntry] = host_dict.get(std_name) + + # active is a host-model-only attribute; read it from the host entry only. + active = host_entry.active if host_entry is not None else '' + sv: Optional[SuiteVar] = suite_vars.get(std_name) + + if host_entry is not None and sv is None: + source = 'control' if host_entry.is_control else 'host' + elif sv is not None and host_entry is None: + source = 'suite' + elif host_entry is None and sv is None: + # Case 2 or 3. + if intent == 'out': + inst_entry = host_dict.get('instance_number') + inst_access = '({})'.format(inst_entry.local_name) if inst_entry else '(1)' + sv = SuiteVar( + standard_name=std_name, + local_name=local, + type_=scheme_var.type, + kind=scheme_var.kind, + units=scheme_var.units, + dimensions=list(scheme_var.dimensions), + source_scheme=scheme_name, + source_phase=phase, + suite_module_name='ccpp_{}_data'.format(suite_name), + inst_access=inst_access, + allocatable=scheme_var.allocatable, + ) + suite_vars[std_name] = sv + source = 'suite' + else: + raise CCPPError( + "Variable '{}' (standard_name='{}') requested by scheme " + "'{}' phase '{}' with intent({}) is not provided by the host " + "metadata or by any prior scheme; " + "either add it to the host metadata or ensure an earlier " + "scheme provides it with intent(out)".format( + local, std_name, scheme_name, phase, intent + ) + ) + else: + # Both found — host takes precedence (suite data shouldn't duplicate host). + source = 'control' if host_entry.is_control else 'host' + + # ---- build access expression ----------------------------------------- + if host_entry is not None: + # ``host_entry.access_path`` is the verbatim form from + # build_flat_host_dict; for DDT-instance arrays it carries the + # ``(instance_number)`` template that needs codegen-time resolution. + base_expr = _substitute_instance_idx(host_entry.access_path, host_dict) + host_dims = host_entry.dimensions + host_units = host_entry.units + host_kind = host_entry.kind + host_allocatable = host_entry.allocatable + else: + base_expr = sv.access_path + host_dims = sv.dimensions + host_units = sv.units + host_kind = sv.kind + host_allocatable = sv.allocatable + + # ---- allocatable compatibility check --------------------------------- + # An actual argument that is not allocatable cannot be passed to an + # allocatable dummy. The reverse direction (allocatable host -> plain + # assumed-shape dummy) is legal Fortran and is permitted: the scheme + # simply forgoes access to the allocation status. + if scheme_var.allocatable and not host_allocatable: + raise CCPPError( + "Variable '{}' (standard_name='{}'): scheme '{}' declares " + "allocatable=True but {} declares allocatable=False; " + "an allocatable dummy cannot receive a non-allocatable actual " + "argument".format( + local, std_name, scheme_name, source + ) + ) + + # ---- vertical-flip detection (top_at_one mismatch) ------------------ + # Both host and scheme declare a top_at_one attribute (default False). + # A mismatch triggers a reverse-stride substitution on the host-side + # subscript at the vertical-dim position so the array section is read + # (and written) in flipped order. Only meaningful when the variable + # actually has a vertical dimension. + if host_entry is not None: + host_top_at_one = host_entry.top_at_one + else: + host_top_at_one = False + scheme_top_at_one = bool(getattr(scheme_var, 'top_at_one', False)) + needs_vert_flip = ( + host_top_at_one != scheme_top_at_one + and any(_dim_has_vertical(d) for d in host_dims) + ) + + if host_allocatable: + # Allocatable actual arguments must omit explicit dimension ranges: + # the callee declares the dummy as allocatable too and assumes the + # array shape from the actual. + subscript: str = '' + used_dim_std: Set[str] = set() + else: + local_sub = host_entry.local_subscript if host_entry is not None else [] + subscript, used_dim_std = _build_merged_subscript( + host_dims, local_sub, phase, host_dict, suite_vars=suite_vars, + flip_vertical=needs_vert_flip, + ) + call_expr = base_expr + subscript + + # Scalar horizontal_dimension in a physics phase: the scheme is asking + # for the size of the horizontal slice it actually receives. During run + # the host passes a chunk (lb:ub); during the other physics phases the + # loop bounds collapse to 1:ncols. In both cases (ub - lb + 1) yields + # the correct extent, so we synthesise it from the loop-bound control + # variables and bypass the host's full-domain scalar (e.g. ncols). + if (phase in _PHYSICS_PHASES + and std_name == _HORIZ_DIM_STD + and not scheme_var.dimensions): + lb = host_dict.get(_HORIZ_BEGIN_STD) + ub = host_dict.get(_HORIZ_END_STD) + if lb is None or ub is None: + raise CCPPError( + "Scheme '{}' phase '{}' requests scalar '{}' but the host " + "metadata lacks '{}'/'{}' (required to compute the per-call " + "horizontal extent)".format( + scheme_name, phase, _HORIZ_DIM_STD, + _HORIZ_BEGIN_STD, _HORIZ_END_STD + ) + ) + call_expr = '({} - {} + 1)'.format(ub.local_name, lb.local_name) + used_dim_std.update({_HORIZ_BEGIN_STD, _HORIZ_END_STD}) + + # ---- active expression translation ----------------------------------- + active_local = _translate_active_expr(active, host_dict) + + # ---- transformation detection ---------------------------------------- + # Normalise both unit strings so that equivalent spellings (``m2`` and + # ``m+2``) compare equal and do not appear as bogus mismatches. + host_units = _normalize_unit_string(host_units) + scheme_units = _normalize_unit_string(scheme_var.units) + scheme_kind = scheme_var.kind + + fwd_fn = find_unit_conversion(host_units, scheme_units) if host_units != scheme_units else None + bwd_fn = find_unit_conversion(scheme_units, host_units) if host_units != scheme_units else None + + needs_unit = fwd_fn is not None or bwd_fn is not None + # Unit mismatch with no known conversion is an error only if units differ. + if host_units != scheme_units and not needs_unit: + raise CCPPError( + "Variable '{}' (standard_name='{}'): host units '{}' differ from " + "scheme '{}' units '{}' but no unit conversion is known; " + "add a conversion to metadata/unit_conversion.py or fix the " + "metadata".format( + local, std_name, host_units, scheme_name, scheme_units + ) + ) + + # Character kind handling (len=N / len=*): + # - len=* in the scheme is always compatible with any host len=. + # - Matching specific len=N values need no transform (naturally equal). + # - Mismatched specific lengths (len=N vs len=M) are a metadata error; + # the scheme must declare len=* or match the defining metadata exactly. + _host_is_len = host_kind.startswith('len=') if host_kind else False + _scheme_is_len = scheme_kind.startswith('len=') if scheme_kind else False + if _host_is_len or _scheme_is_len: + if scheme_kind != 'len=*' and host_kind != scheme_kind: + raise CCPPError( + "Character variable '{}' (standard_name='{}'): host declares " + "kind='{}' but scheme '{}' declares kind='{}'; scheme must " + "use kind=len=* or match the defining kind exactly".format( + local, std_name, host_kind, scheme_name, scheme_kind + ) + ) + needs_kind = False + else: + needs_kind = bool(host_kind) and bool(scheme_kind) and host_kind != scheme_kind + + # Forward transformation expression (host/suite → scheme local). + # ``call_expr`` already carries the flipped vertical subscript when + # ``needs_vert_flip`` is True, so the unit-conversion formula naturally + # composes the flip on the host-side RHS. For a pure-flip case (no + # unit conversion) we emit a plain copy ``temp = host(...flipped)``. + unit_forward = '' + if needs_unit and fwd_fn is not None and intent in ('in', 'inout'): + unit_forward = _apply_transform_formula(fwd_fn, call_expr, scheme_kind) + elif needs_vert_flip and not needs_unit and intent in ('in', 'inout'): + unit_forward = call_expr + + # Backward transformation expression (scheme local → host/suite). + unit_backward = '' + if needs_unit and bwd_fn is not None and intent in ('out', 'inout'): + unit_backward_expr = '{}_l'.format(local) + unit_backward = _apply_transform_formula(bwd_fn, unit_backward_expr, host_kind) + elif needs_vert_flip and not needs_unit and intent in ('out', 'inout'): + unit_backward = '{}_l'.format(local) + + needs_transform = needs_unit or needs_kind or needs_vert_flip + + # ---- local variable names (transformation temp + pointer) ------------ + temp_name = '' + ptr_name = '' + if needs_transform: + candidate = '{}_l'.format(local) + temp_name = _local_name_conflict(candidate, used_local_names) + used_local_names.add(temp_name) + + if optional: + candidate = '{}_p'.format(local) + ptr_name = _local_name_conflict(candidate, used_local_names) + used_local_names.add(ptr_name) + + # ---- transform case -------------------------------------------------- + if optional and needs_transform: + transform_case = 4 + elif optional: + transform_case = 2 + elif needs_transform: + transform_case = 3 + else: + transform_case = 1 + + return ResolvedArg( + standard_name=std_name, + scheme_local_name=local, + intent=intent, + is_optional=optional, + active=active, + active_local=active_local, + source=source, + host_entry=host_entry, + suite_var=sv if source == 'suite' else None, + base_expr=base_expr, + subscript=subscript, + call_expr=call_expr, + used_dim_std_names=used_dim_std, + needs_unit_transform=needs_unit, + needs_kind_transform=needs_kind, + unit_forward=unit_forward, + unit_backward=unit_backward, + kind_scheme=scheme_kind, + kind_host=host_kind, + temp_name=temp_name, + ptr_name=ptr_name, + transform_case=transform_case, + scheme_dimensions=list(scheme_var.dimensions), + needs_vert_flip=needs_vert_flip, + is_constituent=scheme_var.is_constituent, + ) + + +######################################################################## +# Constituent-source synthesis (framework auto-provisioning) +######################################################################## + +def _const_dim_part( + dim: str, + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, +) -> Tuple[str, Set[str], Set[str], Set[str]]: + """One-dim subscript with framework-constituent dim recognition. + + Returns ``(part, used_host_std, used_const_std, used_const_dim_std)``. + + The trailing dim ``number_of_ccpp_constituents`` is emitted as + ``':'`` (whole-axis slice). The std name is added to + ``used_const_dim_std`` so the introspection routine + (:func:`generator.static_api._collect_host_io`) can include it in + its inputs list — original capgen reports framework-constituent dim + names there. No USE statement is emitted for the name: it isn't in + host_dict (the framework provides it via the per-instance + constituent object), so ``_collect_group_uses`` and + ``_extra_dim_ctrl_entries`` both silently skip it. All other dims + fall through to :func:`_one_dim_part` and their std names go into + ``used_host_std``. + + ``used_const_std`` collects framework-constituent *symbols* that + need a USE statement (currently unused at this layer; reserved for + future framework-constituent symbols that might appear inside a + dim expression). + """ + if dim == _CONST_NUM_STD or dim.endswith(':' + _CONST_NUM_STD): + return ':', set(), set(), {_CONST_NUM_STD} + part, used = _one_dim_part(dim, phase, host_dict, suite_vars=suite_vars) + return part, used, set(), set() + + +def _build_const_subscript( + dimensions: List[str], + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, +) -> Tuple[str, Set[str], Set[str], Set[str]]: + """Build a subscript for a constituent-sourced arg. + + Like :func:`_build_call_subscript` but recognises + ``number_of_ccpp_constituents`` as a whole-axis slice. Returns + ``(subscript, used_host_std, used_const_std, used_const_dim_std)``; + *used_const_std* collects framework-constituent symbols that need + a USE statement (reserved for future use), and + *used_const_dim_std* collects framework-constituent dim std names + (e.g. ``number_of_ccpp_constituents``) for introspection. + """ + if not dimensions: + return '', set(), set(), set() + parts: List[str] = [] + used_host: Set[str] = set() + used_const: Set[str] = set() + used_const_dim: Set[str] = set() + for dim in dimensions: + part, uh, uc, ucd = _const_dim_part(dim, phase, host_dict, suite_vars) + parts.append(part) + used_host.update(uh) + used_const.update(uc) + used_const_dim.update(ucd) + return ('({})'.format(', '.join(parts)), + used_host, used_const, used_const_dim) + + +def _resolve_constituent_arg( + scheme_var, + phase: str, + host_dict: Dict[str, HostVarEntry], + suite_vars: Dict[str, 'SuiteVar'], + scheme_name: str, + suite_name: str, +) -> Optional[ResolvedArg]: + """Synthesise a ``source='constituent'`` ResolvedArg, or return ``None``. + + Per-instance access pattern: every constituent state lookup goes + through ``ccpp_model_constituents_obj()%`` where + *inst_num* is the host's local name for ``instance_number`` (or + ``1`` if the host doesn't declare it). The ``index_of_`` + integers and ``ccpp_model_const_stdnames`` parameter array are + module-level scalars on ``ccpp_host_constituents`` (identical + across instances). + + Host metadata always wins: if the host declares ``std_name`` as a + regular variable, this routine returns ``None`` and normal host-arg + resolution takes over. Constituent auto-provisioning is reserved + for names the host has not claimed. + + Three argument categories are recognised: + + 1. **Framework-named std_name** — one of + :data:`_FRAMEWORK_CONST_STDS` or starts with ``index_of_`` *and* + not declared by the host. + + * ``ccpp_constituents`` → ``ccpp_model_constituents_obj(inst)%vars_layer`` + * ``ccpp_constituent_tendencies`` → ``...%vars_layer_tend`` + * ``ccpp_constituent_properties`` → ``...%const_metadata`` + * ``number_of_ccpp_constituents`` → ``...%num_layer_vars`` (scalar) + * ``index_of_`` → ``index_of_`` (module-level integer) + + 2. **Base constituent** — ``scheme_var.is_constituent`` true, intent + in/inout, std_name not a ``tendency_of_*``. Routed to + ``...%vars_layer(, index_of_)``. + + 3. **Constituent tendency** — ``scheme_var.is_constituent`` true, + intent=out, std_name=``tendency_of_``. Routed to + ``...%vars_layer_tend(, index_of_)``. + + Mismatched combinations are hard errors (see error messages below). + The constituent arg always carries ``instance_number`` in + ``used_dim_std_names`` (when the host declares it) so the group cap + auto-injects it as a dummy via :func:`_extra_dim_ctrl_entries`. + """ + std_name = scheme_var.standard_name + intent = scheme_var.intent or 'in' + local = scheme_var.local_name + optional = scheme_var.optional + scheme_dims = list(scheme_var.dimensions) + + is_tendency_name = std_name.startswith(_TEND_PREFIX) + is_index_name = std_name.startswith(_INDEX_PREFIX) + is_framework_name = std_name in _FRAMEWORK_CONST_STDS or is_index_name + + # Host metadata wins: if the host declares this std_name as a regular + # variable (e.g. a `protected integer` named `ntcw` with + # standard_name = index_of_..._tracer_concentration_array), defer to + # normal host-arg resolution so the scheme call uses the host's short + # local name. Constituent auto-provisioning is reserved for + # framework-named std_names the host has not claimed. + if is_framework_name and host_dict and std_name in host_dict: + return None + + constituent_module = _constituent_module_name(suite_name) + inst_entry = host_dict.get(_INSTANCE_NUM_STD) if host_dict else None + inst_local = inst_entry.local_name if inst_entry else None + inst_idx = inst_local if inst_local else '1' + + def _common_kwargs(base_expr, subscript, call_expr, + used_host_std, extra_symbols, + used_const_dim_std=None): + used_host_std = set(used_host_std) + if inst_local: + used_host_std.add(_INSTANCE_NUM_STD) + return dict( + standard_name=std_name, + scheme_local_name=local, + intent=intent, + is_optional=optional, + active='', + active_local='', + source='constituent', + host_entry=None, + suite_var=None, + base_expr=base_expr, + subscript=subscript, + call_expr=call_expr, + used_dim_std_names=used_host_std, + needs_unit_transform=False, + needs_kind_transform=False, + unit_forward='', + unit_backward='', + kind_scheme=scheme_var.kind, + kind_host='', + temp_name='', + ptr_name='', + transform_case=1, + scheme_dimensions=scheme_dims, + is_constituent=scheme_var.is_constituent, + constituent_module_name=constituent_module, + constituent_extra_symbols=extra_symbols, + used_const_dim_std_names=(set(used_const_dim_std) + if used_const_dim_std else set()), + ) + + # ---- Path 1a: index_of_ — module-level integer, no per-instance -- + if is_index_name: + return ResolvedArg(**_common_kwargs( + base_expr=std_name, subscript='', call_expr=std_name, + used_host_std=set(), extra_symbols={std_name}, + )) + + # ---- Path 1b: framework-named std_name → DDT member ----------------- + if is_framework_name: + member = _FRAMEWORK_NAME_TO_MEMBER[std_name] + subscript, used_host_std, used_const_std, used_const_dim_std = \ + _build_const_subscript( + scheme_dims, phase, host_dict, suite_vars, + ) + base_expr = '{}({})%{}'.format(_CONST_OBJ_VAR, inst_idx, member) + call_expr = base_expr + subscript if subscript else base_expr + # Framework-constituent dim refs (e.g. number_of_ccpp_constituents) + # travel on the dedicated used_const_dim_std_names channel — no + # USE statement, but surfaced as inputs by the introspection + # routines in generator.static_api. + return ResolvedArg(**_common_kwargs( + base_expr=base_expr, subscript=subscript, call_expr=call_expr, + used_host_std=used_host_std, + extra_symbols={_CONST_OBJ_VAR} | used_const_std, + used_const_dim_std=used_const_dim_std, + )) + + if not scheme_var.is_constituent: + return None # not constituent-related + + # ---- Paths 2/3: is_constituent base or tendency --------------------- + if intent == 'out': + if not is_tendency_name: + raise CCPPError( + "Constituent-flagged scheme arg '{}' (standard_name='{}', " + "scheme='{}', phase='{}') has intent=out but its standard " + "name does not start with 'tendency_of_'. Physics phases " + "may only produce constituent tendencies; new base " + "constituents must be declared via a " + "ccpp_constituent_properties_t argument in a register-phase " + "scheme.".format(local, std_name, scheme_name, phase) + ) + base_std = std_name[len(_TEND_PREFIX):] + member = 'vars_layer_tend' + else: # in / inout + if is_tendency_name: + raise CCPPError( + "Constituent tendency arg '{}' (standard_name='{}', " + "scheme='{}', phase='{}') must be declared with intent=out; " + "physics phases only produce tendencies, never consume " + "them.".format(local, std_name, scheme_name, phase) + ) + base_std = std_name + member = 'vars_layer' + + leading_sub, used_host_std = _build_call_subscript( + scheme_dims, phase, host_dict, suite_vars=suite_vars, + ) + index_sym = '{}{}'.format(_INDEX_PREFIX, base_std) + if leading_sub: + subscript = leading_sub[:-1] + ', ' + index_sym + ')' + else: + subscript = '(' + index_sym + ')' + base_expr = '{}({})%{}'.format(_CONST_OBJ_VAR, inst_idx, member) + call_expr = base_expr + subscript + + return ResolvedArg(**_common_kwargs( + base_expr=base_expr, subscript=subscript, call_expr=call_expr, + used_host_std=used_host_std, + extra_symbols={index_sym, _CONST_OBJ_VAR}, + )) + + +######################################################################## +# Suite resolution +######################################################################## + +def resolve_suite( + suite, # generator.suite_xml.Suite + scheme_store, # metadata.variable_resolver.SchemeStore + host_dict: Dict[str, HostVarEntry], + phases: Optional[List[str]] = None, +) -> SuiteResolution: + """Resolve all scheme arguments for every group and phase in *suite*. + + Parameters + ---------- + suite : Suite + Parsed suite XML object. + scheme_store : SchemeStore + Scheme metadata organised for lookup. + host_dict : dict + Flat host+control variable dictionary. + phases : list of str, optional + Phases to resolve. Defaults to all six phases in chronological order + (register first), so that suite-owned variables produced by + ``_register`` are visible as dimensions or as ``intent(in)`` reads + in subsequent phases. + + Returns + ------- + SuiteResolution + + Raises + ------ + CCPPError + On any variable matching failure. + """ + if phases is None: + phases = ['register', 'init', 'timestep_init', 'run', + 'timestep_final', 'final'] + + # Detect whether any host variable uses an instance dimension. + uses_instance = any( + any(d in _INSTANCE_DIMS for d in entry.dimensions) + for entry in host_dict.values() + ) + + suite_vars: Dict[str, SuiteVar] = {} + resolved_groups: List[ResolvedGroup] = [] + + for group in suite.groups: + rg = ResolvedGroup(group_name=group.name) + + for phase in phases: + used_local_names_phase: Set[str] = set() + + if phase == 'run': + # Preserve subcycle structure for run-phase loop generation. + items_for_phase = _resolve_run_phase( + group, phase, scheme_store, host_dict, suite_vars, + used_local_names_phase, + suite_name=suite.name, + ) + else: + # Non-run phases: flatten all subcycles and silently + # deduplicate scheme names within the group. A scheme that + # appears multiple times in the suite XML (typically because + # it runs once per constituent in the ``run`` phase) must + # still have its register/init/finalize entry points invoked + # exactly once per group — matches ``design_init_dedup.md``. + scheme_names_flat = _dedup_scheme_names( + _collect_scheme_names(group) + ) + items_for_phase = _resolve_flat_phase( + scheme_names_flat, phase, scheme_store, host_dict, + suite_vars, used_local_names_phase, + suite_name=suite.name, + ) + + if items_for_phase: + rg.phase_calls[phase] = items_for_phase + + # Collect dimension variable USE info for this group. + rg.dim_uses = _collect_dim_uses(rg, host_dict, suite_vars=suite_vars) + resolved_groups.append(rg) + + # Constituent register calls: gather the (scheme_name, scheme_local_name) + # pairs for every register-phase arg that was flagged as a constituent. + # The suite cap uses these to emit two-pass merge logic. + constituent_calls: List[Tuple[str, str]] = [] + for rg in resolved_groups: + for rc in iter_phase_calls(rg.phase_calls.get('register', [])): + for arg in rc.args: + if arg.is_constituent_arg: + constituent_calls.append( + (rc.scheme_name, arg.scheme_local_name) + ) + # Walk every constituent-sourced arg (excluding the legacy + # register-phase ccpp_constituent_properties_t case) and collect: + # * uses_constituents — whether any constituent state is referenced + # * constituent_index_names — base std names X needing an index_of_X + uses_constituents = False + index_names: Set[str] = set() + for rg in resolved_groups: + for items in rg.phase_calls.values(): + for rc in iter_phase_calls(items): + for arg in rc.args: + if arg.source != 'constituent' or arg.is_constituent_arg: + continue + uses_constituents = True + for sym in arg.constituent_extra_symbols: + if sym.startswith(_INDEX_PREFIX): + index_names.add(sym[len(_INDEX_PREFIX):]) + constituent_index_names = sorted(index_names) + + # Under option A the constituent object is generator-owned (lives in + # the ccpp_host_constituents module), so the host is no longer + # required to declare ``ccpp_model_constituents_object`` in its + # type=host metadata. No validation is needed here. + + # ---- suite-level / schemes ------------------------------ + # SDF v2.0 schema accepts an optional single ```` and ```` + # scheme name at the suite root. Resolve each to a ResolvedCall against + # the scheme's ``init`` / ``final`` phase metadata respectively. The + # local-name dedup set is fresh per call (these calls live outside any + # group and don't share locals with group phases). + suite_init_call: Optional[ResolvedCall] = None + suite_final_call: Optional[ResolvedCall] = None + if suite.init_scheme: + suite_init_locals: Set[str] = set() + suite_init_call = _resolve_one_call( + suite.init_scheme, 'init', scheme_store, host_dict, + suite_vars, suite_init_locals, suite_name=suite.name, + ) + if suite_init_call is None: + raise CCPPError( + "Suite '{}' declares {} but scheme '{}' " + "has no ``init`` phase in its metadata.".format( + suite.name, suite.init_scheme, suite.init_scheme, + ) + ) + if suite.final_scheme: + suite_final_locals: Set[str] = set() + suite_final_call = _resolve_one_call( + suite.final_scheme, 'final', scheme_store, host_dict, + suite_vars, suite_final_locals, suite_name=suite.name, + ) + if suite_final_call is None: + raise CCPPError( + "Suite '{}' declares {} but scheme '{}' " + "has no ``final`` phase in its metadata.".format( + suite.name, suite.final_scheme, suite.final_scheme, + ) + ) + + return SuiteResolution( + suite_name=suite.name, + groups=resolved_groups, + suite_vars=suite_vars, + constituent_register_calls=constituent_calls, + uses_instance_dimension=uses_instance, + constituent_index_names=constituent_index_names, + uses_constituents=uses_constituents, + suite_init_call=suite_init_call, + suite_final_call=suite_final_call, + ) + + +def _collect_scheme_names(group) -> List[str]: + """Return ordered list of scheme names from a group, expanding subcycles/subcols.""" + names: List[str] = [] + from generator.suite_xml import SuiteScheme, SuiteSubcycle, SuiteSubcol + for item in group.items: + if isinstance(item, SuiteScheme): + names.append(item.name) + elif isinstance(item, (SuiteSubcycle, SuiteSubcol)): + for sn in item.scheme_names(): + names.append(sn) + return names + + +def _dedup_scheme_names(scheme_names: List[str]) -> List[str]: + """Return *scheme_names* with duplicates removed (first occurrence kept). + + Used by :func:`resolve_suite` for non-run phases: a scheme that appears + more than once in the suite XML still has its register/init/finalize + entry points invoked exactly once per group, while preserving the order + of first appearance. + """ + seen: Set[str] = set() + deduped: List[str] = [] + for sn in scheme_names: + if sn in seen: + continue + seen.add(sn) + deduped.append(sn) + return deduped + + +def _resolve_one_call( + scheme_name: str, + phase: str, + scheme_store, + host_dict: Dict[str, HostVarEntry], + suite_vars: Dict[str, 'SuiteVar'], + used_local_names: Set[str], + suite_name: str = '', +) -> Optional[ResolvedCall]: + """Build a ResolvedCall for one scheme/phase, or return None if not defined.""" + vars_list = scheme_store.variables_for(scheme_name, phase) + if vars_list is None: + return None + rc = ResolvedCall( + scheme_name=scheme_name, phase=phase, + scheme_module=scheme_store.module_for(scheme_name), + ) + for sv in vars_list: + arg = _resolve_one_arg( + sv, phase, host_dict, suite_vars, scheme_name, used_local_names, + suite_name=suite_name, + ) + rc.args.append(arg) + return rc + + +def _resolve_flat_phase( + scheme_names: List[str], + phase: str, + scheme_store, + host_dict: Dict[str, HostVarEntry], + suite_vars: Dict[str, 'SuiteVar'], + used_local_names: Set[str], + suite_name: str = '', +) -> List[ResolvedCall]: + """Resolve a flat (non-subcycle) phase into a list of ResolvedCall.""" + result: List[ResolvedCall] = [] + for sn in scheme_names: + rc = _resolve_one_call(sn, phase, scheme_store, host_dict, + suite_vars, used_local_names, + suite_name=suite_name) + if rc is not None: + result.append(rc) + return result + + +def _resolve_run_phase( + group, + phase: str, + scheme_store, + host_dict: Dict[str, HostVarEntry], + suite_vars: Dict[str, 'SuiteVar'], + used_local_names: Set[str], + suite_name: str = '', +) -> List[PhaseItem]: + """Resolve the run phase, preserving subcycle do-loop structure. + + :class:`SuiteScheme` items become :class:`ResolvedCall`. + :class:`SuiteSubcycle` items become :class:`ResolvedSubcycle`. When + a subcycle contains nested ```` elements, they are + preserved recursively so the cap emitter renders the corresponding + nested ``do`` loops (matches original capgen behaviour). + :class:`SuiteSubcol` items are flattened (treated as plain schemes). + """ + from generator.suite_xml import SuiteScheme, SuiteSubcycle, SuiteSubcol + + def _resolve_items(suite_items) -> List[PhaseItem]: + """Recursively turn a list of SuiteScheme/SuiteSubcycle/SuiteSubcol + children into a list of :data:`PhaseItem`. Used at the top level + of a group AND for the body of every (possibly nested) subcycle. + """ + out: List[PhaseItem] = [] + for sub in suite_items: + if isinstance(sub, SuiteScheme): + rc = _resolve_one_call( + sub.name, phase, scheme_store, host_dict, + suite_vars, used_local_names, + suite_name=suite_name, + ) + if rc is not None: + out.append(rc) + elif isinstance(sub, SuiteSubcycle): + inner = _resolve_items(sub.items) + if inner: + loop_count, loop_std = _resolve_subcycle_loop_bound( + sub.loop, host_dict, suite_vars=suite_vars, + ) + out.append(ResolvedSubcycle( + loop=loop_count, calls=inner, + loop_std_name=loop_std, + )) + elif isinstance(sub, SuiteSubcol): + # SuiteSubcol is flattened in place — the framework + # doesn't render it as a separate loop level. + for sn in sub.scheme_names(): + rc = _resolve_one_call( + sn, phase, scheme_store, host_dict, + suite_vars, used_local_names, + suite_name=suite_name, + ) + if rc is not None: + out.append(rc) + return out + + result: List[PhaseItem] = [] + + for item in group.items: + if isinstance(item, SuiteScheme): + rc = _resolve_one_call(item.name, phase, scheme_store, host_dict, + suite_vars, used_local_names, + suite_name=suite_name) + if rc is not None: + result.append(rc) + elif isinstance(item, SuiteSubcycle): + inner = _resolve_items(item.items) + if inner: + loop_count, loop_std = _resolve_subcycle_loop_bound( + item.loop, host_dict, suite_vars=suite_vars, + ) + result.append(ResolvedSubcycle( + loop=loop_count, calls=inner, + loop_std_name=loop_std, + )) + elif isinstance(item, SuiteSubcol): + for sn in item.scheme_names(): + rc = _resolve_one_call(sn, phase, scheme_store, host_dict, + suite_vars, used_local_names, + suite_name=suite_name) + if rc is not None: + result.append(rc) + + return result + + +def _collect_dim_uses( + rg: ResolvedGroup, + host_dict: Dict[str, HostVarEntry], + suite_vars: Optional[Dict[str, 'SuiteVar']] = None, +) -> Dict[str, Set[str]]: + """Collect dimension variable USE requirements across all phases of a group. + + Host-module dimensions resolve to ``{host_module: {local_name}}``. + Suite-owned dimensions (set by ``_register``) resolve to + ``{ccpp__data: {ccpp_suite_data}}`` so the group cap can USE the + suite data module to access ``ccpp_suite_data(inst)%`` in dimension + expressions. + """ + dim_uses: Dict[str, Set[str]] = {} + for items in rg.phase_calls.values(): + for rc in iter_phase_calls(items): + for arg in rc.args: + for dim_std in arg.used_dim_std_names: + entry = host_dict.get(dim_std) + if entry is not None and entry.module_name is not None: + mod = entry.module_name + sym = entry.local_name + dim_uses.setdefault(mod, set()).add(sym) + elif suite_vars and dim_std in suite_vars: + sv = suite_vars[dim_std] + dim_uses.setdefault(sv.module_name, set()).add( + 'ccpp_suite_data') + # Subcycle loop bounds resolved from CCPP standard names also need + # a USE entry (or, for control vars, a dummy arg — handled elsewhere). + # USE the *root* of the access path so DDT-component bounds pull + # in the parent instance (e.g. ``use mod, only: phys_state``) + # rather than the bare component name. Walk *every* subcycle in + # the phase, including nested ones — each level's bound must be + # in scope at do-loop emission time. + for item in iter_phase_subcycles(items): + if not item.loop_std_name: + continue + entry = host_dict.get(item.loop_std_name) + if entry is not None and entry.module_name is not None: + dim_uses.setdefault(entry.module_name, set()).add( + _root_symbol(entry.access_path) + ) + elif suite_vars and item.loop_std_name in suite_vars: + sv = suite_vars[item.loop_std_name] + dim_uses.setdefault(sv.module_name, set()).add( + 'ccpp_suite_data' + ) + return dim_uses diff --git a/capgen-ng/generator/suite_types.py b/capgen-ng/generator/suite_types.py new file mode 100644 index 00000000..672e64f7 --- /dev/null +++ b/capgen-ng/generator/suite_types.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 + +"""Generate the shared pointer-wrapper types module for a suite. + +``ccpp__types.F90`` is a small Fortran module that declares one derived +type per unique (intrinsic-type, kind, rank) combination required by optional +arguments across all groups in the suite. Each generated type looks like:: + + type :: real_kind_phys_rank1_ptr_type + real(kind=kind_phys), pointer :: ptr(:) => null() + end type real_kind_phys_rank1_ptr_type + +Group cap modules USE this types module to declare optional-argument pointer +variables. + +Only generated when at least one optional argument is present. +""" + +import os +from typing import List, Optional, Set, Tuple + +from generator.suite_resolver import SuiteResolution, iter_phase_calls + +_INDENT = ' ' + + +######################################################################## +# Type-name helpers +######################################################################## + +def _ptr_type_name(type_: str, kind: str, rank: int) -> str: + """Return the Fortran derived-type name for a pointer wrapper. + + Parameters + ---------- + type_ : str + Fortran intrinsic type (e.g. ``'real'``, ``'integer'``). + kind : str + Kind parameter (e.g. ``'kind_phys'``), or ``''`` if none. + rank : int + Number of array dimensions (0 = scalar). + + Returns + ------- + str + + Examples + -------- + >>> _ptr_type_name('integer', '', 1) + 'integer_rank1_ptr_type' + >>> _ptr_type_name('real', 'kind_phys', 1) + 'real_kind_phys_rank1_ptr_type' + >>> _ptr_type_name('real', '', 0) + 'real_rank0_ptr_type' + >>> _ptr_type_name('real', 'kind_phys', 2) + 'real_kind_phys_rank2_ptr_type' + """ + parts = [type_] + if kind and not kind.startswith('len='): + parts.append(kind) + parts.append('rank{}'.format(rank)) + parts.append('ptr_type') + return '_'.join(parts) + + +def _ptr_rank(arg) -> int: + """Return the effective rank of the value pointed to by *arg*'s pointer. + + For optional args without transform (Case 2) or with transform (Case 4), + the pointer rank equals the number of dimensions of the host/suite variable. + """ + if arg.host_entry is not None: + return len(arg.host_entry.dimensions) + if arg.suite_var is not None: + return len(arg.suite_var.dimensions) + return 0 + + +def _ptr_type_for_arg(arg) -> Tuple[str, str, int]: + """Return the (type_, kind, rank) tuple for *arg*'s pointer wrapper. + + For Case 2 (optional, no transform), the pointer targets the host + variable directly — same type and kind. For Case 4 (optional + + transform), the pointer targets the transformation temporary, which + carries the scheme's kind. + """ + if arg.host_entry is not None: + type_ = arg.host_entry.type + else: + type_ = arg.suite_var.type_ + kind = arg.kind_scheme or (arg.host_entry.kind if arg.host_entry else + arg.suite_var.kind) + rank = _ptr_rank(arg) + return type_, kind, rank + + +######################################################################## +# Collection helpers +######################################################################## + +def _collect_ptr_type_combos( + suite_res: SuiteResolution, +) -> Set[Tuple[str, str, int]]: + """Collect unique (type, kind, rank) tuples needed by optional args. + + Parameters + ---------- + suite_res : SuiteResolution + + Returns + ------- + set of (type_, kind, rank) + """ + combos: Set[Tuple[str, str, int]] = set() + for rg in suite_res.groups: + for items in rg.phase_calls.values(): + for rc in iter_phase_calls(items): + for arg in rc.args: + if arg.ptr_name: + combos.add(_ptr_type_for_arg(arg)) + return combos + + +######################################################################## +# Module generator +######################################################################## + +def _fortran_type_str_simple(type_: str, kind: str) -> str: + """Minimal Fortran type-clause builder (intrinsics only).""" + t = type_.strip() + if kind: + if t.lower().startswith('character'): + return 'character({})'.format(kind) + return '{}(kind={})'.format(t, kind) + return t + + +def _dim_spec(rank: int) -> str: + """Return the deferred-shape dimension specifier for a *rank*-dimensional pointer. + + >>> _dim_spec(0) + '' + >>> _dim_spec(1) + '(:)' + >>> _dim_spec(2) + '(:,:)' + """ + if rank == 0: + return '' + return '({})'.format(','.join([':'] * rank)) + + +def _generate_suite_types( + suite_name: str, + combos: Set[Tuple[str, str, int]], +) -> List[str]: + """Generate the Fortran source lines for the suite types module. + + Parameters + ---------- + suite_name : str + combos : set of (type_, kind, rank) + + Returns + ------- + list of str (without trailing newlines) + """ + mod_name = 'ccpp_{}_types'.format(suite_name) + lines: List[str] = [] + + lines.append( + '! {}.F90 -- generated by ccpp_capgen_ng, do not edit'.format(mod_name) + ) + lines.append('module {}'.format(mod_name)) + lines.append('') + + # USE ccpp_kinds for any kind parameters referenced in pointer-wrapper + # type declarations (e.g. ``real(kind=kind_phys)``). + kind_names = sorted({ + kind for _t, kind, _r in combos + if kind and not kind.startswith('len=') + }) + if kind_names: + lines.append( + '{}use ccpp_kinds, only: {}'.format(_INDENT, ', '.join(kind_names)) + ) + lines.append('') + + lines.append('{}implicit none'.format(_INDENT)) + lines.append('{}private'.format(_INDENT)) + lines.append('') + + sorted_combos = sorted(combos) + + for type_, kind, rank in sorted_combos: + tname = _ptr_type_name(type_, kind, rank) + lines.append('{}public :: {}'.format(_INDENT, tname)) + + lines.append('') + + for type_, kind, rank in sorted_combos: + tname = _ptr_type_name(type_, kind, rank) + ftype = _fortran_type_str_simple(type_, kind) + dimspec = _dim_spec(rank) + lines.append('{}type :: {}'.format(_INDENT, tname)) + lines.append( + '{} {}, pointer :: ptr{} => null()'.format(_INDENT, ftype, dimspec) + ) + lines.append('{}end type {}'.format(_INDENT, tname)) + lines.append('') + + lines.append('end module {}'.format(mod_name)) + return lines + + +######################################################################## +# Public API +######################################################################## + +def write_suite_types( + suite_name: str, + suite_res: SuiteResolution, + output_root: str, +) -> Optional[str]: + """Write the suite types module to *output_root*. + + Does nothing and returns ``None`` when the suite has no optional arguments. + + Parameters + ---------- + suite_name : str + suite_res : SuiteResolution + output_root : str + Output directory (created if absent). + + Returns + ------- + str or None + Absolute path of the written file, or ``None`` if nothing was written. + """ + combos = _collect_ptr_type_combos(suite_res) + if not combos: + return None + + os.makedirs(output_root, exist_ok=True) + filename = 'ccpp_{}_types.F90'.format(suite_name) + out_path = os.path.join(output_root, filename) + + lines = _generate_suite_types(suite_name, combos) + with open(out_path, 'w', encoding='utf-8') as fh: + fh.write('\n'.join(lines) + '\n') + return out_path diff --git a/capgen-ng/generator/suite_xml.py b/capgen-ng/generator/suite_xml.py new file mode 100644 index 00000000..863937d2 --- /dev/null +++ b/capgen-ng/generator/suite_xml.py @@ -0,0 +1,639 @@ +#!/usr/bin/env python3 + +"""Suite Definition File (SDF) parser for ccpp-capgen-ng. + +A Suite Definition File is an XML document that describes which physics +schemes to run, in which order, and how to group them. This module: + +1. Reads and schema-validates the SDF (v1.0 or v2.0). +2. For v2.0 SDFs: expands ```` references recursively. +3. Writes the expanded XML to ``/ccpp__expanded.xml``. +4. Builds an in-memory :class:`Suite` object that the code generator uses. + +Suite XML format (v2.0) +----------------------- +:: + + + + + + suite_init_scheme + + + scheme_a + + + + + scheme_b + + + + + + + + + suite_final_scheme + + + +Backward compatibility +---------------------- +* Schema v1.0 suites are accepted (no nested-suite expansion). +* The old element spellings ```` (typo) and ```` are + accepted but a :mod:`logging` warning is emitted directing the author to + use ```` / ```` instead. + +Expanded XML file +----------------- +After expanding all ```` references the result is written as:: + + /ccpp__expanded.xml + +This file is for human inspection only; it is not consumed by subsequent +generator runs. + +Schema location +--------------- +The XSD files live in ``capgen-ng/schema/``. Their path is resolved relative +to this source file at runtime, so no extra configuration is needed. +""" + +import logging +import os +import xml.etree.ElementTree as ET +from typing import Dict, List, Optional, Union + +from metadata.parse_tools import ( + CCPPError, + read_xml_file, + find_schema_version, + expand_nested_suites, + write_xml_file, +) +from metadata.parse_tools.xml_tools import validate_xml_file + +######################################################################## +# Constants +######################################################################## + +#: Directory containing the XSD schemas, relative to this source file. +_SCHEMA_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'schema', +) + +#: Accepted ```` element name. The old ccpp-prebuild schema also +#: tolerated ```` (typo) and ````; capgen-ng +#: rejects both with a hard error so SDFs migrate to the canonical +#: short name. +_INIT_TAG = 'init' + +#: Accepted ```` element name. The old ccpp-prebuild schema +#: also tolerated ````; capgen-ng rejects it. +_FINAL_TAG = 'final' + +#: Element names that were valid in the old schema but are rejected in +#: capgen-ng's v2.0 SDF format. Map to the canonical short form so the +#: error message can point at the right replacement. +_REJECTED_INIT_TAGS = frozenset({'initalize', 'initialize'}) +_REJECTED_FINAL_TAGS = frozenset({'finalize'}) + + +######################################################################## +# In-memory suite data model +######################################################################## + +class SuiteScheme: + """A single scheme call within a group or subcycle. + + Parameters + ---------- + name : str + Scheme name — must match a ``type = scheme`` metadata table. + + Examples + -------- + >>> SuiteScheme('my_scheme').name + 'my_scheme' + """ + + def __init__(self, name: str): + self.name: str = name.strip() + + def scheme_names(self) -> List[str]: + """Return this scheme's name in a one-element list.""" + return [self.name] + + def __repr__(self) -> str: + return f"SuiteScheme({self.name!r})" + + +class SuiteSubcycle: + """A ```` loop wrapping one or more scheme call sites. + + Parameters + ---------- + loop : str or None + Loop-count value as written in the XML ``loop`` attribute. + May be: + + * An integer literal (e.g. ``"2"`` or ``"10"``) — a compile-time + constant. + * A Fortran identifier (e.g. ``"num_subcycles_for_scheme6"``) — a + CCPP standard name resolved against control variable metadata at + cap-generation time; becomes the value of ``ccpp_loop_extent`` + inside the loop. + * ``None`` if the attribute is absent (treated as a single iteration). + + items : list + Ordered sequence of :class:`SuiteScheme`, :class:`SuiteSubcycle`, + or :class:`SuiteSubcol` children. + + Examples + -------- + >>> sc = SuiteSubcycle(loop='2', items=[SuiteScheme('sch_a')]) + >>> sc.loop + '2' + >>> sc.is_literal_count + True + >>> sc2 = SuiteSubcycle(loop='num_subcycles_for_ag', items=[]) + >>> sc2.is_literal_count + False + """ + + def __init__(self, loop: Optional[str], items: list): + self.loop: Optional[str] = loop.strip() if loop else None + self.items: list = items + + @property + def is_literal_count(self) -> bool: + """Return True if the loop count is an integer literal.""" + if self.loop is None: + return True + try: + int(self.loop) + return True + except ValueError: + return False + + def scheme_names(self) -> List[str]: + """Return all scheme names referenced inside this subcycle.""" + names = [] + for item in self.items: + names.extend(item.scheme_names()) + return names + + def __repr__(self) -> str: + return f"SuiteSubcycle(loop={self.loop!r}, nitems={len(self.items)})" + + +class SuiteSubcol: + """A ```` sub-column processing element. + + Sub-column processing uses a ``gen`` routine to generate sub-columns + from the GCM column and an ``avg`` routine to average them back. + + Parameters + ---------- + gen_routine : str + Fortran identifier of the sub-column generation routine. + avg_routine : str + Fortran identifier of the sub-column averaging routine. + items : list + Ordered sequence of children (schemes and/or subcycles). + """ + + def __init__(self, gen_routine: str, avg_routine: str, items: list): + self.gen_routine: str = gen_routine.strip() + self.avg_routine: str = avg_routine.strip() + self.items: list = items + + def scheme_names(self) -> List[str]: + """Return all scheme names referenced inside this subcol.""" + names = [] + for item in self.items: + names.extend(item.scheme_names()) + return names + + def __repr__(self) -> str: + return (f"SuiteSubcol(gen={self.gen_routine!r}, " + f"avg={self.avg_routine!r}, nitems={len(self.items)})") + + +# Type alias for any element that can appear inside a group. +GroupItem = Union[SuiteScheme, SuiteSubcycle, SuiteSubcol] + + +class SuiteGroup: + """A named ```` within a suite. + + Parameters + ---------- + name : str + Group name (must be a valid Fortran identifier, unique within suite). + items : list of GroupItem + Ordered sequence of scheme calls, subcycles, and subcol elements. + + Examples + -------- + >>> grp = SuiteGroup('dynamics', [SuiteScheme('dyn_scheme')]) + >>> grp.name + 'dynamics' + >>> grp.scheme_names() + ['dyn_scheme'] + """ + + def __init__(self, name: str, items: List[GroupItem]): + self.name: str = name.strip() + self.items: List[GroupItem] = items + + def scheme_names(self) -> List[str]: + """Return all unique scheme names called in this group (ordered, no dedup).""" + names = [] + for item in self.items: + names.extend(item.scheme_names()) + return names + + def unique_scheme_names(self) -> List[str]: + """Return unique scheme names preserving first-occurrence order.""" + seen = set() + result = [] + for name in self.scheme_names(): + if name not in seen: + seen.add(name) + result.append(name) + return result + + def __repr__(self) -> str: + return f"SuiteGroup({self.name!r}, nitems={len(self.items)})" + + +class Suite: + """In-memory representation of a fully-parsed and expanded SDF. + + Parameters + ---------- + name : str + Suite name from the ``name`` attribute of ````. + version : list of int + Schema version ``[major, minor]``. + source_file : str + Absolute path to the original ``.xml`` source file. + groups : list of SuiteGroup + Ordered list of groups. + init_scheme : str or None + Name of the suite-level init scheme (```` element), or ``None``. + final_scheme : str or None + Name of the suite-level final scheme (```` element), or ``None``. + expanded_file : str or None + Path to the written expanded XML file, set after :func:`parse_suite_xml` + writes it. + + Examples + -------- + >>> g = SuiteGroup('grp', [SuiteScheme('sch')]) + >>> s = Suite('my_suite', [2, 0], '/path/to/f.xml', [g], None, None) + >>> s.name + 'my_suite' + >>> s.group_names() + ['grp'] + >>> s.all_scheme_names() + ['sch'] + """ + + def __init__( + self, + name: str, + version: List[int], + source_file: str, + groups: List[SuiteGroup], + init_scheme: Optional[str], + final_scheme: Optional[str], + expanded_file: Optional[str] = None, + ): + self.name: str = name + self.version: List[int] = version + self.source_file: str = source_file + self.groups: List[SuiteGroup] = groups + self.init_scheme: Optional[str] = init_scheme + self.final_scheme: Optional[str] = final_scheme + self.expanded_file: Optional[str] = expanded_file + + def group_names(self) -> List[str]: + """Return the group names in declaration order.""" + return [g.name for g in self.groups] + + def get_group(self, name: str) -> Optional[SuiteGroup]: + """Return the group with *name*, or ``None``.""" + for grp in self.groups: + if grp.name == name: + return grp + return None + + def all_scheme_names(self) -> List[str]: + """Return all unique scheme names across all groups (first-occurrence order).""" + seen = set() + result = [] + for grp in self.groups: + for name in grp.scheme_names(): + if name not in seen: + seen.add(name) + result.append(name) + return result + + def __repr__(self) -> str: + return (f"Suite({self.name!r}, version={self.version}, " + f"ngroups={len(self.groups)})") + + +######################################################################## +# XML-to-object conversion +######################################################################## + +def _parse_group_items(parent: ET.Element) -> List[GroupItem]: + """Parse the child elements of a ```` or ````/```` + into a list of :class:`GroupItem` objects. + + Parameters + ---------- + parent : xml.etree.ElementTree.Element + The containing XML element. + + Returns + ------- + list of GroupItem + """ + items: List[GroupItem] = [] + for child in parent: + tag = child.tag.lower() + if tag == 'scheme': + name = (child.text or '').strip() + if not name: + raise CCPPError( + f"Empty element inside <{parent.tag}>" + ) + items.append(SuiteScheme(name)) + elif tag == 'subcycle': + loop_val = child.get('loop') + sub_items = _parse_group_items(child) + items.append(SuiteSubcycle(loop=loop_val, items=sub_items)) + elif tag == 'subcol': + gen = child.get('gen', '').strip() + avg = child.get('avg', '').strip() + if not gen or not avg: + raise CCPPError( + " requires both 'gen' and 'avg' attributes" + ) + sub_items = _parse_group_items(child) + items.append(SuiteSubcol(gen, avg, sub_items)) + # Anything else (whitespace text nodes, comments) is silently ignored + # after nested_suite expansion (those elements are already gone). + return items + + +def _build_suite(root: ET.Element, source_file: str, + version: List[int], logger: logging.Logger) -> Suite: + """Build a :class:`Suite` from an *expanded* XML root element. + + This function must be called after :func:`expand_nested_suites` has + already resolved all ```` references. + + Parameters + ---------- + root : xml.etree.ElementTree.Element + The ```` root element (expanded). + source_file : str + Path to the original ``.xml`` file (for error messages). + version : list of int + Schema version ``[major, minor]``. + logger : logging.Logger + + Returns + ------- + Suite + """ + suite_name = root.get('name', '').strip() + if not suite_name: + raise CCPPError( + f"Suite XML '{source_file}' is missing the 'name' attribute " + "on the element" + ) + + init_scheme: Optional[str] = None + final_scheme: Optional[str] = None + groups: List[SuiteGroup] = [] + + for child in root: + tag = child.tag.lower() + + if tag == _INIT_TAG: + name = (child.text or '').strip() + if not name: + raise CCPPError( + f"SDF '{source_file}': empty <{child.tag}> element" + ) + init_scheme = name + + elif tag in _REJECTED_INIT_TAGS: + raise CCPPError( + f"SDF '{source_file}': element <{child.tag}> is not " + f"accepted; use the short form <{_INIT_TAG}> " + f"(single scheme name as text content)." + ) + + elif tag == _FINAL_TAG: + name = (child.text or '').strip() + if not name: + raise CCPPError( + f"SDF '{source_file}': empty <{child.tag}> element" + ) + final_scheme = name + + elif tag in _REJECTED_FINAL_TAGS: + raise CCPPError( + f"SDF '{source_file}': element <{child.tag}> is not " + f"accepted; use the short form <{_FINAL_TAG}> " + f"(single scheme name as text content)." + ) + + elif tag == 'group': + grp_name = child.get('name', '').strip() + if not grp_name: + raise CCPPError( + f"SDF '{source_file}': is missing 'name' attribute" + ) + items = _parse_group_items(child) + groups.append(SuiteGroup(grp_name, items)) + + elif tag == 'nested_suite': + # Should not occur after expansion; warn and skip. + logger.warning( + "SDF '%s': unexpanded element found after " + "expansion — it will be ignored.", source_file + ) + + # else: whitespace / unknown elements — silently ignored + + if not groups: + logger.warning( + "SDF '%s': suite '%s' contains no elements.", + source_file, suite_name + ) + + # Check for duplicate group names (schema enforces xs:ID uniqueness but + # validation may be skipped or xmllint may not be installed). + seen_groups: Dict[str, bool] = {} + for grp in groups: + if grp.name in seen_groups: + raise CCPPError( + f"SDF '{source_file}': duplicate group name '{grp.name}' " + f"in suite '{suite_name}'" + ) + seen_groups[grp.name] = True + + return Suite( + name=suite_name, + version=version, + source_file=source_file, + groups=groups, + init_scheme=init_scheme, + final_scheme=final_scheme, + ) + + +######################################################################## +# Public API +######################################################################## + +def parse_suite_xml( + suite_file: str, + output_root: str, + logger: Optional[logging.Logger] = None, + schema_path: Optional[str] = None, + skip_validation: bool = False, +) -> Suite: + """Parse a Suite Definition File, expand nested suites, and return a + :class:`Suite` object. + + Processing steps: + + 1. Read and XML-parse the file. + 2. Extract the schema version. + 3. Validate against the bundled XSD (unless *skip_validation* is set). + 4. For v2 suites: expand all ```` references. + 5. Re-validate the expanded XML. + 6. Write the expanded XML to + ``/ccpp__expanded.xml``. + 7. Build and return the in-memory :class:`Suite` object. + + Parameters + ---------- + suite_file : str + Path to the ``.xml`` SDF. + output_root : str + Directory where the expanded XML is written. Created if absent. + logger : logging.Logger, optional + Logger. A module-level logger is used if ``None``. + schema_path : str, optional + Directory containing XSD files. Defaults to the bundled + ``capgen-ng/schema/`` directory. + skip_validation : bool + If ``True``, skip XML schema validation (useful in test environments + where ``xmllint`` is not available). + + Returns + ------- + Suite + Fully parsed suite with all nested suites expanded. + + Raises + ------ + CCPPError + On any structural, schema, or content error. + + Examples + -------- + Parse a simple suite without writing to disk (using *skip_validation* + and a temp directory):: + + suite = parse_suite_xml('my_suite.xml', '/tmp/capgen_out', + skip_validation=True) + print(suite.name) + print(suite.group_names()) + """ + log = logger or logging.getLogger(__name__) + sdir = schema_path or _SCHEMA_DIR + + if not os.path.isfile(suite_file): + raise CCPPError(f"Suite XML file '{suite_file}' does not exist") + + log.info("Reading suite XML: %s", suite_file) + _, root = read_xml_file(suite_file, log) + version = find_schema_version(root) + log.debug("Suite XML schema version: %d.%d", *version) + + # ---- schema validation (pre-expansion) -------------------------------- + if not skip_validation: + validate_xml_file(suite_file, 'suite', version, log, schema_path=sdir) + + # ---- expand nested suites (v2 only) ----------------------------------- + if version[0] >= 2: + suite_dir = os.path.dirname(os.path.abspath(suite_file)) + expand_nested_suites(root, suite_dir, logger=log) + + # ---- build in-memory Suite object ------------------------------------- + suite = _build_suite(root, suite_file, version, log) + + # ---- write expanded XML to output_root -------------------------------- + os.makedirs(output_root, exist_ok=True) + expanded_name = f"ccpp_{suite.name}_expanded.xml" + expanded_path = os.path.join(output_root, expanded_name) + write_xml_file(root, expanded_path, log) + suite.expanded_file = expanded_path + log.info("Wrote expanded suite XML: %s", expanded_path) + + # ---- re-validate the expanded XML (catches duplicate xs:ID errors) ---- + if not skip_validation: + validate_xml_file(expanded_path, 'suite', version, log, schema_path=sdir) + + return suite + + +def parse_suite_xml_files( + suite_files: List[str], + output_root: str, + logger: Optional[logging.Logger] = None, + schema_path: Optional[str] = None, + skip_validation: bool = False, +) -> List[Suite]: + """Parse a list of SDF files and return a :class:`Suite` per file. + + Wrapper around :func:`parse_suite_xml` for processing multiple suites + in one call. + + Parameters + ---------- + suite_files : list of str + Paths to ``.xml`` SDF files. + output_root : str + Passed to :func:`parse_suite_xml`. + logger : logging.Logger, optional + schema_path : str, optional + skip_validation : bool + + Returns + ------- + list of Suite + """ + suites = [] + for fpath in suite_files: + suites.append(parse_suite_xml( + fpath, output_root, + logger=logger, schema_path=schema_path, + skip_validation=skip_validation, + )) + return suites diff --git a/capgen-ng/metadata/__init__.py b/capgen-ng/metadata/__init__.py new file mode 100644 index 00000000..5daa68f9 --- /dev/null +++ b/capgen-ng/metadata/__init__.py @@ -0,0 +1 @@ +"""Metadata parsing and variable resolution for ccpp-capgen-ng.""" diff --git a/capgen-ng/metadata/legacy_compat.py b/capgen-ng/metadata/legacy_compat.py new file mode 100644 index 00000000..366928e1 --- /dev/null +++ b/capgen-ng/metadata/legacy_compat.py @@ -0,0 +1,159 @@ +"""TRANSIENT compatibility shim for legacy CCPP standard names. + +The original ccpp-prebuild + ccpp-capgen toolchain used the standard +name ``horizontal_loop_extent`` where capgen-ng uses +``horizontal_dimension``. This module provides an opt-in shim +(``--legacy-mode`` on the capgen-ng / ccpp_validator CLI) that +silently rewrites legacy names to their canonical equivalents at +metadata parse time so the rest of the toolchain only ever sees the +canonical names. + +This module is **deliberately self-contained** so the migration can +be undone with a clean delete. Removing the feature is: + +1. Delete ``metadata/legacy_compat.py`` +2. Delete ``tests/test_legacy_compat.py`` +3. ``grep -rn 'legacy-compat\\|legacy_compat\\|--legacy-mode' .`` and + remove every remaining touchpoint (each is a 1-3 line snippet + marked with a ``# legacy-compat:`` comment). + +Every hook in the rest of the codebase is a no-op when the mode is +not enabled, so the shim has zero impact on non-legacy workflows. + +Examples +-------- +>>> from metadata import legacy_compat +>>> legacy_compat.is_enabled() +False +>>> legacy_compat.translate('horizontal_loop_extent') +'horizontal_loop_extent' +>>> legacy_compat.translate('air_temperature') +'air_temperature' + +When enabled, legacy names are rewritten: + +>>> import io, logging +>>> logger = logging.getLogger('legacy_compat_doctest') +>>> legacy_compat.enable(logger, _stream=io.StringIO()) +>>> legacy_compat.is_enabled() +True +>>> legacy_compat.translate('horizontal_loop_extent') +'horizontal_dimension' +>>> legacy_compat.translate('air_temperature') +'air_temperature' +>>> legacy_compat.disable() +>>> legacy_compat.is_enabled() +False +""" + +from __future__ import annotations + +import sys +from typing import Dict, Optional, TextIO + + +# ---------------------------------------------------------------------- +# Legacy → canonical name map. +# +# Keep this short and audited. Every entry is a deliberate decision +# that a legacy name has a single, unambiguous canonical replacement. +# ---------------------------------------------------------------------- +_LEGACY_NAME_MAP: Dict[str, str] = { + # ccpp-prebuild / original ccpp-capgen used ``horizontal_loop_extent`` + # in scheme metadata where capgen-ng uses ``horizontal_dimension``. + 'horizontal_loop_extent': 'horizontal_dimension', +} + + +# Process-level on/off flag. Module state is intentional: a single +# CLI invocation is the natural unit, and threading the flag through +# every parse call would bloat the API. Tests must use the +# ``disable()`` helper (or the ``legacy_mode_disabled`` context +# manager) to restore the default between cases. +_ENABLED: bool = False + + +def enable(logger=None, _stream: Optional[TextIO] = None) -> None: + """Turn legacy mode on and emit a single bold warning banner. + + The warning goes to *_stream* (defaults to ``sys.stderr``) and is + also logged at WARNING level on *logger* (if supplied) so that + downstream consumers of the logger see it. + + Idempotent: a second call is a no-op (no double warning). + + Parameters + ---------- + logger : logging.Logger, optional + Logger to emit a ``WARNING``-level companion message on. + _stream : file-like, optional + Override for the banner destination (used by tests to capture + output). ``None`` (default) means ``sys.stderr``. + """ + global _ENABLED + if _ENABLED: + return + _ENABLED = True + + stream = _stream if _stream is not None else sys.stderr + banner_lines = [ + '', + '*' * 70, + '*** WARNING: LEGACY-MODE ENABLED ***', + '*** ***', + '*** Scheme metadata using the deprecated standard name ***', + "*** 'horizontal_loop_extent' ***", + '*** will be silently rewritten to ***', + "*** 'horizontal_dimension' ***", + '*** at parse time. ***', + '*** ***', + '*** This is a TRANSIENT migration shim. Update your scheme ***', + '*** metadata to use the canonical name; legacy mode WILL BE ***', + '*** REMOVED in a future capgen-ng release. ***', + '*' * 70, + '', + ] + stream.write('\n'.join(banner_lines) + '\n') + try: + stream.flush() + except Exception: # pylint: disable=broad-except + pass + + if logger is not None: + logger.warning( + "Legacy mode enabled: 'horizontal_loop_extent' will be " + "rewritten to 'horizontal_dimension' in scheme metadata. " + "This shim is transient and will be removed." + ) + + +def disable() -> None: + """Turn legacy mode off. Intended for tests and library users that + wrap a generator invocation.""" + global _ENABLED + _ENABLED = False + + +def is_enabled() -> bool: + """Return ``True`` iff legacy mode has been enabled in this process.""" + return _ENABLED + + +def translate(name: str) -> str: + """Return the canonical replacement for *name*, or *name* unchanged. + + When legacy mode is **disabled** this is a strict identity — even + a legacy name like ``horizontal_loop_extent`` is returned as-is so + downstream parsers reject it just as they would in non-legacy + workflows. When legacy mode is **enabled** the entries in + :data:`_LEGACY_NAME_MAP` are rewritten and everything else passes + through unchanged. + + The function is tolerant of any input that lookup-by-string is + valid for (``str``). Callers may pre-lowercase the input — the + map keys are already lowercase to match capgen-ng's + case-insensitive standard-name convention. + """ + if not _ENABLED: + return name + return _LEGACY_NAME_MAP.get(name, name) diff --git a/capgen-ng/metadata/metadata_table.py b/capgen-ng/metadata/metadata_table.py new file mode 100644 index 00000000..6cc621fd --- /dev/null +++ b/capgen-ng/metadata/metadata_table.py @@ -0,0 +1,1332 @@ +#!/usr/bin/env python3 + +"""Metadata table parser for ccpp-capgen-ng. + +Each ``.meta`` file contains one or more CCPP metadata tables. Every table +begins with a ``[ccpp-table-properties]`` header, followed by one or more +``[ccpp-arg-table]`` section headers, each followed by variable blocks. + +Supported table types +--------------------- +``scheme`` + Physics scheme subroutine interfaces. One ``[ccpp-arg-table]`` section + per phase (``register``, ``init``, ``timestep_init``, ``run``, + ``timestep_final``, ``final``). All variables carry an ``intent`` + attribute. + +``host`` + Host-model module data (replaces the old ``module`` type — using + ``type = module`` is a hard error in this generator). + +``control`` + Framework control variables passed explicitly as subroutine arguments + (e.g. ``horizontal_loop_begin``, ``ccpp_error_code``). + +``ddt`` + Derived data type (DDT) structural definition. Describes field layout; + no instance information. Instances are declared as variables inside a + ``host`` table. + +``suite`` + Generator-written tables for suite-owned interstitial data. Never + hand-authored. + +Format reference +---------------- +:: + + [ccpp-table-properties] + name = + type = + + [ccpp-arg-table] + name = # scheme: _; others: same as table_name + type = # must match [ccpp-table-properties] type + + [ local_name ] + standard_name = + long_name = # optional + units = # optional; defaults to 'none' + dimensions = (, , ...) # () for scalar + type = + kind = # optional + intent = in | out | inout # required for scheme vars + optional = True | False # default False + active = # optional; uses standard names + protected = True | False # default False + allocatable = True | False # default False + diagnostic_name = # optional; host-tooling hint + diagnostic_name_fixed = # optional; mutually exclusive + # with diagnostic_name + +Multiple properties may appear on one line, separated by ``|``. + +DDT instance entries in a ``host`` table use a DDT type name as ``type``, and +may declare ``dimensions = (number_of_instances)`` for array instances. + +External (non-CCPP) DDT types use the syntax:: + + type = external:: + +e.g. ``type = external:mpi_f08:mpi_comm``. +""" + +import os +import re +from typing import Dict, List, Optional, Tuple + +# legacy-compat: transient migration shim (delete with the rest of +# the legacy_compat touchpoints — grep for ``legacy-compat``). +from . import legacy_compat +from .parse_tools import ( + CCPPError, + ParseContext, + ParseSyntaxError, + check_cf_standard_name, + check_units, + check_dimensions, + check_diagnostic_fixed, + check_diagnostic_id, + check_fortran_id, + check_fortran_ref, + check_fortran_intrinsic, + check_molar_mass, +) + +######################################################################## +# Module-level constants +######################################################################## + +#: All table type values accepted by the parser. +VALID_TABLE_TYPES = frozenset({'scheme', 'host', 'control', 'suite', 'ddt'}) + +#: Table types that have exactly one ``[ccpp-arg-table]`` section per table. +SINGLETON_TABLE_TYPES = frozenset({'host', 'control', 'suite', 'ddt'}) + +#: The scheme table type (the only type with multiple sections). +SCHEME_TABLE_TYPE = 'scheme' + +#: Valid scheme phase suffixes. ``finalize`` is renamed to ``final``; using +#: ``_finalize`` in a section name is a hard error. +VALID_SCHEME_PHASES = frozenset({ + 'register', 'init', 'timestep_init', 'run', 'timestep_final', 'final' +}) + +#: Valid intent values for scheme variables. +VALID_INTENTS = frozenset({'in', 'out', 'inout'}) + +#: Maximum allowed length for a Fortran identifier (F2018 §6.1.1). +FORTRAN_MAX_IDENT_LEN = 63 + +#: Regex for a bare section header ``[ name ]``. +_VAR_HEADER_RE = re.compile(r"^\[\s*(\S+)\s*\]\s*$") + +#: Regex for the reserved section keywords. +_TABLE_PROPS_HDR = '[ccpp-table-properties]' +_ARG_TABLE_HDR = '[ccpp-arg-table]' + +#: Lines that are blank or start with a comment character. +_BLANK_RE = re.compile(r"^\s*([#;].*)?$") + +#: ``external:module:typename`` DDT type syntax. +_EXTERNAL_TYPE_RE = re.compile( + r'^external\s*:\s*([A-Za-z][A-Za-z0-9_]*)\s*:\s*([A-Za-z][A-Za-z0-9_]*)$', + re.IGNORECASE, +) + +#: ``kind_spec`` value in ``[ccpp-table-properties]``. Two accepted forms: +#: +#: :=>spec -- explicit CCPP-visible kind name +#: : -- shorthand; kind_name defaults to spec +#: +#: Captured groups: (module, kind_name_or_None, spec). When the second +#: group is None the caller substitutes ``spec`` for ``kind_name``. +_KIND_SPEC_RE = re.compile( + r'^\s*([A-Za-z][A-Za-z0-9_]*)\s*:\s*' + r'(?:([A-Za-z][A-Za-z0-9_]*)\s*=>\s*)?' + r'([A-Za-z][A-Za-z0-9_]*)\s*$' +) + +######################################################################## +# Helper functions +######################################################################## + +def _is_blank(line: str) -> bool: + """Return True for blank lines and comment-only lines. + + >>> _is_blank('') + True + >>> _is_blank(' ') + True + >>> _is_blank('# comment') + True + >>> _is_blank('; comment') + True + >>> _is_blank(' name = foo') + False + """ + return _BLANK_RE.match(line) is not None + + +def _parse_bool(value: str, context: ParseContext) -> bool: + """Parse a Fortran/Python boolean string to a Python bool. + + Accepts ``True``/``False`` (case-insensitive). + + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'test.meta') + >>> _parse_bool('True', ctx) + True + >>> _parse_bool('false', ctx) + False + >>> _parse_bool('.true.', ctx) + True + >>> _parse_bool('.false.', ctx) + False + """ + normalized = value.strip().lower() + if normalized in ('true', '.true.', 't', '1'): + return True + if normalized in ('false', '.false.', 'f', '0'): + return False + raise CCPPError( + "Invalid boolean '{}', at {}".format(value, context) + ) + + +def _parse_kind_spec_value( + value: str, context: ParseContext, +) -> Tuple[str, str, str]: + """Parse one ``kind_spec`` value into ``(kind_name, module, spec)``. + + Accepted syntax:: + + :=>spec # explicit CCPP-visible kind name + : # kind_name defaults to spec + + All three components must be valid Fortran identifiers. Whitespace + around the separators is tolerated. + + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'test.meta') + >>> _parse_kind_spec_value('temp_kinds:kind_temp=>temp_r8', ctx) + ('kind_temp', 'temp_kinds', 'temp_r8') + >>> _parse_kind_spec_value('host_kinds:kind_r8', ctx) + ('kind_r8', 'host_kinds', 'kind_r8') + >>> _parse_kind_spec_value(' temp_kinds : kind_temp => temp_r8 ', ctx) + ('kind_temp', 'temp_kinds', 'temp_r8') + >>> _parse_kind_spec_value('not_a_kind_spec', ctx) + Traceback (most recent call last): + ... + metadata.parse_tools.parse_source.CCPPError: Malformed kind_spec ... + """ + match = _KIND_SPEC_RE.match(value) + if match is None: + raise CCPPError( + "Malformed kind_spec '{}', at {}: expected " + ":=>spec or :".format( + value, context + ) + ) + module = match.group(1) + kind_name = match.group(2) + spec = match.group(3) + if kind_name is None: + kind_name = spec + return kind_name, module, spec + + +def _parse_dimensions(value: str, context: ParseContext) -> List[str]: + """Parse a dimension list ``(d1, d2, ...)`` into a Python list. + + The empty list ``[]`` represents a scalar (``dimensions = ()``). + + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'test.meta') + >>> _parse_dimensions('()', ctx) + [] + >>> _parse_dimensions('(horizontal_dimension)', ctx) + ['horizontal_dimension'] + >>> _parse_dimensions('(horizontal_dimension, vertical_layer_dimension)', ctx) + ['horizontal_dimension', 'vertical_layer_dimension'] + + CCPP standard names are case-insensitive; mixed-case spellings in + the metadata are normalised to lower-case so downstream lookups + against ``host_dict`` (which stores std names lower-cased per + ``check_cf_standard_name``) succeed: + + >>> _parse_dimensions('(number_of_aerosol_tracers_MG)', ctx) + ['number_of_aerosol_tracers_mg'] + >>> _parse_dimensions('(ccpp_constant_one:Vertical_Layer_Dimension)', ctx) + ['ccpp_constant_one:vertical_layer_dimension'] + """ + stripped = value.strip() + if not (stripped.startswith('(') and stripped.endswith(')')): + raise ParseSyntaxError( + "dimensions value (must be parenthesised list)", token=value, + context=context + ) + inner = stripped[1:-1].strip() + if not inner: + return [] + parts = [p.strip() for p in inner.split(',')] + normalised: List[str] = [] + for part in parts: + if not part: + raise ParseSyntaxError( + "empty dimension entry in '{}'".format(value), + context=context + ) + check_dimensions([part], None, error=True) + # Lowercase every non-integer token so the resolver's + # host_dict lookups succeed regardless of the user's metadata + # casing. Range form ``lower:upper`` lowercases each half; + # integer literals are unchanged by ``.lower()``. + # + # legacy-compat: after lowercasing, run each token through the + # legacy translator so ``horizontal_loop_extent`` (etc.) is + # rewritten to its canonical name. No-op when legacy mode is + # disabled. + normalised.append(':'.join( + legacy_compat.translate(t.strip().lower()) + for t in part.split(':') + )) + return normalised + + +def _check_var_type(value: str, context: ParseContext) -> str: + """Validate and normalise a variable ``type`` attribute. + + Accepts: + * Fortran intrinsic types (case-insensitive, returned as given). + * ``type()`` DDT references (returned as given). + * Plain ```` DDT type names (returned as given). + * ``external::`` for non-CCPP types. + + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'test.meta') + >>> _check_var_type('real', ctx) + 'real' + >>> _check_var_type('integer', ctx) + 'integer' + >>> _check_var_type('gfs_statein_type', ctx) + 'gfs_statein_type' + >>> _check_var_type('external:mpi_f08:mpi_comm', ctx) + 'external:mpi_f08:mpi_comm' + """ + stripped = value.strip() + # Fortran intrinsic? + if check_fortran_intrinsic(stripped, error=False) is not None: + return stripped + # external:module:typename? + if _EXTERNAL_TYPE_RE.match(stripped): + return stripped + # type(identifier) form? + m = re.match(r'(?i)^type\s*\(\s*([A-Za-z][A-Za-z0-9_]*)\s*\)$', stripped) + if m: + return stripped + # Plain DDT name (a valid Fortran identifier)? + if check_fortran_id(stripped, None, error=False) is not None: + return stripped + raise ParseSyntaxError( + "variable type", token=value, context=context + ) + + +def _parse_config_line(line: str, context: ParseContext) -> List[Tuple[str, str]]: + """Parse one ini-format key=value line (possibly multiple pairs per line + separated by ``|``). + + Returns a list of ``(key, value)`` pairs. + + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'test.meta') + >>> _parse_config_line(' name = foo ', ctx) + [('name', 'foo')] + >>> _parse_config_line('units = 1 | dimensions = ()', ctx) + [('units', '1'), ('dimensions', '()')] + >>> _parse_config_line('', ctx) + [] + """ + if _is_blank(line): + return [] + pairs = [] + for segment in line.split('|'): + parts = segment.split('=', 1) + if len(parts) != 2: + raise ParseSyntaxError( + "key=value pair", token=segment.strip(), context=context + ) + key = parts[0].strip().lower() + val = parts[1].strip() + if not key: + raise ParseSyntaxError( + "empty key in property", token=segment.strip(), context=context + ) + pairs.append((key, val)) + return pairs + + +######################################################################## +# Core data classes +######################################################################## + +class MetaVar: + """A single variable entry parsed from a CCPP metadata table. + + Attributes + ---------- + local_name : str + The Fortran local variable name (from the ``[ name ]`` header). + standard_name : str + CF-compliant standard name (lowercase). + long_name : str + Human-readable description (may be empty). + units : str + Physical units string. Defaults to ``'none'`` if the metadata + entry omits the ``units`` attribute. + dimensions : list of str + Ordered list of dimension standard names; empty list for scalars. + type : str + Fortran type string (intrinsic, DDT name, or ``external:m:t``). + kind : str + Optional Fortran kind parameter (empty string if absent). + intent : str or None + ``'in'``, ``'out'``, or ``'inout'`` for scheme variables; ``None`` + for host/ddt/control variables. + optional : bool + Whether the variable is optional (default ``False``). + active : str + Fortran conditional expression (in standard names) controlling when + the variable is present; empty string if unconditionally active. + protected : bool + If ``True``, any scheme declaring ``intent`` other than ``in`` is a + metadata error. + allocatable : bool + Whether the variable is declared with the Fortran ``allocatable`` + attribute (default ``False``). Host and scheme metadata must agree + on this flag for matching standard names. Affects code generation: + actual arguments at call sites omit explicit dimension subscripts + for allocatable variables. + diagnostic_name : str + Optional host-tooling hint: the name under which the variable is + exposed to the host model's diagnostic / history-output system. May + contain ``${process}`` or ``${scheme_name}`` substitutions. Mutually + exclusive with :attr:`diagnostic_name_fixed`. When neither attribute + is explicitly set this property defaults to :attr:`local_name`, so + downstream consumers always see a non-empty value unless + ``diagnostic_name_fixed`` was provided instead. + diagnostic_name_fixed : str + Like :attr:`diagnostic_name` but a bare Fortran identifier with no + substitutions allowed. Mutually exclusive with + :attr:`diagnostic_name`. + context : ParseContext + Source location for diagnostic messages. + + Examples + -------- + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(10, 'example.meta') + >>> v = MetaVar('im', ctx) + >>> v.set_attr('standard_name', 'horizontal_loop_extent', ctx) + >>> v.set_attr('units', 'count', ctx) + >>> v.set_attr('dimensions', '()', ctx) + >>> v.set_attr('type', 'integer', ctx) + >>> v.set_attr('intent', 'in', ctx) + >>> v.standard_name + 'horizontal_loop_extent' + >>> v.intent + 'in' + >>> v.dimensions + [] + """ + + # Attributes that are boolean flags. + _BOOL_ATTRS = frozenset({'optional', 'protected', 'allocatable'}) + + # All recognised per-variable attributes. + # ``diagnostic_name`` / ``diagnostic_name_fixed`` are host-tooling hints + # propagated to ``datatable.xml``; the cap code itself does not consume + # them. The two are mutually exclusive. + _KNOWN_ATTRS = frozenset({ + 'standard_name', 'long_name', 'units', 'dimensions', + 'type', 'kind', 'intent', 'optional', 'active', 'protected', + 'allocatable', + 'diagnostic_name', 'diagnostic_name_fixed', + 'constituent', 'advected', 'molar_mass', + 'top_at_one', + }) + + def __init__(self, local_name: str, context: ParseContext): + """Initialise with *local_name* from the ``[ name ]`` section header. + + The bracket header accepts either a bare Fortran identifier + (``[foo]``) or a sliced reference (``[dqdt(:,:,index_of_)]``). + The base identifier must be a valid Fortran name within the + 63-char limit; subscript tokens may be CCPP standard names + (which routinely exceed 63 chars) and are validated separately + as references later — apply the length check only to the base. + + All other attributes are set later via :meth:`set_attr`. + """ + # Step 1: validate the overall syntactic form (bare id or array + # reference) without imposing length limits on subscript tokens. + if check_fortran_ref(local_name, None, error=False, max_len=0) is None: + raise ParseSyntaxError( + "variable local name (must be a valid Fortran identifier " + "or scalar array reference)", + token=local_name, context=context + ) + # Step 2: enforce the Fortran-identifier length limit on the + # base name only (everything before the first ``(``). + paren = local_name.find('(') + base_name = local_name[:paren].strip() if paren >= 0 else local_name.strip() + if len(base_name) > FORTRAN_MAX_IDENT_LEN: + raise ParseSyntaxError( + "variable local name base '{}' is longer than the " + "Fortran identifier limit ({} chars)".format( + base_name, FORTRAN_MAX_IDENT_LEN, + ), + token=local_name, context=context + ) + self.local_name: str = local_name + self.standard_name: str = '' + self.long_name: str = '' + self.units: str = 'none' + self.dimensions: List[str] = [] + self.type: str = '' + self.kind: str = '' + self.intent: Optional[str] = None + self.optional: bool = False + self.active: str = '' + self.protected: bool = False + self.allocatable: bool = False + # Backing field for the :attr:`diagnostic_name` property. ``set_attr`` + # writes the explicitly-supplied value here; the property layers the + # ``local_name`` default on top. + self._diagnostic_name: str = '' + self.diagnostic_name_fixed: str = '' + # Constituent-related hints (scheme metadata only). A variable is + # treated as a constituent if any of ``constituent``, ``advected``, or + # ``molar_mass`` is set to a non-default value — see + # :attr:`is_constituent`. + self.constituent: bool = False + self.advected: bool = False + self.molar_mass: float = 0.0 + # ``top_at_one`` declares the vertical-axis ordering for arrays with a + # vertical dimension. ``True`` means the model top is at index 1 (k=1 + # topmost, k=nz surface); the default ``False`` means surface at 1 + # (k=1 surface, k=nz top). When a scheme and the host disagree on + # this flag the generator emits a vertical-flip transform that + # substitutes the vertical index with `` - k + 1`` on the + # host-side access expression. + self.top_at_one: bool = False + self.context: ParseContext = context + # Track which attributes have been explicitly set (for validation). + self._set_attrs: set = set() + + # ------------------------------------------------------------------ + def set_attr(self, key: str, value: str, context: ParseContext) -> None: + """Store attribute *key* = *value* after validating the value. + + Parameters + ---------- + key : str + Lower-case attribute name. + value : str + Raw string value from the metadata file. + context : ParseContext + Source location (for error messages). + + Raises + ------ + CCPPError + On unknown attribute names or invalid values. + """ + if key not in self._KNOWN_ATTRS: + raise CCPPError( + "Unknown variable attribute '{}' for '{}', at {}".format( + key, self.local_name, context + ) + ) + if key in self._set_attrs: + raise CCPPError( + "Duplicate attribute '{}' for variable '{}', at {}".format( + key, self.local_name, context + ) + ) + self._set_attrs.add(key) + + if key == 'standard_name': + # legacy-compat: rewrite deprecated names (e.g. + # horizontal_loop_extent → horizontal_dimension) when + # legacy mode is enabled. Applied *after* + # check_cf_standard_name (which lowercases) so mixed-case + # legacy spellings are captured. No-op otherwise. + self.standard_name = legacy_compat.translate( + check_cf_standard_name(value, None, error=True)) + elif key == 'long_name': + self.long_name = value + elif key == 'units': + self.units = check_units(value, None, error=True) + elif key == 'dimensions': + self.dimensions = _parse_dimensions(value, context) + elif key == 'type': + self.type = _check_var_type(value, context) + elif key == 'kind': + self.kind = value.strip() + elif key == 'intent': + iv = value.strip().lower() + if iv not in VALID_INTENTS: + raise CCPPError( + "Invalid intent '{}' for '{}'; must be one of {}, at {}".format( + value, self.local_name, sorted(VALID_INTENTS), context + ) + ) + self.intent = iv + elif key == 'optional': + self.optional = _parse_bool(value, context) + elif key == 'active': + self.active = value.strip() + elif key == 'protected': + self.protected = _parse_bool(value, context) + elif key == 'allocatable': + self.allocatable = _parse_bool(value, context) + elif key == 'diagnostic_name': + self._diagnostic_name = check_diagnostic_id( + value.strip(), self._prop_snapshot(), error=True + ) + elif key == 'diagnostic_name_fixed': + self.diagnostic_name_fixed = check_diagnostic_fixed( + value.strip(), self._prop_snapshot(), error=True + ) + elif key == 'constituent': + self.constituent = _parse_bool(value, context) + elif key == 'advected': + self.advected = _parse_bool(value, context) + elif key == 'molar_mass': + self.molar_mass = check_molar_mass(value.strip(), None, error=True) + elif key == 'top_at_one': + self.top_at_one = _parse_bool(value, context) + + # ------------------------------------------------------------------ + @property + def is_constituent(self) -> bool: + """Return True iff this variable is flagged as a constituent. + + A variable is treated as a constituent when any of the three + constituent-hint attributes is set to a non-default value: + + * ``constituent = True`` + * ``advected = True`` + * ``molar_mass != 0.0`` + + Matches the rollup in the original capgen + (``scripts/metavar.py`` ``__is_constituent``). + """ + return self.constituent or self.advected or self.molar_mass != 0.0 + + # ------------------------------------------------------------------ + @property + def diagnostic_name(self) -> str: + """Effective diagnostic name. + + Returns the explicitly-set value when one was provided. Otherwise, + when ``diagnostic_name_fixed`` is also unset, falls back to + :attr:`local_name` (matching the original capgen + ``local_name_to_diag_name`` default). Returns an empty string only + when ``diagnostic_name_fixed`` was provided instead. + """ + if self._diagnostic_name: + return self._diagnostic_name + if self.diagnostic_name_fixed: + return '' + return self.local_name + + def _prop_snapshot(self) -> Dict[str, str]: + """Return a dict snapshot of attributes the diagnostic checkers may inspect. + + The ``check_diagnostic_id`` and ``check_diagnostic_fixed`` helpers + cross-validate the two diagnostic attributes and reference the + variable's local and standard names in error messages. The snapshot + reports the *explicitly-set* ``diagnostic_name`` (not the + ``local_name`` default) so the mutual-exclusion check fires only when + the author actually set it. + """ + return { + 'local_name' : self.local_name, + 'standard_name' : self.standard_name, + 'diagnostic_name' : self._diagnostic_name, + 'diagnostic_name_fixed': self.diagnostic_name_fixed, + } + + # ------------------------------------------------------------------ + def validate(self, require_intent: bool, context: ParseContext) -> None: + """Check that all required attributes are present. + + Parameters + ---------- + require_intent : bool + If ``True`` (scheme variables), ``intent`` must be set. + context : ParseContext + Source location (for error messages). + + Raises + ------ + CCPPError + If any required attribute is missing. + """ + required = {'standard_name', 'dimensions', 'type'} + if require_intent: + required.add('intent') + missing = required - self._set_attrs + if missing: + raise CCPPError( + "Variable '{}' is missing required attributes: {}, at {}".format( + self.local_name, sorted(missing), context + ) + ) + + # ------------------------------------------------------------------ + def __repr__(self) -> str: + return "MetaVar({!r}, standard_name={!r})".format( + self.local_name, self.standard_name + ) + + def is_external_ddt(self) -> bool: + """Return True if this variable's type is an external (non-CCPP) DDT.""" + return _EXTERNAL_TYPE_RE.match(self.type) is not None + + def external_ddt_module(self) -> Optional[str]: + """Return the module name for an external DDT type, or None.""" + m = _EXTERNAL_TYPE_RE.match(self.type) + return m.group(1) if m else None + + def external_ddt_typename(self) -> Optional[str]: + """Return the type name for an external DDT type, or None.""" + m = _EXTERNAL_TYPE_RE.match(self.type) + return m.group(2) if m else None + + +######################################################################## + +class MetadataSection: + """One ``[ccpp-arg-table]`` section: name, type, and variables. + + For scheme tables the section name encodes the phase: + ``_`` where *phase* is one of + ``register``, ``init``, ``timestep_init``, ``run``, + ``timestep_final``, ``final``. + + For non-scheme tables the section name equals the table name. + + Parameters + ---------- + section_name : str + The ``name`` attribute from ``[ccpp-arg-table]``. + section_type : str + The ``type`` attribute (must match the enclosing table type). + table_name : str + The enclosing table's name (used for validation and error messages). + context : ParseContext + Source location. + + Examples + -------- + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(5, 'scheme.meta') + >>> sec = MetadataSection('my_scheme_run', 'scheme', 'my_scheme', ctx) + >>> sec.section_name + 'my_scheme_run' + >>> sec.phase + 'run' + >>> sec = MetadataSection('host_data', 'host', 'host_data', ctx) + >>> sec.phase is None + True + """ + + def __init__(self, section_name: str, section_type: str, + table_name: str, context: ParseContext): + if section_type not in VALID_TABLE_TYPES: + raise CCPPError( + "Section type '{}' is not a valid table type; " + "must be one of {}, at {}".format( + section_type, sorted(VALID_TABLE_TYPES), context + ) + ) + self.section_name: str = section_name + self.section_type: str = section_type + self.context: ParseContext = context + self.variables: List[MetaVar] = [] + self._std_name_index: Dict[str, MetaVar] = {} + # Derive the scheme phase from the section name (scheme only). + self._phase: Optional[str] = None + if section_type == SCHEME_TABLE_TYPE: + self._phase = self._extract_phase(section_name, table_name, context) + + @staticmethod + def _extract_phase(section_name: str, scheme_name: str, + context: ParseContext) -> str: + """Extract and validate the phase suffix from a scheme section name. + + The section name must have the form ``_``. + If the suffix is ``finalize`` (the old name), a hard error is raised + directing the user to rename it to ``final``. + """ + prefix = scheme_name + '_' + if not section_name.startswith(prefix): + raise CCPPError( + "Scheme section name '{}' does not begin with scheme name '{}', " + "at {}".format(section_name, scheme_name, context) + ) + phase = section_name[len(prefix):] + if phase == 'finalize': + raise CCPPError( + "Phase 'finalize' has been renamed to 'final'; " + "rename '{}' to '{}', at {}".format( + section_name, scheme_name + '_final', context + ) + ) + if phase not in VALID_SCHEME_PHASES: + raise CCPPError( + "Unknown scheme phase '{}' in section '{}'; " + "must be one of {}, at {}".format( + phase, section_name, sorted(VALID_SCHEME_PHASES), context + ) + ) + return phase + + @property + def phase(self) -> Optional[str]: + """The scheme phase (``'run'``, ``'init'``, etc.) or ``None``.""" + return self._phase + + def add_variable(self, var: MetaVar) -> None: + """Append *var* to this section, checking for duplicate standard names.""" + if var.standard_name in self._std_name_index: + existing = self._std_name_index[var.standard_name] + raise CCPPError( + "Duplicate standard name '{}' in section '{}': " + "first at {}, duplicate at {}".format( + var.standard_name, self.section_name, + existing.context, var.context + ) + ) + self.variables.append(var) + self._std_name_index[var.standard_name] = var + + def get_variable(self, standard_name: str) -> Optional[MetaVar]: + """Return the variable with *standard_name*, or ``None``.""" + return self._std_name_index.get(standard_name) + + def __repr__(self) -> str: + return "MetadataSection({!r}, nvars={})".format( + self.section_name, len(self.variables) + ) + + +######################################################################## + +class MetadataTable: + """A complete CCPP metadata table (one ``[ccpp-table-properties]`` block + and all its ``[ccpp-arg-table]`` sections). + + Parameters + ---------- + table_name : str + The ``name`` from ``[ccpp-table-properties]``. + table_type : str + One of ``scheme``, ``host``, ``control``, ``suite``, ``ddt``. + file_path : str + Source file path (used in error messages and ``USE`` statements). + context : ParseContext + The location of the ``[ccpp-table-properties]`` header. + + Examples + -------- + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 'host.meta') + >>> tbl = MetadataTable('my_module', 'host', 'host.meta', ctx) + >>> tbl.table_name + 'my_module' + >>> tbl.table_type + 'host' + >>> tbl.is_scheme + False + """ + + def __init__(self, table_name: str, table_type: str, + file_path: str, context: ParseContext): + if table_type not in VALID_TABLE_TYPES: + raise CCPPError( + "Table type '{}' is not valid; must be one of {}, at {}".format( + table_type, sorted(VALID_TABLE_TYPES), context + ) + ) + self.table_name: str = table_name + self.table_type: str = table_type + self.file_path: str = file_path + self.context: ParseContext = context + self._sections: List[MetadataSection] = [] + # Optional table-level properties (set by apply_table_props). + self.dependencies: List[str] = [] + self.source_path: str = '' + # Fortran module name that exports this table's Fortran symbols. + # When absent in metadata, falls back to :attr:`table_name` (the + # common case: the .meta file shares its base name with the + # Fortran module). Applies to any table type whose contents are + # imported from a Fortran module (``scheme``, ``host``, ``ddt``); + # the cap generator uses it to emit ``use , only: ...`` + # lines targeting the actual module rather than the table name. + self.module_name: str = '' + # Each entry is ``(kind_name, module, spec)``; aggregated by + # ccpp_capgen_ng into the kind map for ccpp_kinds.F90. + self.kind_specs: List[Tuple[str, str, str]] = [] + + def apply_table_props(self, props: dict) -> None: + """Apply extra ``[ccpp-table-properties]`` key-value pairs to this table. + + Recognised keys + --------------- + ``source_path`` + Relative path from the ``.meta`` file directory to the directory + containing the corresponding Fortran source (``.F90``). Resolved + to an absolute path and stored in :attr:`source_path`. Defaults + to the ``.meta`` file's own directory. + + ``dependencies`` + Comma-separated list of dependency file paths (may include + ``../../``-style relative paths). Resolved against the ``.meta`` + directory, optionally adjusted by ``dependencies_path``. + + ``dependencies_path`` + Relative path from the ``.meta`` directory used as the base for + resolving entries in ``dependencies``. + + ``kind_spec`` + Either a single ``:=>spec`` (or shorthand + ``:``) string, or a list of such strings when the + ``kind_spec`` key appears more than once in the table header. + Each entry is parsed into a ``(kind_name, module, spec)`` triple + and appended to :attr:`kind_specs`. + + Examples + -------- + >>> from metadata.parse_tools import ParseContext + >>> ctx = ParseContext(0, 's.meta') + >>> t = MetadataTable('s', 'scheme', '/project/src/s.meta', ctx) + >>> t.apply_table_props({'source_path': 'fortran', 'dependencies': 'util.F90', 'dependencies_path': 'lib'}) + >>> t.source_path == '/project/src/fortran' + True + >>> t.dependencies + ['/project/src/lib/util.F90'] + >>> t = MetadataTable('s', 'scheme', '/p/s.meta', ctx) + >>> t.apply_table_props({'kind_spec': 'temp_kinds:kind_temp=>temp_r8'}) + >>> t.kind_specs + [('kind_temp', 'temp_kinds', 'temp_r8')] + >>> t = MetadataTable('s', 'scheme', '/p/s.meta', ctx) + >>> t.apply_table_props({'kind_spec': [ + ... 'temp_kinds:kind_temp=>temp_r8', + ... 'host_kinds:kind_r4', + ... ]}) + >>> t.kind_specs + [('kind_temp', 'temp_kinds', 'temp_r8'), ('kind_r4', 'host_kinds', 'kind_r4')] + """ + meta_dir = os.path.dirname(os.path.abspath(self.file_path)) + + if 'source_path' in props: + self.source_path = os.path.normpath( + os.path.join(meta_dir, props['source_path']) + ) + else: + self.source_path = meta_dir + + dep_base = meta_dir + if 'dependencies_path' in props: + dep_base = os.path.normpath( + os.path.join(meta_dir, props['dependencies_path']) + ) + + if 'dependencies' in props: + # ``dependencies`` may legitimately appear more than once in a + # single ``[ccpp-table-properties]`` block; the parser + # collects repeats into a list (analogous to ``kind_spec``). + # A single occurrence still arrives as a string. Each entry + # is a comma-separated list of file paths (or "none" to + # signal an empty dependency set). + raw_entries = props['dependencies'] + if isinstance(raw_entries, str): + raw_entries = [raw_entries] + for raw in raw_entries: + raw = raw.strip() + if raw.lower() == 'none': + continue + for entry in raw.split(','): + entry = entry.strip() + if entry: + self.dependencies.append( + os.path.normpath(os.path.join(dep_base, entry)) + ) + + if 'kind_spec' in props: + raw_specs = props['kind_spec'] + if isinstance(raw_specs, str): + raw_specs = [raw_specs] + for entry in raw_specs: + self.kind_specs.append( + _parse_kind_spec_value(entry, self.context) + ) + + if 'module_name' in props: + raw = props['module_name'].strip() + if raw: + self.module_name = raw + + @property + def is_scheme(self) -> bool: + """True if this is a ``scheme`` table.""" + return self.table_type == SCHEME_TABLE_TYPE + + def sections(self) -> List[MetadataSection]: + """Return all sections (``[ccpp-arg-table]`` blocks) in this table.""" + return list(self._sections) + + def variables(self) -> List[MetaVar]: + """Return all variables across all sections (de-duplicated by standard name). + + For a singleton table (``host``, ``control``, ``ddt``) there is only + one section, so this simply returns that section's variables. For + scheme tables all variables from all phases are returned; variables + that appear in multiple phases are included only once (first occurrence + wins). + """ + seen: Dict[str, MetaVar] = {} + for sec in self._sections: + for var in sec.variables: + if var.standard_name not in seen: + seen[var.standard_name] = var + return list(seen.values()) + + def add_section(self, section: MetadataSection) -> None: + """Append *section* to this table. + + Enforces that singleton table types (``host``, ``control``, ``suite``, + ``ddt``) have at most one section. Also verifies that the section's + type matches the table type. + """ + if section.section_type != self.table_type: + raise CCPPError( + "Section type '{}' does not match table type '{}' " + "in table '{}', at {}".format( + section.section_type, self.table_type, + self.table_name, section.context + ) + ) + if self.table_type in SINGLETON_TABLE_TYPES and self._sections: + raise CCPPError( + "Table type '{}' allows only one section per table; " + "found a second section '{}' in table '{}', at {}".format( + self.table_type, section.section_name, + self.table_name, section.context + ) + ) + self._sections.append(section) + + def section_for_phase(self, phase: str) -> Optional[MetadataSection]: + """Return the section for the given scheme *phase*, or ``None``. + + Only meaningful for scheme tables; always returns ``None`` for + other table types. + """ + for sec in self._sections: + if sec.phase == phase: + return sec + return None + + def __repr__(self) -> str: + return "MetadataTable({!r}, type={!r}, nsections={})".format( + self.table_name, self.table_type, len(self._sections) + ) + + +######################################################################## +# File parser +######################################################################## + +def parse_metadata_file(file_path: str) -> List[MetadataTable]: + """Parse a ``.meta`` file and return all :class:`MetadataTable` objects. + + Parameters + ---------- + file_path : str + Absolute or relative path to the ``.meta`` file. + + Returns + ------- + list of MetadataTable + One entry per ``[ccpp-table-properties]`` block found in the file. + + Raises + ------ + CCPPError + On any structural or content error. The error message includes the + file path and line number. + + Notes + ----- + ``type = module`` is rejected with a descriptive error directing the user + to use ``type = host`` instead (breaking rename from the old generator). + + All blank lines and lines starting with ``#`` or ``;`` are ignored. + + Examples + -------- + >>> import tempfile, os + >>> content = ''' + ... [ccpp-table-properties] + ... name = test_host + ... type = host + ... + ... [ccpp-arg-table] + ... name = test_host + ... type = host + ... [ im ] + ... standard_name = horizontal_loop_extent + ... units = count + ... dimensions = () + ... type = integer + ... ''' + >>> with tempfile.NamedTemporaryFile(mode='w', suffix='.meta', + ... delete=False) as f: + ... _ = f.write(content) + ... fname = f.name + >>> tables = parse_metadata_file(fname) + >>> os.unlink(fname) + >>> len(tables) + 1 + >>> tables[0].table_name + 'test_host' + >>> tables[0].table_type + 'host' + >>> tables[0].sections()[0].variables[0].standard_name + 'horizontal_loop_extent' + """ + if not os.path.isfile(file_path): + raise CCPPError("Metadata file '{}' does not exist".format(file_path)) + + with open(file_path, 'r', encoding='utf-8') as fh: + lines = fh.readlines() + + return _parse_lines(lines, file_path) + + +def _parse_lines(lines: List[str], file_path: str) -> List[MetadataTable]: + """Internal line-by-line parser. Exposed for unit-testing without + needing real files. + + Parameters + ---------- + lines : list of str + Source lines (with or without trailing newlines). + file_path : str + Used only for :class:`ParseContext` error messages. + + Returns + ------- + list of MetadataTable + """ + tables: List[MetadataTable] = [] + current_table: Optional[MetadataTable] = None + current_section: Optional[MetadataSection] = None + current_var: Optional[MetaVar] = None + collecting_table_props = False + collecting_section_props = False + + # Accumulate key=value pairs for the current table/section/variable header. + pending_props: Dict[str, str] = {} + pending_start: int = 0 + + def ctx(lineno: int) -> ParseContext: + return ParseContext(linenum=lineno, filename=file_path) + + def flush_var(lineno: int) -> None: + """Validate and attach the buffered variable to the current section.""" + nonlocal current_var + if current_var is None: + return + require_intent = (current_section is not None and + current_section.section_type == SCHEME_TABLE_TYPE) + current_var.validate(require_intent=require_intent, context=ctx(lineno)) + if current_section is None: + raise CCPPError( + "Variable '{}' found outside any section, at {}".format( + current_var.local_name, ctx(lineno) + ) + ) + current_section.add_variable(current_var) + current_var = None + + def flush_section(lineno: int) -> None: + """Attach the current section to the current table.""" + nonlocal current_section + if current_section is None: + return + flush_var(lineno) + if current_table is None: + raise CCPPError( + "Section found outside any table, at {}".format(ctx(lineno)) + ) + current_table.add_section(current_section) + current_section = None + + def flush_table_props() -> None: + """Apply any extra table-property keys to the current table.""" + if current_table is not None and collecting_table_props: + current_table.apply_table_props(pending_props) + + for lineno, raw_line in enumerate(lines): + line = raw_line.rstrip('\n').rstrip('\r') + + # ---- [ccpp-table-properties] ---------------------------------------- + if line.strip().lower() == _TABLE_PROPS_HDR: + # Finish whatever we were doing. + flush_table_props() + flush_section(lineno) + if current_table is not None: + tables.append(current_table) + current_table = None + pending_props = {} + pending_start = lineno + collecting_table_props = True + collecting_section_props = False + continue + + # ---- [ccpp-arg-table] ----------------------------------------------- + if line.strip().lower() == _ARG_TABLE_HDR: + flush_table_props() + flush_section(lineno) + collecting_section_props = True + collecting_table_props = False + pending_props = {} + pending_start = lineno + continue + + # ---- Blank / comment lines ------------------------------------------ + if _is_blank(line): + continue + + # ---- Variable header [ name ] ---------------------------------------- + var_match = _VAR_HEADER_RE.match(line) + if var_match: + collecting_table_props = False + collecting_section_props = False + # Flush the previous variable. + flush_var(lineno) + # Start a new variable. + local_name = var_match.group(1) + current_var = MetaVar(local_name, ctx(lineno)) + continue + + # ---- Key = value line ----------------------------------------------- + if collecting_table_props: + pairs = _parse_config_line(line, ctx(lineno)) + for key, val in pairs: + if key in ('kind_spec', 'dependencies'): + # These keys may legitimately appear more than once + # in a single table header; accumulate occurrences + # into a list. ``apply_table_props`` accepts either + # form (string when seen once, list when repeated). + pending_props.setdefault(key, []).append(val) + continue + if key in pending_props: + raise CCPPError( + "Duplicate table property '{}', at {}".format( + key, ctx(lineno) + ) + ) + pending_props[key] = val + # Try to build the MetadataTable as soon as we have name + type. + if 'name' in pending_props and 'type' in pending_props and current_table is None: + current_table = _make_table( + pending_props['name'], pending_props['type'], + file_path, ctx(pending_start) + ) + continue + + if collecting_section_props: + pairs = _parse_config_line(line, ctx(lineno)) + for key, val in pairs: + if key in pending_props: + raise CCPPError( + "Duplicate section property '{}', at {}".format( + key, ctx(lineno) + ) + ) + pending_props[key] = val + # Try to build the MetadataSection as soon as we have name + type. + if ('name' in pending_props and 'type' in pending_props + and current_section is None and current_table is not None): + current_section = _make_section( + pending_props['name'], pending_props['type'], + current_table.table_name, ctx(pending_start) + ) + continue + + if current_var is not None: + pairs = _parse_config_line(line, ctx(lineno)) + for key, val in pairs: + if current_section is not None: + sec_type = current_section.section_type + if sec_type == SCHEME_TABLE_TYPE and key == 'active': + raise ParseSyntaxError( + "'active' is a host-model-only attribute and cannot " + "appear in scheme metadata", + token=key, context=ctx(lineno) + ) + if sec_type in ('host', 'control', 'ddt') and key == 'optional': + raise ParseSyntaxError( + "'optional' is a scheme-only attribute and cannot " + "appear in host, control, or ddt metadata", + token=key, context=ctx(lineno) + ) + if (sec_type != SCHEME_TABLE_TYPE and + key in ('constituent', 'advected', 'molar_mass')): + raise ParseSyntaxError( + "'{}' is a scheme-only constituent hint and cannot " + "appear in host, control, ddt, or suite metadata".format(key), + token=key, context=ctx(lineno) + ) + current_var.set_attr(key, val, ctx(lineno)) + continue + + # Line is not blank and not handled — syntax error. + raise ParseSyntaxError("unexpected line", token=line.strip(), + context=ctx(lineno)) + + # ---- End of file: flush any in-progress objects ------------------------- + flush_table_props() + flush_section(len(lines)) + if current_table is not None: + tables.append(current_table) + + return tables + + +def _make_table(name: str, type_str: str, file_path: str, + context: ParseContext) -> MetadataTable: + """Construct a :class:`MetadataTable`, with helpful error for ``type=module``.""" + ttype = type_str.strip().lower() + if ttype == 'module': + raise CCPPError( + "Table type 'module' is not supported; use 'type = host' instead, " + "at {}".format(context) + ) + return MetadataTable(name.strip(), ttype, file_path, context) + + +def _make_section(name: str, type_str: str, table_name: str, + context: ParseContext) -> MetadataSection: + """Construct a :class:`MetadataSection`.""" + return MetadataSection( + name.strip(), type_str.strip().lower(), table_name, context + ) diff --git a/capgen-ng/metadata/parse_tools/__init__.py b/capgen-ng/metadata/parse_tools/__init__.py new file mode 100644 index 00000000..424d8f14 --- /dev/null +++ b/capgen-ng/metadata/parse_tools/__init__.py @@ -0,0 +1,39 @@ +"""Parse utilities shared across metadata parsing and validation.""" + +from .parse_source import ( + CCPPError, + ParseSyntaxError, + ParseInternalError, + ParseContextError, + ParseContext, + ParseSource, + context_string, + type_name, +) +from .parse_log import init_log, set_log_level, set_log_to_null, set_log_to_stdout +from .parse_checkers import ( + check_units, + check_dimensions, + check_cf_standard_name, + check_diagnostic_fixed, + check_diagnostic_id, + check_fortran_id, + check_fortran_ref, + check_fortran_type, + check_fortran_intrinsic, + check_molar_mass, + FORTRAN_ID, + FORTRAN_SCALAR_REF_RE, + FORTRAN_INTRINSIC_TYPES, +) +from .parse_object import ParseObject +from .fortran_conditional import ( + FORTRAN_CONDITIONAL_REGEX, + FORTRAN_CONDITIONAL_REGEX_WORDS, +) +from .xml_tools import ( + read_xml_file, + find_schema_version, + expand_nested_suites, + write_xml_file, +) diff --git a/capgen-ng/metadata/parse_tools/fortran_conditional.py b/capgen-ng/metadata/parse_tools/fortran_conditional.py new file mode 100755 index 00000000..17ae6859 --- /dev/null +++ b/capgen-ng/metadata/parse_tools/fortran_conditional.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# + +"""Definitions to convert a conditional statement in the metadata, expressed in standard names, +into a Fortran conditional (used in an if statement), expressed in local names. +""" + +import re + +FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '(', ')', '==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.', + '.true.', '.false.', '.lt.', '.le.', '.eq.', '.ge.', '.gt.', '.ne.', + '.not.', '.and.', '.or.', '.xor.'] +FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([word.replace('(',r'\(').replace(')', r'\)') for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) diff --git a/capgen-ng/metadata/parse_tools/parse_checkers.py b/capgen-ng/metadata/parse_tools/parse_checkers.py new file mode 100644 index 00000000..dd4d1124 --- /dev/null +++ b/capgen-ng/metadata/parse_tools/parse_checkers.py @@ -0,0 +1,1100 @@ +#!/usr/bin/env python3 + +"""Helper functions to validate parsed input""" + +# Python library imports +import re +# CCPP framework imports +from .parse_source import CCPPError, ParseInternalError + +######################################################################## + +_UNITLESS_REGEX = "1" +_NON_LEADING_ZERO_NUM = r"[1-9]\d*" +_CHAR_WITH_UNDERSCORE = "([a-zA-Z]+_[a-zA-Z]+)+" +_NEGATIVE_NON_LEADING_ZERO_NUM = f"[-]{_NON_LEADING_ZERO_NUM}" +_POSITIVE_NON_LEADING_ZERO_NUM = f"[+]{_NON_LEADING_ZERO_NUM}" +_UNIT_EXPONENT = f"({_NEGATIVE_NON_LEADING_ZERO_NUM}|{_POSITIVE_NON_LEADING_ZERO_NUM}|{_NON_LEADING_ZERO_NUM})" +_UNIT_REGEX = f"[a-zA-Z]+{_UNIT_EXPONENT}?" +_UNITS_REGEX = rf"^({_CHAR_WITH_UNDERSCORE}|{_UNIT_REGEX}(\s{_UNIT_REGEX})*|{_UNITLESS_REGEX})$" +_UNITS_RE = re.compile(_UNITS_REGEX) +_MAX_MOLAR_MASS = 10000.0 + +def check_units(test_val, prop_dict, error): + """Return if a valid unit, otherwise, None + if is True, raise an Exception if is not valid. + >>> check_units('m s-1', None, True) + 'm s-1' + >>> check_units('kg m-3', None, True) + 'kg m-3' + >>> check_units('m2 s-2', None, True) + 'm2 s-2' + >>> check_units('m+2 s-2', None, True) + 'm+2 s-2' + >>> check_units('1', None, True) + '1' + >>> check_units('', None, False) + + >>> check_units('', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '' is not a valid unit + >>> check_units(' ', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '' is not a valid unit + >>> check_units(['foo'], None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: ['foo'] is invalid; not a string + """ + if isinstance(test_val, str): + if _UNITS_RE.match(test_val.strip()) is None: + if error: + raise CCPPError("'{}' is not a valid unit".format(test_val)) + else: + test_val = None + # end if + # end if + else: + if error: + raise CCPPError("'{}' is invalid; not a string".format(test_val)) + else: + test_val = None + # end if + # end if + return test_val + +def check_dimensions(test_val, prop_dict, error, max_len=0): + """Return if a valid dimensions list, otherwise, None + If > 0, each string in must not be longer than + . + if is True, raise an Exception if is not valid. + >>> check_dimensions(["dim1", "dim2name"], None, False) + ['dim1', 'dim2name'] + >>> check_dimensions([":", ":"], None, False) + [':', ':'] + >>> check_dimensions([":", "dim2"], None, False) + [':', 'dim2'] + >>> check_dimensions(["dim1", ":"], None, False) + ['dim1', ':'] + >>> check_dimensions(["8", "::"], None, False) + ['8', '::'] + >>> check_dimensions(['start1:end1', 'start2:end2'], None, False) + ['start1:end1', 'start2:end2'] + >>> check_dimensions(['start1:', 'start2:end2'], None, False) + ['start1:', 'start2:end2'] + >>> check_dimensions(['start1 :end1', 'start2: end2'], None, False) + ['start1 :end1', 'start2: end2'] + >>> check_dimensions(['size(foo)'], None, False) + ['size(foo)'] + >>> check_dimensions(['size(foo,1) '], None, False) + ['size(foo,1) '] + >>> check_dimensions(['size(foo,1'], None, False) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: Invalid dimension component, size(foo,1 + >>> check_dimensions(["dim1", "dim2name"], None, False, max_len=5) + + >>> check_dimensions(["dim1", "dim2name"], None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'dim2name' is too long (> 5 chars) + >>> check_dimensions("hi_mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'hi_mom' is invalid; not a list + >>> check_dimensions(["1:dim1", "dim2name"], None, True) + ['1:dim1', 'dim2name'] + >>> check_dimensions(["2:dim1", "dim2name"], None, True) + ['2:dim1', 'dim2name'] + >>> check_dimensions(["ccpp_constant_one:1", "dim2name"], None, True) + ['ccpp_constant_one:1', 'dim2name'] + """ + if not isinstance(test_val, list): + if error: + raise CCPPError("'{}' is invalid; not a list".format(test_val)) + else: + test_val = None + # end if + else: + for item in test_val: + isplit = item.split(':') + # Check for too many colons + if (len(isplit) > 3): + if error: + errmsg = "'{}' is an invalid dimension range" + raise CCPPError(errmsg.format(item)) + else: + test_val = None + # end if + break + # end if + # Check possible dim styles (a, a:b, a:, :b, :, ::, a:b:c, a::c). + # Integer literals are valid in any bound position; semantic + # restrictions (e.g. horizontal_dimension lower bound must be 1) + # are enforced by the resolver, not here. + tdims = [x.strip() for x in isplit if len(x) > 0] + for tdim in tdims: + try: + valid = isinstance(int(tdim), int) + except ValueError as ve: + # Not an integer, try a Fortran ID + valid = check_fortran_id(tdim, None, + error, max_len=max_len) is not None + if not valid: + # Check for size entry -- simple check + tcheck = tdim.strip().lower() + if tcheck[0:4] == 'size': + ploc = check_balanced_paren(tdim[4:]) + if -1 in ploc: + emsg = 'Invalid dimension component, {}' + raise CCPPError(emsg.format(tdim)) + else: + valid = tdim + # end if + # end if + # end if + # End try + if not valid: + if error: + raise CCPPError(f"'{item}' is an invalid dimension name") + else: + test_val = None + # end if + break + # end if + # end for + # end for + # end if + return test_val + +######################################################################## + +# CF_ID is a string representing the regular expression for CF Standard Names +CF_ID = r"(?i)[a-z][a-z0-9_]*" +__CFID_RE = re.compile(CF_ID+r"$") + +def check_cf_standard_name(test_val, prop_dict, error): + """Return if a valid CF Standard Name, otherwise, None + http://cfconventions.org/Data/cf-standard-names/docs/guidelines.html + if is True, raise an Exception if is not valid. + >>> check_cf_standard_name("hi_mom", None, False) + 'hi_mom' + >>> check_cf_standard_name("hi mom", None, False) + + >>> check_cf_standard_name("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'hi_mom' is not a valid CF Standard Name + >>> check_cf_standard_name("", None, False) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: CCPP Standard Name cannot be blank + >>> check_cf_standard_name("_hi_mom", None, False) + + >>> check_cf_standard_name("2pac", None, False) + + >>> check_cf_standard_name("Agood4tranID", None, False) + 'agood4tranid' + >>> check_cf_standard_name("agoodcfid", None, False) + 'agoodcfid' + """ + if len(test_val) == 0: + raise CCPPError("CCPP Standard Name cannot be blank") + else: + match = __CFID_RE.match(test_val) + # end if + if match is None: + if error: + errmsg = "'{}' is not a valid CCPP Standard Name" + raise CCPPError(errmsg.format(test_val)) + else: + test_val = None + # end if + else: + test_val = test_val.lower() + # end if + return test_val + +######################################################################## + +### Fortran-specific parsing helper variables and functions + +######################################################################## + +# FORTRAN_ID is a string representing the regular expression for Fortran names +FORTRAN_ID = r"([A-Za-z][A-Za-z0-9_]*)" +__FID_RE = re.compile(FORTRAN_ID+r"$") +# Note that the scalar array reference expressions below are not really for +# scalar references because a colon can be a placeholder, unlike in Fortran code +__FORTRAN_AID = r"(?:[A-Za-z][A-Za-z0-9_]*)" +__FORT_INT = r"[0-9]+" +__FORT_DIM = r"(?:"+__FORTRAN_AID+r"|[:]|"+__FORT_INT+r")" +__REPEAT_DIM = r"(?:,\s*"+__FORT_DIM+r"\s*)" +__FORTRAN_SCALAR_ARREF = r"[(]\s*("+__FORT_DIM+r"\s*"+__REPEAT_DIM+r"{0,6})[)]" +# FORTRAN_SCALAR_REF: Pattern of a valid Fortran array reference +# NB: Only allows symbols, no expressions and/or function calls +FORTRAN_SCALAR_REF = r"(?:"+FORTRAN_ID+r"\s*"+__FORTRAN_SCALAR_ARREF+r")" +FORTRAN_SCALAR_REF_RE = re.compile(FORTRAN_SCALAR_REF+r"$") +# FORTRAN_FUNCTION_REF: A Fortran function reference +# NB: Currenly does not support function arguments +FORTRAN_FUNCTION_REF = r"(?:"+FORTRAN_ID+r"\s*[(]\s*[)])" +FORTRAN_FUNCTION_REF_RE = re.compile(FORTRAN_FUNCTION_REF) +FORTRAN_INTRINSIC_TYPES = ["integer", "real", "logical", "complex", + "double precision", "character"] +FORTRAN_DP_RE = re.compile(r"(?i)double\s*precision") +FORTRAN_TYPE_RE = re.compile(r"(?i)type\s*\(\s*("+FORTRAN_ID+r")\s*\)") + +_REGISTERED_FORTRAN_DDT_NAMES = ["ccpp_constituent_prop_ptr_t"] + +######################################################################## + +def check_fortran_id(test_val, prop_dict, error, max_len=0): + """Return if a valid Fortran identifier, otherwise, None + If > 0, must not be longer than . + if is True, raise an Exception if is not valid. + >>> check_fortran_id("hi_mom", None, False) + 'hi_mom' + >>> check_fortran_id("hi_mom", None, False, max_len=5) + + >>> check_fortran_id("hi_mom", None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'hi_mom' is too long (> 5 chars) + >>> check_fortran_id("hi mom", None, False) + + >>> check_fortran_id("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'hi_mom' is not a valid Fortran identifier + >>> check_fortran_id("", None, False) + + >>> check_fortran_id("_hi_mom", None, False) + + >>> check_fortran_id("2pac", None, False) + + >>> check_fortran_id("Agood4tranID", None, False) + 'Agood4tranID' + """ + match = __FID_RE.match(test_val) + if match is None: + if error: + raise CCPPError("'{}' is not a valid Fortran identifier".format(test_val)) + else: + test_val = None + # end if + elif (max_len > 0) and (len(test_val) > max_len): + if error: + raise CCPPError("'{}' is too long (> {} chars)".format(test_val, max_len)) + else: + test_val = None + # end if + # end if + return test_val + +######################################################################## + +def fortran_list_match(test_str): + """Check if could be a list of Fortran expressions. + The list must be enclosed in parentheses and separated by commas. + If the list appears okay, return the items (for further checking) + >>> fortran_list_match('(ccpp_constant_one:dim1)') + ['ccpp_constant_one:dim1'] + >>> fortran_list_match('(foo, bar)') + ['foo', 'bar'] + >>> fortran_list_match('()') + [''] + >>> fortran_list_match('(foo, ,)') + + >>> fortran_list_match('foo, bar') + + >>> fortran_list_match('(foo, bar') + + """ + parens, parene = check_balanced_paren(test_str) + if (parens >= 0) and (parene > parens): + litems = [x.strip() for x in test_str[parens+1:parene].split(',')] + if (len(litems) > 1) and (min([len(x) for x in litems]) == 0): + litems = None + # end if + else: + litems = None + # end if + return litems + +######################################################################## + +def check_fortran_ref(test_val, prop_dict, error, max_len=0): + """Return if a valid simple Fortran variable reference, + otherwise, None. A simple Fortran variable reference is defined as + a scalar id or a scalar array reference. + if is True, raise an Exception if is not valid. + >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(1) + 'foo' + >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2) + 'bar, baz ' + >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2).split(',')[0].strip() + 'bar' + >>> FORTRAN_SCALAR_REF_RE.match("foo( :, baz )").group(2).split(',')[0].strip() + ':' + >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2).split(',')[1].strip() + 'baz' + >>> check_fortran_ref("hi_mom", None, False) + 'hi_mom' + >>> check_fortran_ref("hi_mom", None, False, max_len=5) + + >>> check_fortran_ref("hi_mom", None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'hi_mom' is too long (> 5 chars) + >>> check_fortran_ref("hi mom", None, False) + + >>> check_fortran_ref("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'hi_mom' is not a valid Fortran identifier + >>> check_fortran_ref("", None, False) + + >>> check_fortran_ref("_hi_mom", None, False) + + >>> check_fortran_ref("2pac", None, False) + + >>> check_fortran_ref("Agood4tranID", None, False) + 'Agood4tranID' + >>> check_fortran_ref("foo(bar)", None, False) + 'foo(bar)' + >>> check_fortran_ref("foo( bar, baz )", None, False) + 'foo( bar, baz )' + >>> check_fortran_ref("foo( :, baz )", None, False) + 'foo( :, baz )' + >>> check_fortran_ref("foo( bar, )", None, False) + + >>> check_fortran_ref("foo( bar, )", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'foo( bar, )' is not a valid Fortran scalar reference + >>> check_fortran_ref("foo()", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'foo()' is not a valid Fortran scalar reference + >>> check_fortran_ref("foo(bar, bazz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'bazz' is too long (> 3 chars) in foo(bar, bazz) + >>> check_fortran_ref("foo(barr, baz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'bazr' is too long (> 3 chars) in foo(barr, baz) + >>> check_fortran_ref("fooo(bar, baz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'foo' is too long (> 3 chars) in fooo(bar, baz) + """ + idval = check_fortran_id(test_val, prop_dict, False, max_len=max_len) + if idval is None: + match = FORTRAN_SCALAR_REF_RE.match(test_val) + if match is None: + if error: + emsg = "'{}' is not a valid Fortran scalar reference" + raise CCPPError(emsg.format(test_val)) + else: + test_val = None + # end if + elif max_len > 0: + tokens = test_val.strip().rstrip(')').split('(') + tokens = [tokens[0].strip()] + [x.strip() + for x in tokens[1].split(',')] + for token in tokens: + if len(token) > max_len: + if error: + emsg = "'{}' is too long (> {} chars) in {}" + raise CCPPError(emsg.format(token, max_len, test_val)) + else: + test_val = None + break + # end if + # end if + # end for + # end if + # end if + return test_val + +######################################################################## + +def check_local_name(test_val, prop_dict, error, max_len=0): + """Return if a valid simple Fortran variable reference, + or Fortran constant, otherwise, None. + A simple Fortran variable reference is defined as a scalar id or a + scalar array reference. + A constant is only valid if is not None, the 'protected' + property is present and True, and the 'type' property matches the + type of . + if is True, raise an Exception if is not valid. + >>> check_local_name("hi_mom", None, error=False) + 'hi_mom' + >>> check_local_name('122', {'protected':True,'type':'integer'}, error=False) + '122' + >>> check_local_name('122', None, error=False) + + >>> check_local_name('122', {}, error=False) + + >>> check_local_name('122', {'protected':False,'type':'integer'}, error=False) + + >>> check_local_name('122', {'protected':True,'type':'real'}, error=False) + + >>> check_local_name('-122.e4', {'protected':True,'type':'real'}, error=False) + '-122.e4' + >>> check_local_name('-122.', {'protected':True,'type':'real','kind':'kp'}, error=False) + + >>> check_local_name('-122._kp', {'protected':True,'type':'real','kind':'kp'}, error=False) + '-122._kp' + >>> check_local_name('q(:,:,index_of_water_vapor_specific_humidity)', {}, error=False) + 'q(:,:,index_of_water_vapor_specific_humidity)' + """ + valid_val = None + # First check for a constant + if (prop_dict is not None) and ('protected' in prop_dict): + protected = prop_dict['protected'] + else: + protected = False + # end if + if (prop_dict is not None) and ('type' in prop_dict): + vtype = prop_dict['type'] + else: + vtype = "" + # end if + if (prop_dict is not None) and ('kind' in prop_dict): + kind = prop_dict['kind'] + else: + kind = "" + # end if + if protected and vtype and check_fortran_literal(test_val, vtype, kind): + valid_val = test_val + # end if + if valid_val is None: + valid_val = check_fortran_ref(test_val, prop_dict, error, max_len=max_len) + # end if + return valid_val + + +######################################################################## + +def check_fortran_intrinsic(typestr, error=False): + """Return if a valid Fortran intrinsic type, otherwise, None + if is True, raise an Exception if is not valid. + >>> check_fortran_intrinsic("real", error=False) + 'real' + >>> check_fortran_intrinsic("complex") + 'complex' + >>> check_fortran_intrinsic("integer") + 'integer' + >>> check_fortran_intrinsic("InteGer") + 'InteGer' + >>> check_fortran_intrinsic("logical") + 'logical' + >>> check_fortran_intrinsic("character") + 'character' + >>> check_fortran_intrinsic("double precision") + 'double precision' + >>> check_fortran_intrinsic("double precision") + 'double precision' + >>> check_fortran_intrinsic("doubleprecision") + 'doubleprecision' + >>> check_fortran_intrinsic("char", error=True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'char' is not a valid Fortran type + >>> check_fortran_intrinsic("int") + + >>> check_fortran_intrinsic("char", error=False) + + >>> check_fortran_intrinsic("type") + + >>> check_fortran_intrinsic("complex(kind=r8)") + + """ + chk_type = typestr.strip().lower() + match = chk_type in FORTRAN_INTRINSIC_TYPES + if (not match) and (chk_type[0:6] == 'double'): + # Special case for double precision + match = FORTRAN_DP_RE.match(chk_type) is not None + # End if + if not match: + if error: + raise CCPPError("'{}' is not a valid Fortran type".format(typestr)) + else: + typestr = None + # end if + # end if + return typestr + +######################################################################## + +def check_fortran_type(typestr, prop_dict, error): + """Return if a valid Fortran type, otherwise, None + if is True, raise an Exception if is not valid. + >>> check_fortran_type("real", None, False) + 'real' + >>> check_fortran_type("integer", None, False) + 'integer' + >>> check_fortran_type("InteGer", None, False) + 'InteGer' + >>> check_fortran_type("character", None, False) + 'character' + >>> check_fortran_type("double precision", None, False) + 'double precision' + >>> check_fortran_type("double precision", None, False) + 'double precision' + >>> check_fortran_type("doubleprecision", None, False) + 'doubleprecision' + >>> check_fortran_type("complex", None, False) + 'complex' + >>> check_fortran_type("char", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'char' is not a valid Fortran type + >>> check_fortran_type("int", None, False) + + >>> check_fortran_type("char", {}, False) + + >>> check_fortran_type("type", None, False) + + >>> check_fortran_type("type", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'type' is not a valid derived Fortran type + >>> check_fortran_type("type(hi mom)", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'type(hi mom)' is not a valid derived Fortran type + """ + dt = "" + match = check_fortran_intrinsic(typestr, error=False) + if match is None: + match = registered_fortran_ddt_name(typestr) + dt = " derived" + # end if + if match is None: + if error: + emsg = "'{}' is not a valid{} Fortran type" + raise CCPPError(emsg.format(typestr, dt)) + else: + typestr = None + # end if + # end if + return typestr + +######################################################################## + +def check_fortran_literal(value, typestr, kind): + """Return True iff is a valid Fortran literal of type, . + Note: no attempt is made to handle the older D syntax for real literals. + To promote clean coding, real values MUST have a decimal point, however, + this check is not available for the complex type so we just require + the two components to either both be integers or both be reals. + If is not an empty string, it is required to be present (i.e., if + == 'kind_phys', should be of the form, 123.4_kind_phys) + >>> check_fortran_literal("123", "integer", "") + True + >>> check_fortran_literal("123", "INTEGER", "") + True + >>> check_fortran_literal("-123", "integer", "") + True + >>> check_fortran_literal("+123", "integer", "") + True + >>> check_fortran_literal("+123", "integer", "kind_int") + False + >>> check_fortran_literal("+123_kind_int", "integer", "kind_int") + True + >>> check_fortran_literal("+123_int", "integer", "kind_int") + False + >>> check_fortran_literal("123", "real", "") + False + >>> check_fortran_literal("123.", "real", "") + True + >>> check_fortran_literal("123.45", "real", "kind_phys") + False + >>> check_fortran_literal("123.45_8", "real", "kind_phys") + False + >>> check_fortran_literal("123.45_kind_phys", "real", "kind_phys") + True + >>> check_fortran_literal("123", "double precision", "") + False + >>> check_fortran_literal("123.", "doubleprecision", "") + True + >>> check_fortran_literal("123.45", "double precision", "kind_phys") + False + >>> check_fortran_literal("123.45_8", "doubleprecision", "kind_phys") + False + >>> check_fortran_literal("123.45_kp", "doubleprecision", "kp") + True + >>> check_fortran_literal("123", "logical", "") + False + >>> check_fortran_literal(".true.", "logical", "") + True + >>> check_fortran_literal(".false.", "logical", "") + True + >>> check_fortran_literal("T", "logical", "") + False + >>> check_fortran_literal("F", "logical", "") + False + >>> check_fortran_literal(".TRUE.", "logical", "kind_log") + False + >>> check_fortran_literal(".TRUE._kind_log", "logical", "kind_log") + True + >>> check_fortran_literal("(123.,456.)", "complex", "") + True + >>> check_fortran_literal("(123. , 456.)", "complex", "") + True + >>> check_fortran_literal("(123.,456", "complex", "") + False + >>> check_fortran_literal("(123. , 456.)", "complex", "kp") + False + >>> check_fortran_literal("(123._kp , 456)", "complex", "kp") + False + >>> check_fortran_literal("(123._kp , 456._kp)", "complex", "kp") + True + >>> check_fortran_literal("'hi mom'", "character", "") + True + >>> check_fortran_literal("'hi mom", "character", "") + False + >>> check_fortran_literal('"hi mom"', "character", "") + True + >>> check_fortran_literal('"hi""mom"', "character", "") + True + >>> check_fortran_literal('"hi" "mom"', "character", "") + False + >>> check_fortran_literal("'hi''there''mom'", "character", "") + True + >>> check_fortran_literal("'hi mom'", "character", "kc") + False + >>> check_fortran_literal("kc_'hi mom'", "character", "kc") + True + >>> check_fortran_literal("123._kp", "float", "kp") #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ParseInternalError: ERROR: 'float' is not a Fortran intrinsic type + """ + valid = True + if FORTRAN_DP_RE.match(typestr.strip()) is not None: + vtype = 'real' + else: + vtype = typestr.lower() + # end if + # Check complex first + if vtype == 'complex': + cvals = value.strip().split(',') + if len(cvals) == 2: + tp = 'integer' + if ('.' in cvals[0]) and ('.' in cvals[1]): + tp = 'real' + elif ('.' in cvals[0]) or ('.' in cvals[1]): + valid = False + # end if + if (cvals[0][0] == '(') and (cvals[1][-1] == ')'): + valid = valid and check_fortran_literal(cvals[0][1:], tp, kind) + valid = valid and check_fortran_literal(cvals[1][:-1], tp, kind) + else: + valid = False + # end if + else: + valid = False + elif valid: + vparts = value.strip().split('_') + if vtype == 'character': + if len(vparts) > 1: + val = vparts[-1] + vkind = '_'.join(vparts[0:-1]) + else: + val = vparts[0] + vkind = '' + # end if + else: + val = vparts[0] + if len(vparts) > 1: + vkind = '_'.join(vparts[1:]) + else: + vkind = '' + # end if + # end if + if vkind != kind.lower(): + valid = False + # end if, kind is okay, check value + if valid and (vtype == 'integer'): + try: + vtest = int(val) + except ValueError as ve: + valid = False + # End try + elif valid and (vtype == 'real'): + if '.' not in val: + valid = False + else: + try: + vtest = float(val) + except ValueError as ve: + valid = False + # End try + # end if + elif valid and (vtype == 'logical'): + valid = (val.upper() == '.TRUE.') or (val.upper() == '.FALSE.') + elif valid and (vtype == 'character'): + sep = val[0] + cparts = val.split(sep) + # We must have balanced delimiters + if len(cparts)%2 == 0: + valid = False + else: + for index in range(len(cparts)): + if (index%2 == 0) and (len(cparts[index]) > 0): + valid = False + break + # end if + # end for + # end if (else okay) + elif valid: + errmsg = "ERROR: '{}' is not a Fortran intrinsic type" + raise ParseInternalError(errmsg.format(typestr)) + # end if (no else) + # end if + return valid + +def check_default_value(test_val, prop_dict, error): + """Return if a valid default value for a CCPP field, + otherwise, None. + If is True, raise an Exception if is not valid. + A valid value is determined by the 'type' of the variable. It is an + error for there to be no 'type' property in . + >>> check_default_value('314', {'type':'integer'}, False) + '314' + >>> check_default_value('314', {'type':'integer'}, True) + '314' + >>> check_default_value('314', {'type':'integer', 'kind':'ikind'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 314 is not a valid Fortran integer of kind, ikind + >>> check_default_value('314_ikind', {'type':'integer', 'kind':'ikind'}, True) + '314_ikind' + >>> check_default_value('314', {'type':'real'}, False) + + >>> check_default_value('314', {'type':'real'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 314 is not a valid Fortran real + >>> check_default_value('3.14', {'type':'real'}, False) + '3.14' + >>> check_default_value('314', {'tipe':'integer'}, False) + + >>> check_default_value('314', {'local_name':'foo'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: foo does not have a 'type' attribute + >>> check_default_value('314', {'tipe':'integer'}, False) + + >>> check_default_value('314', None, True) + '314' + """ + valid = None + if prop_dict and ('type' in prop_dict): + valid = test_val + var_type = prop_dict['type'].lower().strip() + if 'kind' in prop_dict: + vkind = prop_dict['kind'].lower().strip() + else: + vkind = '' + # end if + if not check_fortran_literal(test_val, var_type, vkind): + valid = None + if error: + emsg = '{} is not a valid Fortran {}' + if vkind: + emsg += ' of kind, {}' + raise CCPPError(emsg.format(test_val, var_type, vkind)) + # end if + # end if (no else, is okay) + elif prop_dict is None: + # Special case for checks during parsing, always pass + valid = test_val + elif error: + emsg = "{} does not have a 'type' attribute" + if 'local_name' in prop_dict: + lname = prop_dict['local_name'] + else: + lname = 'UNKNOWN' + # end if + raise CCPPError(emsg.format(lname)) + # end if + return valid + +def check_valid_values(test_val, prop_dict, error): + """Return if a valid 'valid_values' attribute value, + otherwise, None. + If is True, raise an Exception if is not valid. + """ + raise ParseInternalError("NOT IMPLEMENTED") + +def check_diagnostic_fixed(test_val, prop_dict, error): + """Return if a valid descriptor for a CCPP diagnostic, + otherwise, None. + If is True, raise an Exception if is not valid. + A fixed diagnostic name is any Fortran identifier, however, it is + an error to specify both 'diagnostic_name' and 'diagnostic_name_fixed'. + >>> check_diagnostic_fixed("foo", {'diagnostic_name_fixed' : 'foo'}, False) + 'foo' + >>> check_diagnostic_fixed("foo", {'diagnostic_name_fixed' : 'foo'}, True) + 'foo' + >>> check_diagnostic_fixed("foo", {'diagnostic_name' : 'foo'}, False) + + >>> check_diagnostic_fixed("foo", {'diagnostic_name':'','local_name':'hi','standard_name':'mom'}, True) + 'foo' + >>> check_diagnostic_fixed("foo", {'diagnostic_name':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: hi (mom) cannot have both 'diagnostic_name' and 'diagnostic_name_fixed' attributes + >>> check_diagnostic_fixed("2foo", {'diagnostic_name_fixed':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '2foo' (hi) is not a valid fixed diagnostic name + """ + valid = test_val + if (prop_dict and ('diagnostic_name' in prop_dict) and + prop_dict['diagnostic_name']): + valid = None + if error: + emsg = "{} ({}) cannot have both 'diagnostic_name' and " + emsg += "'diagnostic_name_fixed' attributes" + if 'local_name' in prop_dict: + lname = prop_dict['local_name'] + else: + lname = 'UNKNOWN' + # end if + if 'standard_name' in prop_dict: + sname = prop_dict['standard_name'] + else: + sname = 'UNKNOWN' + # end if + raise CCPPError(emsg.format(lname, sname)) + # end if + elif check_fortran_id(test_val, prop_dict, False) is None: + valid = None + if error: + emsg = "'{}' ({}) is not a valid fixed diagnostic name" + if 'local_name' in prop_dict: + lname = prop_dict['local_name'] + else: + lname = 'UNKNOWN' + # end if + raise CCPPError(emsg.format(test_val, lname)) + # end if + # end if + return valid + +######################################################################## + +_DIAG_PRE = r"("+FORTRAN_ID+")?" +_DIAG_SUFF = r"([_0-9A-Za-z]+)?" +_DIAG_PROP = r"((\${process}|\${scheme_name})"+_DIAG_SUFF+r")" +_DIAG_RE = re.compile(_DIAG_PRE+_DIAG_PROP+r"?$") + +def check_diagnostic_id(test_val, prop_dict, error): + """Return if a valid descriptor for a CCPP diagnostic, + otherwise, None. + If is True, raise an Exception if is not valid. + A diagnostic name is a Fortran identifier with the optional + addition of one variable substitution. + A variable substitution is a substring of the form of either: + ${process}: The scheme process name will be substituted for this + substring. If this substring is included, it is an error for + there to be no process specified by the scheme (although this + error cannot be detected by this routine). + ${scheme_name}: The scheme name will be substituted for this substring. + It is an error to specify both 'diagnostic_name' and + 'diagnostic_name_fixed'. + >>> check_diagnostic_id("foo", {'diagnostic_name' : 'foo'}, False) + 'foo' + >>> check_diagnostic_id("foo", {'diagnostic_name' : 'foo'}, True) + 'foo' + >>> check_diagnostic_id("foo", {'diagnostic_name_fixed' : 'foo'}, False) + + >>> check_diagnostic_id("foo_${process}", {}, False) + 'foo_${process}' + >>> check_diagnostic_id("foo_${process}_2bad", {}, False) + 'foo_${process}_2bad' + >>> check_diagnostic_id("${process}_2bad", {}, False) + '${process}_2bad' + >>> check_diagnostic_id("foo_${scheme_name}", {}, False) + 'foo_${scheme_name}' + >>> check_diagnostic_id("foo_${scheme_name}_2bad", {}, False) + 'foo_${scheme_name}_2bad' + >>> check_diagnostic_id("${scheme_name}_suff", {}, False) + '${scheme_name}_suff' + >>> check_diagnostic_id("pref_${scheme}_suff", {}, False) + + >>> check_diagnostic_id("pref_${scheme_name_suff", {}, False) + + >>> check_diagnostic_id("pref_$scheme_name}_suff", {}, False) + + >>> check_diagnostic_id("pref_{scheme_name}_suff", {}, False) + + >>> check_diagnostic_id("foo", {'diagnostic_name_fixed':'','local_name':'hi','standard_name':'mom'}, True) + 'foo' + >>> check_diagnostic_id("foo", {'diagnostic_name_fixed':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: hi (mom) cannot have both 'diagnostic_name' and 'diagnostic_name_fixed' attributes + >>> check_diagnostic_id("2foo", {'diagnostic_name':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '2foo' (hi) is not a valid diagnostic name + """ + if (prop_dict and ('diagnostic_name_fixed' in prop_dict) and + prop_dict['diagnostic_name_fixed']): + valid = None + if error: + emsg = "{} ({}) cannot have both 'diagnostic_name' and " + emsg += "'diagnostic_name_fixed' attributes" + if 'local_name' in prop_dict: + lname = prop_dict['local_name'] + else: + lname = 'UNKNOWN' + # end if + if 'standard_name' in prop_dict: + sname = prop_dict['standard_name'] + else: + sname = 'UNKNOWN' + # end if + raise CCPPError(emsg.format(lname, sname)) + # end if + else: + match = _DIAG_RE.match(test_val) + if match is None: + valid = None + if error: + emsg = "'{}' is not a valid diagnostic_name value" + raise CCPPError(emsg.format(test_val)) + # end if + else: + valid = test_val + # end if + # end if + return valid + +######################################################################## + +def check_molar_mass(test_val, prop_dict, error): + """Return if valid molar mass, otherwise, None + if is True, raise an Exception if is not valid. + >>> check_molar_mass('1', None, True) + 1.0 + >>> check_molar_mass('1.0', None, True) + 1.0 + >>> check_molar_mass('1.0', None, False) + 1.0 + >>> check_molar_mass('-1', None, False) + + >>> check_molar_mass('-1.0', None, False) + + >>> check_molar_mass('string', None, False) + + >>> check_molar_mass(10001, None, False) + + >>> check_molar_mass('-1', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '-1' is not a valid molar mass + >>> check_molar_mass('-1.0', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '-1.0' is not a valid molar mass + >>> check_molar_mass('string', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '-1.0' is not a valid molar mass + >>> check_molar_mass(10001, None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: '10001' is not a valid molar mass + """ + # Check if input value is an int or float + try: + test_val = float(test_val) + if test_val < 0.0 or test_val > _MAX_MOLAR_MASS: + if error: + raise CCPPError(f"{test_val} is not a valid molar mass") + else: + test_val = None + # end if + # end if + except: + # not an int or float, conditionally throw error + if error: + raise CCPPError(f"{test_val} is invalid; not a float or int") + else: + test_val=None + # end if + # end try + return test_val + +######################################################################## + +def check_balanced_paren(string, start=0, error=False): + """Return indices delineating a balance set of parentheses. + Parentheses in character context do not count. + Left parenthesis search begins at . + Return start and end indices if found + If no parentheses are found, return (-1, -1). + If a left parenthesis is found but no balancing right, return (begin, -1) + where begin is the index where the left parenthesis was found. + If error is True, raise a CCPPError. + >>> check_balanced_paren("foo") + (-1, -1) + >>> check_balanced_paren("(foo, bar)") + (0, 9) + >>> check_balanced_paren("( (foo, bar) )", start=1) + (2, 11) + >>> check_balanced_paren("(size(foo,1), qux)") + (0, 17) + >>> check_balanced_paren("(foo('bar()'))") + (0, 13) + >>> check_balanced_paren("(foo('bar()')") + (0, -1) + >>> check_balanced_paren("(foo('bar()')", error=True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: ERROR: Unbalanced parenthesis in '(foo('bar()')' + """ + index = start + begin = -1 + end = -1 + depth = 0 + inchar = None + str_len = len(string) + while index < str_len: + if (string[index] == '"') or (string[index] == "'"): + if inchar == string[index]: + inchar = None + elif inchar is None: + inchar = string[index] + # else in character context, keep going + # end if + elif inchar is not None: + # In character context, keep going + pass + elif string[index] == '(': + if depth == 0: + begin = index + # end if + depth = depth + 1 + if depth == 0: + break + # end if + elif string[index] == ')': + depth = depth - 1 + if depth == 0: + end = index + break + # end if + # else just keep going + # end if + index = index + 1 + # End while + if (begin >= 0) and (end < 0) and error: + raise CCPPError("ERROR: Unbalanced parenthesis in '{}'".format(string)) + # end if + return begin, end + +######################################################################## + +def registered_fortran_ddt_names(): + return _REGISTERED_FORTRAN_DDT_NAMES + +######################################################################## + +def registered_fortran_ddt_name(name): + if name in _REGISTERED_FORTRAN_DDT_NAMES: + return name + else: + return None + +######################################################################## + +def register_fortran_ddt_name(name): + if name not in _REGISTERED_FORTRAN_DDT_NAMES: + _REGISTERED_FORTRAN_DDT_NAMES.append(name) + +######################################################################## + +if __name__ == "__main__": + # pylint: disable=ungrouped-imports + import doctest + # pylint: enable=ungrouped-imports + fail, _ = doctest.testmod() + sys.exit(fail) +# end if diff --git a/capgen-ng/metadata/parse_tools/parse_log.py b/capgen-ng/metadata/parse_tools/parse_log.py new file mode 100644 index 00000000..fec65161 --- /dev/null +++ b/capgen-ng/metadata/parse_tools/parse_log.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +"""Shared logger utilities for parse processes. + +Copied from scripts/parse_tools/parse_log.py. +""" + +import logging + + +def init_log(name, level=None): + """Initialize and return a named logger. + + Defaults to WARNING level when *level* is not specified and the logger + has no existing level set. + + >>> logger = init_log('test_logger') + >>> logger.name + 'test_logger' + """ + logger = logging.getLogger(name) + llevel = logger.getEffectiveLevel() + if level is None and llevel == logging.NOTSET: + logger.setLevel(logging.WARNING) + elif level: + logger.setLevel(level) + set_log_to_stdout(logger) + return logger + + +def set_log_level(logger, level): + """Set *logger*'s level to *level*.""" + logger.setLevel(level) + + +def remove_handlers(logger): + """Remove all handlers from *logger*.""" + for handler in list(logger.handlers): + logger.removeHandler(handler) + + +def set_log_to_stdout(logger): + """Direct *logger* output to standard output.""" + remove_handlers(logger) + logger.addHandler(logging.StreamHandler()) + + +def set_log_to_null(logger): + """Suppress all *logger* output.""" + remove_handlers(logger) + logger.addHandler(logging.NullHandler()) + + +def set_log_to_file(logger, filename): + """Direct *logger* output to *filename*.""" + remove_handlers(logger) + logger.addHandler(logging.FileHandler(filename)) + + +def flush_log(logger): + """Flush all pending output from *logger*.""" + for handler in list(logger.handlers): + handler.flush() + + +def verbose(logger): + """Return True if *logger* is at DEBUG level.""" + return logger.isEnabledFor(logging.DEBUG) diff --git a/capgen-ng/metadata/parse_tools/parse_object.py b/capgen-ng/metadata/parse_tools/parse_object.py new file mode 100644 index 00000000..1095c210 --- /dev/null +++ b/capgen-ng/metadata/parse_tools/parse_object.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""A module for the base, ParseObject class""" + +# CCPP framework imports +from .parse_source import ParseContext, CCPPError, context_string + +######################################################################## + +class ParseObject(ParseContext): + """ParseObject is a simple class that keeps track of an object's + place in a file and safely produces lines from an array of lines + >>> ParseObject('foobar.F90', []) #doctest: +ELLIPSIS + + >>> ParseObject('foobar.F90', []).filename + 'foobar.F90' + >>> ParseObject('foobar.F90', ["##hi mom",], line_start=1).curr_line() + (None, 1) + >>> ParseObject('foobar.F90', ["first line","## hi mom"], line_start=1).curr_line() + ('## hi mom', 1) + >>> ParseObject('foobar.F90', ["##hi mom",], line_start=1).next_line() + (None, 1) + >>> ParseObject('foobar.F90', ["##first line","## hi mom"], line_start=1).next_line() + ('## hi mom', 1) + >>> ParseObject('foobar.F90', ["## hi \\\\","mom"], line_start=0).next_line() + ('## hi mom', 0) + >>> ParseObject('foobar.F90', ["line1","##line2","## hi mom"], line_start=2).next_line() + ('## hi mom', 2) + >>> ParseObject('foobar.F90', ["## hi \\\\","there \\\\","mom"], line_start=0).next_line() + ('## hi there mom', 0) + >>> ParseObject('foobar.F90', ["!! line1","!! hi mom"], line_start=1).next_line() + ('!! hi mom', 1) + """ + + _max_errors = 32 + + def __init__(self, filename, lines_in, line_start=0): + """Initialize this ParseObject""" + self.__lines = lines_in + self.__line_start = line_start + self.__line_end = line_start + self.__line_next = line_start + self.__num_lines = len(self.__lines) + self.__error_message = "" + self.__num_errors = 0 + super().__init__(linenum=line_start, filename=filename) + + @property + def first_line_num(self): + """Return the first line parsed""" + return self.__line_start + + @property + def last_line_num(self): + """Return the last line parsed""" + return self.__line_end + + def valid_line(self): + """Return True if the current line is valid""" + return (self.line_num >= 0) and (self.line_num < self.__num_lines) + + @property + def error_message(self): + """Return this object's error message""" + return self.__error_message + + def curr_line(self): + """Return the current line (if valid) and the current line number. + If the current line is invalid, return None""" + valid_line = self.valid_line() + _curr_line = None + _my_curr_lineno = self.line_num + if valid_line: + try: + _curr_line = self.__lines[self.line_num].rstrip() + self.__line_next = self.line_num + 1 + self.__line_end = self.__line_next + except CCPPError: + self.add_syntax_err("line", self.line_num) + valid_line = False + # end if + # We allow continuation self.__lines (ending with a single backslash) + if valid_line and _curr_line.endswith('\\'): + next_line, _ = self.next_line() + if next_line is None: + # We ran out of lines, just strip the backslash + _curr_line = _curr_line[0:len(_curr_line)-1] + else: + _curr_line = _curr_line[0:len(_curr_line)-1] + next_line + # end if + # end if + # curr_line should not change the line number + self.line_num = _my_curr_lineno + return _curr_line, self.line_num + + def next_line(self): + """Return the next line in our file (if valid) and the next line's + line number. If the next line is not valid, return None""" + self.line_num = self.__line_next + return self.curr_line() + + def peek_line(self, line_num): + """Return the text of without advancing to that line. + if is out of bounds, return None.""" + if (line_num >= 0) and (line_num < len(self.__lines)): + return self.__lines[line_num] + # end if + return None + + def add_syntax_err(self, token_type, token=None, skip_context=False): + """Add a ParseSyntaxError-type message to this object's error + log, separating it from any previous messages with a newline.""" + if self.__error_message: + if self.__num_errors == self._max_errors: + self.__error_message += '\nMaximum number of errors exceeded' + self.line_num = self.__num_lines # Intentionally walk off end + self.__line_next = self.line_num + elif self.__num_errors > self._max_errors: + # Oops, something went wrong, panic! + raise CCPPError(self.error_message) + # end if + self.__error_message += '\n' + # end if + if self.__num_errors < self._max_errors: + if skip_context: + cstr = "" + else: + cstr = context_string(self) + # end if + if token is None: + self.__error_message += "{}{}".format(token_type, cstr) + else: + self.__error_message += "Invalid {}, '{}'{}".format(token_type, + token, cstr) + # end if + # end if + self.__num_errors += 1 + + def reset_pos(self, line_start=0): + """Attempt to set the current file position to . + If is out of bounds, raise an exception.""" + if (line_start < 0) or (line_start >= self.__num_lines): + emsg = 'Attempt to reset_pos to non-existent line, {}' + raise CCPPError(emsg.format(line_start)) + # end if + self.line_num = line_start + self.__line_next = line_start + + def write_line(self, line_num, line): + """Overwrite line, with . + If is out of bounds, raise an exception.""" + if (line_num < 0) or (line_num >= len(self.__lines)): + emsg = 'Attempt to write non-existent line, {}' + raise CCPPError(emsg.format(line_num)) + # end if + self.__lines[line_num] = line + + def __del__(self): + """Attempt to cleanup memory used by this object""" + try: + del self.__lines + del self.regions + except Exception: + pass # Python does not guarantee much about __del__ conditions + # end try + +######################################################################## diff --git a/capgen-ng/metadata/parse_tools/parse_source.py b/capgen-ng/metadata/parse_tools/parse_source.py new file mode 100644 index 00000000..53e5fb9c --- /dev/null +++ b/capgen-ng/metadata/parse_tools/parse_source.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 + +"""Parsing primitives: context tracking, exception types, and source tagging. + +Copied from scripts/parse_tools/parse_source.py and adapted for the +capgen-ng package structure. +""" + +from collections.abc import Iterable +import copy +import os.path +import logging + + +class _StdNameCounter: + """Global counter for generating unique placeholder standard names.""" + + __SNAME_NUM = 0 + + @classmethod + def new_stdname_number(cls): + """Increment and return the counter.""" + _StdNameCounter.__SNAME_NUM += 1 + return _StdNameCounter.__SNAME_NUM + + @classmethod + def reset_stdname_counter(cls, reset_val=0): + """Reset the counter to *reset_val*.""" + _StdNameCounter.__SNAME_NUM = reset_val + + +def unique_standard_name(): + """Return a unique placeholder standard name. + + Used during parsing when a real standard name is not yet known. + + >>> n1 = unique_standard_name() + >>> n2 = unique_standard_name() + >>> n1 != n2 + True + >>> n1.startswith('enter_standard_name_') + True + """ + return 'enter_standard_name_{}'.format(_StdNameCounter.new_stdname_number()) + + +def reset_standard_name_counter(): + """Reset the unique_standard_name counter to zero.""" + _StdNameCounter.reset_stdname_counter() + + +def context_string(context=None, with_comma=True, nodir=False): + """Return a human-readable location string from *context*. + + Parameters + ---------- + context : ParseContext or None + Parsing location. ``None`` returns an empty string. + with_comma : bool + Prepend ``', at '`` or ``', in '`` when *context* is given. + nodir : bool + Strip the directory portion of the filename. + + Returns + ------- + str + + >>> context_string() + '' + >>> context_string(context=ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False) + 'dir/source.F90:33' + >>> context_string(context=ParseContext(linenum=32, filename="dir/source.F90"), with_comma=True) + ', at dir/source.F90:33' + >>> context_string(context=ParseContext(filename="dir/source.F90"), with_comma=False) + 'dir/source.F90' + >>> context_string(context=ParseContext(filename="dir/source.F90"), with_comma=True) + ', in dir/source.F90' + >>> context_string(context=ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False, nodir=True) + 'source.F90:33' + """ + if context is None: + return '' + if context.line_num < 0: + where_str = 'in ' + else: + where_str = 'at ' + comma = ', ' if with_comma else '' + if not with_comma: + where_str = '' + spec = '{ctx:nodir}' if nodir else '{ctx}' + return ('{comma}{where_str}' + spec).format(comma=comma, where_str=where_str, ctx=context) + + +def type_name(obj): + """Return the class name of *obj*. + + >>> type_name(42) + 'int' + >>> type_name("hello") + 'str' + """ + return type(obj).__name__ + + +class CCPPError(ValueError): + """User-facing error with a plain message and no traceback noise.""" + + def __init__(self, message): + logging.shutdown() + super().__init__(message) + + +class ParseSyntaxError(CCPPError): + """Syntax error that includes parsing context in the message.""" + + def __init__(self, token_type, token=None, context=None): + logging.shutdown() + cstr = context_string(context) + if token is None: + message = "{}{}".format(token_type, cstr) + else: + message = "Invalid {}, '{}'{}".format(token_type, token, cstr) + super().__init__(message) + + +class ParseInternalError(Exception): + """Internal parser logic error — not caught by normal user-error handlers.""" + + def __init__(self, errmsg, context=None): + logging.shutdown() + message = "{}{}".format(errmsg, context_string(context)) + super().__init__(message) + + +class ParseContextError(CCPPError): + """Error arising from mis-use of ParseContext.""" + + def __init__(self, errmsg, context): + logging.shutdown() + message = "{}{}".format(errmsg, context_string(context)) + super().__init__(message) + + +class ContextRegion(Iterable): + """LIFO stack of (region_type, region_name) pairs.""" + + def __init__(self): + self._lifo = [] + + def push(self, rtype, rname): + """Push a new region onto the stack.""" + self._lifo.append([rtype, rname]) + + def pop(self): + """Remove and return the top item.""" + return self._lifo.pop() + + def type_list(self): + """Return a list of just the region types.""" + return [x[0] for x in self._lifo] + + def __iter__(self): + for item in self._lifo: + yield item[0] + + def __len__(self): + return len(self._lifo) + + def __getitem__(self, index): + return self._lifo[index] + + +class ParseContext: + """Tracks a parser's current position inside a source file. + + Parameters + ---------- + linenum : int, optional + Zero-based line number. Negative means «file level, no specific line». + filename : str, optional + Path to the source file. + context : ParseContext, optional + Copy position from an existing context (overrides *linenum*/*filename*). + + Examples + -------- + >>> ctx = ParseContext(linenum=0, filename="foo.F90") + >>> str(ctx) + 'foo.F90:1' + >>> ctx.increment(4) + >>> str(ctx) + 'foo.F90:5' + >>> ParseContext(linenum="bad", filename="f.F90") + Traceback (most recent call last): + ... + metadata.parse_tools.parse_source.CCPPError: ParseContext linenum must be an int + """ + + def __init__(self, linenum=None, filename=None, context=None): + if context is not None: + self.__regions = copy.deepcopy(context.regions) + else: + self.__regions = ContextRegion() + + if context is not None: + linenum = context.line_num + elif linenum is None: + linenum = -1 + elif not isinstance(linenum, int): + raise CCPPError('ParseContext linenum must be an int') + + if context is not None: + filename = context.filename + elif filename is None: + filename = "" + elif not isinstance(filename, str): + raise CCPPError('ParseContext filename must be a string') + + self.__linenum = linenum + self.__filename = filename + + def default_module_name(self): + """Return the base name (without extension) of the source file.""" + return os.path.splitext(os.path.basename(self.__filename))[0] + + @property + def line_num(self): + """Current zero-based line number (negative = file level).""" + return self.__linenum + + @line_num.setter + def line_num(self, newnum): + self.__linenum = newnum + + @property + def filename(self): + """Path to the source file being parsed.""" + return self.__filename + + @property + def regions(self): + """The nested-region stack.""" + return self.__regions + + def __format__(self, spec): + """Format the context as ``filename:line``. + + Supported format specs: ``'nodir'`` strips the directory component. + """ + fname = os.path.basename(self.__filename) if spec == 'nodir' else self.__filename + if self.__linenum >= 0: + return "{}:{}".format(fname, self.__linenum + 1) + return fname + + def __str__(self): + if self.__linenum >= 0: + return "{}:{}".format(self.__filename, self.__linenum + 1) + return self.__filename + + def increment(self, inc=1): + """Advance the line counter by *inc* (default 1).""" + if self.__linenum < 0: + self.__linenum = 0 + self.__linenum += inc + + def enter_region(self, region_type, region_name=None, nested_ok=True): + """Record entering a named region (module, DDT, subroutine, …). + + If *nested_ok* is False, raises :exc:`ParseContextError` when already + inside a region of the same type. + """ + if (region_type not in self.__regions.type_list()) or nested_ok: + self.__regions.push(region_type, region_name) + else: + raise ParseContextError( + "Cannot enter a nested {} region".format(region_type), self + ) + + def leave_region(self, region_type, region_name=None): + """Record leaving a region, with optional name verification.""" + if self.__regions: + curr_type, curr_name = self.__regions.pop() + if curr_type != region_type: + raise ParseContextError( + "Trying to exit {} region while currently in {} region".format( + region_type, curr_type + ), + self, + ) + if region_name is not None and curr_name is not None: + if region_name != curr_name: + raise ParseContextError( + "Trying to exit {} {} while currently in {} {}".format( + region_type, region_name, curr_type, curr_name + ), + self, + ) + elif region_name is not None and curr_name is None: + raise ParseContextError( + "Trying to exit {} {} while currently in unnamed {} region".format( + region_type, region_name, curr_type + ), + self, + ) + else: + raise ParseContextError("Cannot exit, not currently in any region", self) + + def curr_region(self): + """Return the innermost (type, name) pair, or None if not in any region.""" + return self.__regions[-1] if self.__regions else None + + def in_region(self, region_type, region_name=None): + """Return True iff currently inside *region_type* (optionally *region_name*).""" + return self.curr_region() == [region_type, region_name] + + +class ParseSource: + """Lightweight tag associating a name and type with a parse context. + + >>> src = ParseSource("my_func", "subroutine", ParseContext(0, "src.F90")) + >>> src.name + 'my_func' + >>> src.ptype + 'subroutine' + >>> str(src.context) + 'src.F90:1' + """ + + def __init__(self, name_in, type_in, context_in): + self.__name = name_in + self.__type = type_in + self.__context = context_in + + @property + def ptype(self): + """The type label (e.g. 'scheme', 'host', 'subroutine').""" + return self.__type + + @property + def name(self): + """The name of the parsed entity.""" + return self.__name + + @property + def context(self): + """The :class:`ParseContext` where this entity was found.""" + return self.__context diff --git a/capgen-ng/metadata/parse_tools/xml_tools.py b/capgen-ng/metadata/parse_tools/xml_tools.py new file mode 100644 index 00000000..337b9954 --- /dev/null +++ b/capgen-ng/metadata/parse_tools/xml_tools.py @@ -0,0 +1,571 @@ +#!/usr/bin/env python3 + +""" +Parse a host-model registry XML file and return the captured variables. +""" + +# Python library imports +import os +import re +import shutil +import subprocess +import sys +import xml.etree.ElementTree as ET +import xml.dom.minidom +# CCPP framework imports +from .parse_source import CCPPError +from .parse_log import init_log, set_log_to_null + +# Global data +_INDENT_STR = " " +beg_tag_re = re.compile(r"([<][^/][^<>]*[^/][>])") +end_tag_re = re.compile(r"([<][/][^<>/]+[>])") +simple_tag_re = re.compile(r"([<][^/][^<>/]+[/][>])") + +# Find python version +PYSUBVER = sys.version_info[1] +_LOGGER = None + +############################################################################### +class XMLToolsInternalError(ValueError): +############################################################################### + """Error class for reporting internal errors""" + def __init__(self, message): + """Initialize this exception""" + super().__init__(message) + +############################################################################### +def find_schema_version(root): +############################################################################### + """ + Find the version of the host registry file represented by root + >>> find_schema_version(ET.fromstring('')) + [1, 0] + >>> find_schema_version(ET.fromstring('')) + [2, 0] + >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: Illegal version string, '1.a' + Format must be . + >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: Illegal version string, '0.0' + Major version must be at least 1 + >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: Illegal version string, '0.0' + Minor version must be at least 0 + """ + verbits = None + if 'version' not in root.attrib: + raise CCPPError("Version attribute required") + # end if + version = root.attrib['version'] + versplit = version.split('.') + try: + if len(versplit) != 2: + raise CCPPError('Major and minor version required') + # end if (no else needed) + try: + verbits = [int(x) for x in versplit] + except ValueError as verr: + raise CCPPError(verr) from verr + # end try + if verbits[0] < 1: + raise CCPPError('Major version must be at least 1') + # end if + if verbits[1] < 0: + raise CCPPError('Minor version must be non-negative') + # end if + except CCPPError as verr: + errstr = """Illegal version string, '{}' + Format must be .""" + ve_str = str(verr) + if ve_str: + errstr = ve_str + '\n' + errstr + # end if + raise CCPPError(errstr.format(version)) from verr + # end try + return verbits + +############################################################################### +def find_schema_file(schema_root, version, schema_path=None): +############################################################################### + """Find and return the schema file based on and + or return None. + If is present, use that as the directory to find the + appropriate schema file. Otherwise, just look in the current directory.""" + + verstring = '_'.join([str(x) for x in version]) + schema_filename = "{}_v{}.xsd".format(schema_root, verstring) + if schema_path: + schema_file = os.path.join(schema_path, schema_filename) + else: + schema_file = schema_filename + # end if + if os.path.exists(schema_file): + return schema_file + # end if + return None + +############################################################################### +def validate_xml_file(filename, schema_root, version, logger, schema_path=None): +############################################################################### + """ + Find the appropriate schema and validate the XML file, , + against it using xmllint + """ + # Check the filename + if not os.path.isfile(filename): + raise CCPPError("validate_xml_file: Filename, '{}', does not exist".format(filename)) + # end if + if not os.access(filename, os.R_OK): + raise CCPPError("validate_xml_file: Cannot open '{}'".format(filename)) + # end if + if os.path.isfile(schema_root): + # We already have a file, just use it + schema_file = schema_root + else: + if not schema_path: + # Find the schema, based on the model version + thispath = os.path.abspath(__file__) + pdir = os.path.dirname(os.path.dirname(os.path.dirname(thispath))) + schema_path = os.path.join(pdir, 'schema') + # end if + schema_file = find_schema_file(schema_root, version, schema_path) + if not (schema_file and os.path.isfile(schema_file)): + verstring = '.'.join([str(x) for x in version]) + emsg = f"""validate_xml_file: Cannot find schema for version {verstring}, + {schema_file} does not exist""" + raise CCPPError(emsg) + # end if + # end if + if not os.access(schema_file, os.R_OK): + emsg = "validate_xml_file: Cannot open schema, '{}'" + raise CCPPError(emsg.format(schema_file)) + # end if + + # Find xmllint + xmllint = shutil.which('xmllint') # Blank if not installed + if not xmllint: + msg = "xmllint not found, could not validate file {}" + raise CCPPError("validate_xml_file: " + msg.format(filename)) + # end if + + # Validate XML file against schema + logger.debug("Checking file {} against schema {}".format(filename, + schema_file)) + cmd = [xmllint, '--noout', '--schema', schema_file, filename] + cproc = subprocess.run(cmd, check=False, capture_output=True) + if cproc.returncode == 0: + # We got a pass return code but some versions of xmllint do not + # correctly return an error code on non-validation so double check + # the result + result = b'validates' in cproc.stdout or b'validates' in cproc.stderr + else: + result = False + # end if + if result: + logger.debug(cproc.stdout) + logger.debug(cproc.stderr) + return result + else: + cmd = ' '.join(cmd) + outstr = f"Execution of '{cmd}' failed with code: {cproc.returncode}\n" + if cproc.stdout: + outstr += f"{cproc.stdout.decode('utf-8', errors='replace').strip()}\n" + if cproc.stderr: + outstr += f"{cproc.stderr.decode('utf-8', errors='replace').strip()}\n" + raise CCPPError(outstr) + +############################################################################### +def read_xml_file(filename, logger=None): +############################################################################### + """Read the XML file, , and return its tree and root + + Parameters: + filename (str): The path to an XML file to read and search. + logger (logging.Logger, optional): Logger for warnings/errors. + + Returns: + tree (xml.etree.ElementTree): The element tree from the input file. + root (xml.etree.ElementTree.Element): The root element of tree. + + Raises: + CCPPError: If the file cannot be found or read. + """ + if os.path.isfile(filename) and os.access(filename, os.R_OK): + file_open = (lambda x: open(x, 'r', encoding='utf-8')) + with file_open(filename) as file_: + try: + tree = ET.parse(file_) + root = tree.getroot() + except ET.ParseError as perr: + emsg = "read_xml_file: Cannot read {}, {}" + raise CCPPError(emsg.format(filename, perr)) from perr + elif not os.access(filename, os.R_OK): + raise CCPPError("read_xml_file: Cannot open '{}'".format(filename)) + else: + emsg = "read_xml_file: Filename, '{}', does not exist" + raise CCPPError(emsg.format(filename)) + # end if + if logger: + logger.debug(f"Reading XML file {filename}") + # end if + return tree, root + +############################################################################### +def load_suite_by_name(suite_name, group_name, file, logger=None): +############################################################################### + """ + Load a suite by its name, or a group of a suite by the suite and group names. + + Parameters: + suite_name (str): The name of the suite to find. + group_name (str or None): The name of the group to find within the suite. + file (str): The path to an XML file to read and search. + logger (logging.Logger, optional): Logger for warnings/errors. + + Returns: + xml.etree.ElementTree.Element: The matching suite or group element. + + Raises: + CCPPError: If the suite or group is not found, or if the schema is invalid. + + Examples: + >>> import tempfile + >>> import xml.etree.ElementTree as ET + >>> logger = init_log('xml_tools') + >>> set_log_to_null(logger) + >>> # Create temporary files for the nested suites + >>> tmpdir = tempfile.TemporaryDirectory() + >>> file1_path = os.path.join(tmpdir.name, "file1.xml") + >>> # Write XML contents to temporary file + >>> with open(file1_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... + ... + ... ''') + >>> load_suite_by_name("physics_suite", None, file1_path, logger).tag + 'suite' + >>> load_suite_by_name("physics_suite", "dynamics", file1_path, logger).attrib['name'] + 'dynamics' + >>> load_suite_by_name("physics_suite", "missing_group", file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + CCPPError: Nested suite physics_suite, group missing_group, not found + >>> load_suite_by_name("missing_suite", None, file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + CCPPError: Nested suite missing_suite not found + >>> tmpdir.cleanup() + """ + _, root = read_xml_file(file, logger) + schema_version = find_schema_version(root) + res = validate_xml_file(file, 'suite', schema_version, logger) + if not res: + raise CCPPError(f"Invalid suite definition file, '{file}'") + suite = root + if suite.attrib.get("name") == suite_name: + if group_name: + for group in suite.findall("group"): + if group.attrib.get("name") == group_name: + return group + else: + return suite + emsg = f"Nested suite {suite_name}" \ + + (f", group {group_name}," if group_name else "") \ + + " not found" + (f" in file {file}" if file else "") + raise CCPPError(emsg) + +############################################################################### +def replace_nested_suite(element, nested_suite, default_path, logger): +############################################################################### + """ + Replace a tag with the actual suite or group it references. + + This function looks up a referenced suite or suite group from an external + file, deep copies its children, and replaces the element + in the parent `element` with the copied contents. + + Parameters: + element (xml.etree.ElementTree.Element): The parent element containing the nested suite. + nested_suite (xml.etree.ElementTree.Element): The element to be replaced. + default_path (str): The default path to look for nested SDFs if file is not a absolute path. + logger (logging.Logger or None): Logger to record debug information. + + Returns: + str: The name of the suite that was replaced + + Example: + >>> import tempfile + >>> import xml.etree.ElementTree as ET + >>> from types import SimpleNamespace + >>> logger = init_log('xml_tools') + >>> set_log_to_null(logger) + >>> tmpdir = tempfile.TemporaryDirectory() + >>> file1_path = os.path.join(tmpdir.name, "file1.xml") + >>> with open(file1_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... my_scheme + ... + ... + ... ''') + >>> # Import nested suite at suite level + >>> xml = f''' + ... + ... + ... + ... ''' + >>> top_suite = ET.fromstring(xml) + >>> nested = top_suite.find("nested_suite") + >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) + 'my_suite' + >>> [child.tag for child in top_suite] + ['group'] + >>> top_suite.find("group").find("scheme").text + 'my_scheme' + >>> # Import group from nested suite at group level + >>> xml = f''' + ... + ... + ... + ... + ... + ... ''' + >>> top_suite = ET.fromstring(xml) + >>> top_group = top_suite.find("group") + >>> nested = top_group.find("nested_suite") + >>> replace_nested_suite(top_group, nested, tmpdir.name, logger) + 'my_suite' + >>> [child.tag for child in top_suite] + ['group'] + >>> top_suite.find("group").find("scheme").text + 'my_scheme' + >>> # Import group from nested suite at suite level + >>> xml = f''' + ... + ... + ... + ... ''' + >>> top_suite = ET.fromstring(xml) + >>> nested = top_suite.find("nested_suite") + >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) + 'my_suite' + >>> [child.tag for child in top_suite] + ['group'] + >>> top_suite.find("group").find("scheme").text + 'my_scheme' + >>> tmpdir.cleanup() + """ + suite_name = nested_suite.attrib.get("name") + group_name = nested_suite.attrib.get("group") + file = nested_suite.attrib.get("file") + if not os.path.isabs(file): + file = os.path.join(default_path, file) + referenced_suite = load_suite_by_name(suite_name, group_name, file, + logger=logger) + imported_content = [ET.fromstring(ET.tostring(child)) + for child in referenced_suite] + # Swap nested suite with imported content + for item in imported_content: + # If we are inserting a nested suite at the suite level (element.tag is suite), + # but we only want one group (group_name is not none), then we need to wrap + # the item in a group element. If on the other hand we insert an entire suite + # (all groups) at the suite level, or a specific group at the group level, + # then we can insert the item as is. + if element.tag == 'suite' and group_name: + item_to_insert = ET.Element("group", attrib={"name": group_name}) + item_to_insert.append(item) + else: + item_to_insert = item + element.insert(list(element).index(nested_suite), item_to_insert) + element.remove(nested_suite) + if logger: + msg = f"Expanded nested suite '{suite_name}'" \ + + (f", group '{group_name}'," if group_name else "") \ + + (f" in file '{file}'" if file else "") + logger.debug(msg.rstrip(',')) + # Return the name of the suite that we just replaced + return suite_name + +############################################################################### +def expand_nested_suites(suite, default_path, logger=None): +############################################################################### + """ + Recursively expand all elements within the XML element. + + This function finds elements within or elements, + and replaces them with the corresponding content from another suite. + + This operation is recursive and will continue expanding until no + elements remain. + + Parameters: + suite (xml.etree.ElementTree.Element): The root element. + logger (logging.Logger, optional): Logger for debug messages. + + Returns: + None. The XML tree is modified in place. + + Example: + >>> import tempfile + >>> import xml.etree.ElementTree as ET + >>> logger = init_log('xml_tools') + >>> set_log_to_null(logger) + >>> tmpdir = tempfile.TemporaryDirectory() + >>> file1_path = os.path.join(tmpdir.name, "file1.xml") + >>> file2_path = os.path.join(tmpdir.name, "file2.xml") + >>> file3_path = os.path.join(tmpdir.name, "file3.xml") + >>> file4_path = os.path.join(tmpdir.name, "file4.xml") + >>> file5_path = os.path.join(tmpdir.name, "file5.xml") + >>> # Write mock XML contents for the nested suites + >>> with open(file1_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... cloud_scheme + ... + ... + ... ''') + >>> with open(file2_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... pbl_scheme + ... + ... + ... ''') + >>> with open(file3_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... rrtmg_lw_scheme + ... + ... + ... rrtmg_sw_scheme + ... + ... + ... ''') + >>> with open(file4_path, "w") as f: + ... _ = f.write(f''' + ... + ... + ... + ... ''') + >>> with open(file5_path, "w") as f: + ... _ = f.write(f''' + ... + ... + ... + ... ''') + >>> # Parent suite + >>> xml_content = f''' + ... + ... + ... + ... + ... + ... + ... + ... ''' + >>> suite = ET.fromstring(xml_content) + >>> expand_nested_suites(suite, tmpdir.name, logger) + >>> ET.dump(suite) + + + cloud_scheme + + pbl_scheme + + rrtmg_lw_scheme + + rrtmg_sw_scheme + + >>> # Test infite recursion + >>> xml_content = f''' + ... + ... + ... + ... + ... + ... + ... ''' + >>> suite = ET.fromstring(xml_content) + >>> expand_nested_suites(suite, tmpdir.name, logger) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + CCPPError: Exceeded number of iterations while expanding nested suites + >>> tmpdir.cleanup() + """ + # To avoid infinite recursion, we simply count the number + # of iterations and stop at a certain limit. If someone is + # smart enough to come up with nested suite constructs that + # require more iterations, than he/she should be able to + # track down this variable and adjust it! + max_iterations = 10 + # Collect the names of the expanded suites + suite_names = [] + # Iteratively expand nested suites until they are all gone + keep_expanding = True + for num_iterations in range(max_iterations): + keep_expanding = False + # First, search all groups for nested_suite elements + groups = suite.findall("group") + for group in groups: + nested_suites = group.findall("nested_suite") + for nested in nested_suites: + suite_names.append(replace_nested_suite(group, nested, default_path, logger)) + # Trigger another pass over the root element + keep_expanding = True + # Second, search all suites for nested_suite elements + nested_suites = suite.findall("nested_suite") + for nested in nested_suites: + suite_names.append(replace_nested_suite(suite, nested, default_path, logger)) + # Trigger another pass over the root element + keep_expanding = True + if not keep_expanding: + return + raise CCPPError("Exceeded number of iterations while expanding nested suites. " + \ + "Check for infinite recursion or adjust limit max_iterations. " + \ + f"Suites expanded so far: {suite_names}") + +############################################################################### +def write_xml_file(root, file_path, logger=None): +############################################################################### + """Pretty-prints element root to an ASCII file using xml.dom.minidom""" + + def remove_whitespace_nodes(node): + """Helper function to recursively remove all text nodes that contain + only whitespace, which eliminates blank lines in the output.""" + for child in list(node.childNodes): + if child.nodeType == child.TEXT_NODE and not child.data.strip(): + node.removeChild(child) + elif child.hasChildNodes(): + remove_whitespace_nodes(child) + + # Convert ElementTree to a byte string + byte_string = ET.tostring(root, 'us-ascii') + + # Parse string using minidom for pretty printing + reparsed = xml.dom.minidom.parseString(byte_string) + + # Clean whitespace-only text nodes + remove_whitespace_nodes(reparsed) + + # Generate pretty-printed XML string + pretty_xml = reparsed.toprettyxml(indent=" ") + + # Write to file + with open(file_path, 'w', errors='xmlcharrefreplace') as f: + f.write(pretty_xml) + + # Tell everyone! + if logger: + logger.debug(f"Writing XML file {file_path}") + +############################################################################## diff --git a/capgen-ng/metadata/unit_conversion.py b/capgen-ng/metadata/unit_conversion.py new file mode 100755 index 00000000..25a44cab --- /dev/null +++ b/capgen-ng/metadata/unit_conversion.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 + +"""A pilot version to perform unit conversions. Each conversion must be representable +as a formula where {var} is substituted by the actual variable (scalar, array) to convert, +and {kind} by either _ followed by the kind of the variable, or an emptry string. +This allows having formulars such as func({var}) where func is defined as an elemental +function or as an interface to scalar and array-based functions that perform more +complex conversions than the ones listed here. It is also possible, but in some cases +less performant, to construct conversions for composed units by combining some of the +basic conversions listed here. For instance, one could create a speed conversion from +km h-1 to m s-1 by combining the formulas for km to m and h to min, which will be +slower than boiling it down to a single mathematical expression (see example below).""" + +############ +# Length # +############ + +def mm__to__m(): + """Convert millimeter to meter""" + return '1.0E-3{kind}*{var}' + +def m__to__mm(): + """Convert meter to millimeter""" + return '1.0E+3{kind}*{var}' + +def cm__to__m(): + """Convert centimeter to meter""" + return '1.0E-2{kind}*{var}' + +def m__to__cm(): + """Convert meter to centimeter""" + return '1.0E+2{kind}*{var}' + +def um__to__m(): + """Convert micrometer to meter""" + return '1.0E-6{kind}*{var}' + +def m__to__um(): + """Convert meter to micrometer""" + return '1.0E+6{kind}*{var}' + +def m__to__km(): + """Convert meter to kilometer""" + return '1.0E-3{kind}*{var}' + +def km__to__m(): + """Convert kilometer to meter""" + return '1.0E+3{kind}*{var}' + +def mm__to__km(): + """Convert millimeter to kilometer""" + return '1.0E-6{kind}*{var}' + +def km__to__mm(): + """Convert kilometer to millimeter""" + return '1.0E+6{kind}*{var}' + +############ +# Time # +############ + +def s__to__min(): + """Convert second to minute""" + return '{var}/6.0E+1{kind}' + +def min__to__s(): + """Convert minute to second""" + return '6.0E+1{kind}*{var}' + +def s__to__h(): + """Convert second to hour""" + return '{var}/3.6E+3{kind}' + +def h__to__s(): + """Convert hour to second""" + return '3.6E+3{kind}*{var}' + +def h__to__d(): + """Convert hour to day""" + return '{var}/2.4E+1{kind}' + +def d__to__h(): + """Convert day to hour""" + return '2.4E+1{kind}*{var}' + +def s__to__d(): + """Convert second to day""" + return '{var}/8.64E+4{kind}' + +def d__to__s(): + """Convert day to second""" + return '8.64E+4{kind}*{var}' + +################## +# Temperature # +################## + +def K__to__C(): + """Convert Kelvin to Celcius""" + return '{var}-273.15{kind}' + +def C__to__K(): + """Convert Celcius to Kelvin""" + return '{var}+273.15{kind}' + +################## +# Mass # +################## + +def kg_kg_minus_1__to__g_kg_minus_1(): + """Convert kilogram per kilogram to gram per kilogram""" + return '1.0E+3{kind}*{var}' + +def g_kg_minus_1__to__kg_kg_minus_1(): + """Convert gram per kilogram to kilogram per kilogram""" + return '1.0E-3{kind}*{var}' + +################## +# Plane angle # +################## + +def radian__to__degree(): + """Convert radian to degree""" + return '57.295779513{kind}*{var}' + +def degree__to__radian(): + """Convert degree to radian""" + return '{var}/57.295779513{kind}' + +def radian__to__degree_north(): + """Convert radian to degree north""" + return radian__to__degree() + +def degree_north__to__radian(): + """Convert degree north to radian""" + return degree__to__radian() + +def radian__to__degree_east(): + """Convert radian to degree east""" + return radian__to__degree() + +def degree_east__to__radian(): + """Convert degree east to radian""" + return degree__to__radian() + +################## +# Composed units # +################## + +def Pa__to__hPa(): + """Convert Pascal to Hectopascal""" + return '1.0E-2{kind}*{var}' + +def hPa__to__Pa(): + """Convert Hectopascal to Pascal""" + return '1.0E+2{kind}*{var}' + +def m_s_minus_1__to__km_h_minus_1(): + """Convert meter per second to kilometer per hour. A more expensive + and less accurate option would be to combine the above conversions + for meter to kilometer and second to hours into the following formula: + '({0})/({1})'.format(m__to__km(),s__to__h()) + '*{var}'""" + return '3.6E+0{kind}*{var}' + +def km_h_minus_1__to__m_s_minus_1(): + """Convert kilometer per hour to meter per second. A more expensive + and less accurate option would be to combine the above conversions + for kilometer to meter and hours to second into the following formula: + '({0})/({1})'.format(km__to__m(),h__to__s()) + '*{var}'""" + return '{var}/3.6E+0{kind}' + +def W_m_minus_2__to__erg_cm_minus_2_s_minus_1(): + """Convert Watt per square meter to erg per square centimeter and second.""" + return '1.0E+3{kind}*{var}' + +def erg_cm_minus_2_s_minus_1__to__W_m_minus_2(): + """Convert erg per square centimeter and second to Watt per square meter""" + return '1.0E-3{kind}*{var}' + +#################### +# Equivalent units # +#################### + +def m_plus_2_s_minus_2__to__J_kg_minus_1(): + """Equivalent units""" + return '{var}' + +def J_kg_minus_1__to__m_plus_2_s_minus_2(): + """Equivalent units""" + return '{var}' + +def V_A__to__W(): + """Equivalent units""" + return '{var}' + +def W__to__V_A(): + """Equivalent units""" + return '{var}' + +def N_m_minus_2__to__Pa(): + """Equivalent units""" + return '{var}' + +def Pa__to__N_m_minus_2(): + """Equivalent units""" + return '{var}' diff --git a/capgen-ng/metadata/variable_resolver.py b/capgen-ng/metadata/variable_resolver.py new file mode 100644 index 00000000..5bea7f89 --- /dev/null +++ b/capgen-ng/metadata/variable_resolver.py @@ -0,0 +1,687 @@ +#!/usr/bin/env python3 + +"""Variable resolution and access-path construction for ccpp-capgen-ng. + +This module flattens the host/control/DDT metadata hierarchy into a single +keyed dictionary and provides the ``SchemeStore`` lookup table that the code +generator uses to resolve scheme arguments. + +Public API +---------- +``HostVarEntry`` + One resolved variable in the flat host+control dictionary. + +``build_flat_host_dict(host_tables, control_tables, ddt_tables)`` + Flatten host + control tables (expanding DDT instances into their fields) + into a ``Dict[str, HostVarEntry]`` keyed by standard name. + +``SchemeStore`` + Organises scheme metadata by scheme name and phase for O(1) lookup during + variable resolution. + +Registered dimension standard names (Section 4.3 of the redesign spec) +------------------------------------------------------------------------ +The generator has built-in semantic knowledge of these standard names when they +appear as dimensions in host variables: + +``instance_dimension`` + Scalar extraction — the DDT instance array is subscripted with the + ``instance_number`` control variable: + ``gfs_statein(instance_number)%field`` + + Note: the sample metadata in section 3.5 of the spec uses + ``number_of_instances`` for the same role. Both names are treated as + instance dimensions here; see ``_INSTANCE_DIMS``. + +``horizontal_dimension``, ``horizontal_loop_extent`` + Horizontal slice — emitted as ``lb:ub`` (run phase) or + ``1:`` (non-run). The code generator handles the slicing; + the resolver only records the dimension standard name. + +``vertical_*`` + Vertical slice — emitted as ``1:``. Same handling. + +All other dimension standard names are "arbitrary" — the resolver looks up +the corresponding variable's local name and the code generator emits +``1:``. +""" + +import re +from typing import Dict, List, Optional + +from .metadata_table import MetaVar, MetadataTable +from .parse_tools import CCPPError, check_fortran_intrinsic, FORTRAN_SCALAR_REF_RE + +######################################################################## +# Registered dimension constants +######################################################################## + +#: Dimension standard names that indicate a DDT is indexed by instance. +#: The host access path gains a ``(instance_number)`` subscript. +#: See Section 4.3 ("instance_dimension") and the Section 3.5 example +#: ("number_of_instances"). +_INSTANCE_DIMS: frozenset = frozenset({ + 'instance_dimension', + 'number_of_instances', +}) + +#: Regex that normalises ``type(typename)`` → ``typename``. +_TYPE_PAREN_RE = re.compile( + r'(?i)^type\s*\(\s*([A-Za-z][A-Za-z0-9_]*)\s*\)$' +) + +#: Regex for ``external:module:typename`` type syntax. +_EXTERNAL_RE = re.compile(r'^external\s*:', re.IGNORECASE) + +######################################################################## +# Helpers +######################################################################## + +def _ddt_typename(type_str: str) -> str: + """Return the bare DDT type name, stripping ``type(...)`` if present.""" + m = _TYPE_PAREN_RE.match(type_str.strip()) + return m.group(1) if m else type_str.strip() + + +def _is_intrinsic(type_str: str) -> bool: + """Return True if *type_str* is a Fortran intrinsic type name.""" + return check_fortran_intrinsic(type_str.strip(), error=False) is not None + + +def _is_external(type_str: str) -> bool: + """Return True if *type_str* uses the ``external:module:typename`` syntax.""" + return bool(_EXTERNAL_RE.match(type_str.strip())) + + +def _is_known_ddt(type_str: str, ddt_index: Dict[str, MetadataTable]) -> bool: + """Return True if *type_str* refers to a DDT type present in *ddt_index*.""" + if _is_intrinsic(type_str) or _is_external(type_str): + return False + return _ddt_typename(type_str) in ddt_index + + +def _split_local_name(local_name: str): + """Split a local name into (base, subscript) tuple. + + For plain identifiers returns (local_name, ''). + For slice expressions like ``chunk_begin(ccpp_chunk_number)`` returns + (``'chunk_begin'``, ``'ccpp_chunk_number'``). + + >>> _split_local_name('chunk_begin') + ('chunk_begin', '') + >>> _split_local_name('chunk_begin(ccpp_chunk_number)') + ('chunk_begin', 'ccpp_chunk_number') + >>> _split_local_name('q(:,:,index_of_water_vapor_specific_humidity)') + ('q', ':,:,index_of_water_vapor_specific_humidity') + """ + m = FORTRAN_SCALAR_REF_RE.match(local_name) + if m is None: + return local_name, '' + return m.group(1), m.group(2).rstrip() + + +def _resolve_subscript(subscript: str, host_dict: Dict[str, 'HostVarEntry']) -> str: + """Replace each standard-name token in *subscript* with its local name. + + Tokens that are ``:`` (colon slices) or integer literals are left as-is. + Tokens that are standard names present in *host_dict* are replaced by + the entry's ``local_name``. Unrecognised tokens are left as-is (they + may be integer constants or already-local names). + + >>> from collections import namedtuple + >>> E = namedtuple('E', ['local_name']) + >>> d = {'ccpp_chunk_number': E('inst_num')} + >>> _resolve_subscript('ccpp_chunk_number', d) + 'inst_num' + >>> _resolve_subscript(':, ccpp_chunk_number', d) + ':, inst_num' + >>> _resolve_subscript('1', d) + '1' + """ + tokens = [t.strip() for t in subscript.split(',')] + resolved = [] + for token in tokens: + if token == ':' or token.isdigit(): + resolved.append(token) + elif token in host_dict: + resolved.append(host_dict[token].local_name) + else: + resolved.append(token) + return ', '.join(resolved) + + +def _instance_subscript(var: MetaVar) -> str: + """Return ``'(instance_number)'`` if *var* is a DDT instance array, else ``''``. + + A variable is treated as a DDT instance array when any of its declared + dimension standard names matches one of the :data:`_INSTANCE_DIMS` names. + """ + for dim in var.dimensions: + if dim in _INSTANCE_DIMS: + return '(instance_number)' + return '' + + +######################################################################## +# HostVarEntry +######################################################################## + +class HostVarEntry: + """One resolved variable in the flat host+control dictionary. + + All fields are set at construction time and treated as read-only + afterwards. + + Parameters + ---------- + standard_name : str + CF-compliant standard name (key in the flat dict). + local_name : str + Fortran local name of the innermost variable (e.g. ``'phii'``). + access_path : str + Fully-qualified Fortran access expression, with any DDT + component separators and instance subscripts applied + (e.g. ``'gfs_statein(instance_number)%phii'``). For plain + variables this equals *local_name*. + module_name : str or None + Fortran module that exports this variable, used to emit + ``use , only: `` in the generated cap. + ``None`` for control variables (passed as subroutine arguments). + type : str + Fortran type string. + kind : str + Optional kind parameter (empty string if not specified). + units : str + Physical units. + dimensions : list of str + Ordered dimension standard names; empty for scalars. + protected : bool + Whether any scheme is forbidden from declaring ``intent`` other + than ``in`` for this variable. + optional : bool + Whether the variable may be absent (uses optional pointer in cap). + allocatable : bool + Whether the variable is declared with the Fortran ``allocatable`` + attribute. Host and scheme metadata must agree. Affects code + generation: actual arguments at call sites omit explicit dimension + subscripts for allocatable variables. + active : str + Fortran conditional expression in standard names; empty if always + active. + """ + + __slots__ = ( + 'standard_name', 'local_name', 'access_path', 'module_name', + 'type', 'kind', 'units', 'dimensions', + 'protected', 'optional', 'allocatable', 'active', 'local_subscript', + 'top_at_one', + ) + + def __init__( + self, + standard_name: str, + local_name: str, + access_path: str, + module_name: Optional[str], + type_: str, + kind: str, + units: str, + dimensions: List[str], + protected: bool, + optional: bool, + active: str, + local_subscript: Optional[List[str]] = None, + allocatable: bool = False, + top_at_one: bool = False, + ): + self.standard_name = standard_name + self.local_name = local_name + self.access_path = access_path + self.module_name = module_name + self.type = type_ + self.kind = kind + self.units = units + self.dimensions = list(dimensions) + self.protected = protected + self.optional = optional + self.allocatable = allocatable + self.active = active + self.local_subscript = list(local_subscript) if local_subscript else [] + self.top_at_one = top_at_one + + @property + def is_control(self) -> bool: + """True when this variable is a control variable (no module USE needed).""" + return self.module_name is None + + def __repr__(self) -> str: + return "HostVarEntry({!r}, access_path={!r})".format( + self.standard_name, self.access_path + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, HostVarEntry): + return NotImplemented + return self.standard_name == other.standard_name + + def __hash__(self) -> int: + return hash(self.standard_name) + + +######################################################################## +# DDT index +######################################################################## + +def _build_ddt_index(ddt_tables: List[MetadataTable]) -> Dict[str, MetadataTable]: + """Build a dict from DDT type name → ``MetadataTable`` for O(1) lookup.""" + return {tbl.table_name: tbl for tbl in ddt_tables} + + +def build_ddt_module_map( + all_tables: List[MetadataTable], +) -> Dict[str, str]: + """Build a map from DDT type name → Fortran module that defines it. + + A DDT table inherits its defining Fortran module from a co-located + ``host``, ``control``, or ``scheme`` table in the same ``.meta`` file. + The convention is that a CCPP scheme/host/control table's name is the + name of the Fortran module that contains it; a DDT type defined alongside + such a table is assumed to be defined in the same Fortran module. + + DDT tables in a file with no co-located scheme/host/control table are + skipped (no entry written). DDTs that are only referenced as types of + host instance variables (declared in the host's own Fortran code) do not + need an entry — the host's Fortran code already imports the type. + + Parameters + ---------- + all_tables : list of MetadataTable + Mixed list of all parsed metadata tables (host, control, scheme, + ddt). ``suite`` tables are ignored. + + Returns + ------- + dict mapping DDT type name → Fortran module name + """ + by_file: Dict[str, List[MetadataTable]] = {} + for tbl in all_tables: + by_file.setdefault(tbl.file_path, []).append(tbl) + + result: Dict[str, str] = {} + for fpath, tables in by_file.items(): + module_name: Optional[str] = None + for tbl in tables: + if tbl.table_type in ('scheme', 'host', 'control'): + module_name = tbl.table_name + break + if module_name is None: + continue + for tbl in tables: + if tbl.table_type == 'ddt': + result[tbl.table_name] = module_name + return result + + +######################################################################## +# DDT instance flattening +######################################################################## + +def _flatten_ddt_instance( + var: MetaVar, + module_name: str, + ddt_index: Dict[str, MetadataTable], + access_prefix: str = '', + depth: int = 0, + max_depth: int = 8, +) -> List[HostVarEntry]: + """Expand a DDT instance variable into flat ``HostVarEntry`` objects. + + The DDT instance entry itself is included first, followed by one entry + per field. If a field is itself a known DDT type, it is expanded + recursively (up to *max_depth* levels deep). + + Parameters + ---------- + var : MetaVar + The DDT instance variable from a host table section. + module_name : str + Fortran module from which the DDT instance is imported. + ddt_index : dict + Map from DDT type name → ``MetadataTable``, built by + :func:`_build_ddt_index`. + access_prefix : str + Fortran access-path prefix accumulated from enclosing DDT levels, + e.g. ``'outer(instance_number)%'``. + depth, max_depth : int + Recursion guard — raises ``CCPPError`` if exceeded. + + Returns + ------- + list of HostVarEntry + + Raises + ------ + CCPPError + If the DDT type is not in *ddt_index*, or if the nesting depth + exceeds *max_depth* (circular reference guard). + """ + if depth > max_depth: + raise CCPPError( + "DDT hierarchy for '{}' exceeds maximum nesting depth {}; " + "possible circular reference".format(var.standard_name, max_depth) + ) + + ddt_name = _ddt_typename(var.type) + if ddt_name not in ddt_index: + raise CCPPError( + "Variable '{}' (standard_name='{}') has type '{}' but no " + "matching 'type = ddt' table was found; " + "add the DDT metadata file to --host-files or --scheme-files".format( + var.local_name, var.standard_name, var.type + ) + ) + + ddt_table = ddt_index[ddt_name] + subscript = _instance_subscript(var) + # Fortran access path to this DDT instance (without field component). + instance_access = access_prefix + var.local_name + subscript + + entries: List[HostVarEntry] = [] + + # The DDT instance variable itself — keyed by its own standard_name. + entries.append(HostVarEntry( + standard_name=var.standard_name, + local_name=var.local_name, + access_path=access_prefix + var.local_name, + module_name=module_name, + type_=var.type, + kind=var.kind, + units=var.units, + dimensions=var.dimensions, + protected=var.protected, + optional=var.optional, + active=var.active, + local_subscript=[], + allocatable=var.allocatable, + top_at_one=var.top_at_one, + )) + + # Expand each field of the DDT. + for sec in ddt_table.sections(): + for field in sec.variables: + if _is_known_ddt(field.type, ddt_index): + # Nested DDT — recurse. + entries.extend(_flatten_ddt_instance( + field, + module_name, + ddt_index, + access_prefix=instance_access + '%', + depth=depth + 1, + max_depth=max_depth, + )) + else: + base_field, sub_str = _split_local_name(field.local_name) + sub_tokens = [t.strip() for t in sub_str.split(',') if t.strip()] if sub_str else [] + field_path = instance_access + '%' + base_field + entries.append(HostVarEntry( + standard_name=field.standard_name, + local_name=base_field, + access_path=field_path, + module_name=module_name, + type_=field.type, + kind=field.kind, + units=field.units, + dimensions=field.dimensions, + protected=field.protected, + optional=field.optional, + active=field.active, + local_subscript=sub_tokens, + allocatable=field.allocatable, + top_at_one=field.top_at_one, + )) + + return entries + + +######################################################################## +# Public: build_flat_host_dict +######################################################################## + +def build_flat_host_dict( + host_tables: List[MetadataTable], + control_tables: List[MetadataTable], + ddt_tables: List[MetadataTable], +) -> Dict[str, 'HostVarEntry']: + """Build the flat host+control variable dictionary. + + All host and control variables are flattened into a single + ``Dict[str, HostVarEntry]`` keyed by standard name. DDT instance + variables are expanded into one entry per field; the instance variable + itself is also stored under its own standard name. + + Control variables have ``module_name=None`` — they are passed as + subroutine arguments, not imported via ``use``. + + Parameters + ---------- + host_tables : list of MetadataTable + Tables with ``table_type == 'host'``. + control_tables : list of MetadataTable + Tables with ``table_type == 'control'``. + ddt_tables : list of MetadataTable + Tables with ``table_type == 'ddt'``. + + Returns + ------- + dict mapping standard_name → HostVarEntry + + Raises + ------ + CCPPError + On duplicate standard names across tables, or on a DDT reference + without a matching DDT table. + """ + ddt_index = _build_ddt_index(ddt_tables) + result: Dict[str, HostVarEntry] = {} + + def _add(entry: HostVarEntry, source_label: str) -> None: + prior = result.get(entry.standard_name) + if prior is not None: + prior_loc = ( + "module '{}'".format(prior.module_name) + if prior.module_name else '' + ) + new_loc = "module '{}'".format(entry.module_name) \ + if entry.module_name else '' + raise CCPPError( + "Duplicate standard name '{}':\n" + " already registered from {} via access path '{}'\n" + " re-registered from {} (source: {}) via access path '{}'\n" + "If both paths come from the same parent DDT, the parent " + "likely declares two sibling DDT instances of the same " + "type — components of each get flattened into the host " + "dictionary under the same standard names. Drop one of " + "the sibling DDT instances, or give one a non-overlapping " + "standard name on every component.".format( + entry.standard_name, + prior_loc, prior.access_path, + new_loc, source_label, entry.access_path, + ) + ) + result[entry.standard_name] = entry + + # ---- host tables ------------------------------------------------------- + for tbl in host_tables: + # Explicit ``module_name`` from ``[ccpp-table-properties]`` overrides + # the convention "module name = table name"; falls back to the table + # name when not declared. + host_module = (tbl.module_name or '').strip() or tbl.table_name + for sec in tbl.sections(): + for var in sec.variables: + if _is_known_ddt(var.type, ddt_index): + for entry in _flatten_ddt_instance( + var, host_module, ddt_index + ): + _add(entry, tbl.table_name) + elif _is_intrinsic(var.type) or _is_external(var.type): + base_name, sub_str = _split_local_name(var.local_name) + sub_tokens = [t.strip() for t in sub_str.split(',') if t.strip()] if sub_str else [] + _add(HostVarEntry( + standard_name=var.standard_name, + local_name=base_name, + access_path=base_name, + module_name=host_module, + type_=var.type, + kind=var.kind, + units=var.units, + dimensions=var.dimensions, + protected=var.protected, + optional=var.optional, + active=var.active, + local_subscript=sub_tokens, + allocatable=var.allocatable, + top_at_one=var.top_at_one, + ), tbl.table_name) + else: + raise CCPPError( + "Variable '{}' (standard_name='{}') in table '{}' has " + "type '{}' which is not a Fortran intrinsic, not an " + "'external:' type, and has no matching 'type = ddt' table; " + "declare the DDT in a metadata file and pass it via " + "--host-files, or use 'type = external::' " + "for non-CCPP types".format( + var.local_name, var.standard_name, + tbl.table_name, var.type, + ) + ) + + # ---- control tables ---------------------------------------------------- + for tbl in control_tables: + for sec in tbl.sections(): + for var in sec.variables: + base_name, sub_str = _split_local_name(var.local_name) + sub_tokens = [t.strip() for t in sub_str.split(',') if t.strip()] if sub_str else [] + _add(HostVarEntry( + standard_name=var.standard_name, + local_name=base_name, + access_path=base_name, + module_name=None, + type_=var.type, + kind=var.kind, + units=var.units, + dimensions=var.dimensions, + protected=var.protected, + optional=var.optional, + active=var.active, + local_subscript=sub_tokens, + allocatable=var.allocatable, + top_at_one=var.top_at_one, + ), tbl.table_name) + + return result + + +######################################################################## +# SchemeStore +######################################################################## + +class SchemeStore: + """Organises scheme metadata for variable resolution. + + Provides O(1) lookup of scheme arguments by scheme name and phase. + Constructed via :meth:`build_from`. + """ + + def __init__(self) -> None: + # _data[scheme_name][phase] = list of MetaVar (in metadata order) + self._data: Dict[str, Dict[str, List[MetaVar]]] = {} + # _modules[scheme_name] = Fortran module that exports the scheme's + # subroutines. Populated from the metadata table's ``module_name`` + # attribute (``[ccpp-table-properties]``) when declared; otherwise + # falls back to the scheme name (the common case where the .meta + # file shares its base name with the Fortran module). + self._modules: Dict[str, str] = {} + + @classmethod + def build_from(cls, scheme_tables: List[MetadataTable]) -> 'SchemeStore': + """Build a :class:`SchemeStore` from *scheme_tables*. + + Non-scheme tables are silently skipped so the caller may pass a + mixed list without filtering. + + Parameters + ---------- + scheme_tables : list of MetadataTable + One or more metadata tables (only ``scheme`` tables are used). + + Returns + ------- + SchemeStore + """ + store = cls() + for tbl in scheme_tables: + if not tbl.is_scheme: + continue + name = tbl.table_name + if name not in store._data: + store._data[name] = {} + # Resolve module: explicit ``module_name`` from the table + # properties overrides the implicit "module name equals scheme + # name" convention. See doc/scheme metadata format. + mod = tbl.module_name.strip() if tbl.module_name else '' + store._modules[name] = mod or name + for sec in tbl.sections(): + if sec.phase is None: + continue + if sec.phase in store._data[name]: + raise CCPPError( + "Duplicate phase '{}' for scheme '{}'; " + "check that the same scheme metadata is not loaded twice".format( + sec.phase, name + ) + ) + store._data[name][sec.phase] = list(sec.variables) + return store + + def scheme_names(self) -> List[str]: + """Return sorted list of all known scheme names.""" + return sorted(self._data.keys()) + + def module_for(self, name: str) -> str: + """Return the Fortran module name that exports scheme *name*. + + Returns the explicit ``module_name`` from the scheme's + ``[ccpp-table-properties]`` block when set, falling back to the + scheme name (the common case where the .meta file's table name + equals the Fortran module name). Returns *name* unchanged for + unknown schemes so callers always get a usable token for USE + emission; whether the unknown scheme exists in Fortran is + validated elsewhere (the cap simply fails to link). + """ + return self._modules.get(name, name) + + def has_scheme(self, name: str) -> bool: + """Return True if *name* is a known scheme.""" + return name in self._data + + def phases_for(self, name: str) -> List[str]: + """Return sorted list of phases defined for scheme *name*. + + Returns an empty list for unknown schemes. + """ + return sorted(self._data.get(name, {}).keys()) + + def variables_for(self, name: str, phase: str) -> Optional[List[MetaVar]]: + """Return the variable list for *name* / *phase*, or ``None``. + + The returned list preserves the metadata declaration order, which + determines argument order in the generated scheme call. + """ + phases = self._data.get(name) + if phases is None: + return None + inner = phases.get(phase) + return list(inner) if inner is not None else None + + def __repr__(self) -> str: + return "SchemeStore(schemes={})".format(self.scheme_names()) diff --git a/capgen-ng/schema/suite_v1_0.xsd b/capgen-ng/schema/suite_v1_0.xsd new file mode 100644 index 00000000..dfa96cc5 --- /dev/null +++ b/capgen-ng/schema/suite_v1_0.xsd @@ -0,0 +1,128 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/capgen-ng/schema/suite_v2_0.xsd b/capgen-ng/schema/suite_v2_0.xsd new file mode 100644 index 00000000..6ce3cfc6 --- /dev/null +++ b/capgen-ng/schema/suite_v2_0.xsd @@ -0,0 +1,119 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/capgen-ng/src/ccpp_constituent_prop_mod.F90 b/capgen-ng/src/ccpp_constituent_prop_mod.F90 new file mode 100644 index 00000000..dbe33f84 --- /dev/null +++ b/capgen-ng/src/ccpp_constituent_prop_mod.F90 @@ -0,0 +1,2679 @@ +module ccpp_constituent_prop_mod + + ! ccpp_contituent_prop_mod contains types and procedures for storing + ! and retrieving constituent properties + + use ccpp_hashable, only: ccpp_hashable_t, & + ccpp_hashable_char_t + use ccpp_hash_table, only: ccpp_hash_table_t, & + ccpp_hash_iterator_t + use ccpp_kinds, only: kind_phys + + implicit none + private + + !!XXgoldyXX: Implement "last_error" method so that functions do not + !! need to have output variables. + + ! Private module data + integer, parameter :: stdname_len = 256 + integer, parameter :: dimname_len = 32 + integer, parameter :: errmsg_len = 256 + integer, parameter :: dry_mixing_ratio = -2 + integer, parameter :: moist_mixing_ratio = -3 + integer, parameter :: wet_mixing_ratio = -4 + integer, parameter :: mass_mixing_ratio = -5 + integer, parameter :: volume_mixing_ratio = -6 + integer, parameter :: number_concentration = -7 + integer, public, parameter :: int_unassigned = -huge(1) + real(kind=kind_phys), parameter :: kphys_unassigned = huge(1.0_kind_phys) + + !! \section arg_table_ccpp_constituent_properties_t + !! \htmlinclude ccpp_constituent_properties_t.html + !! + type, public, extends(ccpp_hashable_char_t) :: ccpp_constituent_properties_t + ! A ccpp_constituent_properties_t object holds relevant metadata + ! for a constituent species and provides interfaces to access that data. + character(len=:), private, allocatable :: var_std_name + character(len=:), private, allocatable :: var_long_name + character(len=:), private, allocatable :: var_diag_name + character(len=:), private, allocatable :: var_units + character(len=:), private, allocatable :: vert_dim + integer, private :: const_ind = int_unassigned + logical, private :: advected = .false. + logical, private :: thermo_active = .false. + logical, private :: water_species = .false. + ! While the quantities below can be derived from the standard name, + ! this implementation avoids string searching in parameterizations + ! const_type distinguishes mass, volume, and number conc. mixing ratios + integer, private :: const_type = int_unassigned + ! const_water distinguishes dry, moist, and "wet" mixing ratios + integer, private :: const_water = int_unassigned + ! minimum_mr is the minimum allowed value (default zero) + real(kind=kind_phys), private :: min_val = 0.0_kind_phys + ! molar_mass_val is the molar mass of the constituent (kg mol-1) + real(kind=kind_phys), private :: molar_mass_val = kphys_unassigned + ! default_value is the default value that the constituent array will be + ! initialized to + real(kind=kind_phys), private :: const_default_value = kphys_unassigned + ! framework_owns_me is set by the caller of ccpp_model_constituents_t%new_field + ! (via set_framework_owned) when the caller has allocated the object on + ! the heap and is transferring ownership to the framework. When .false. + ! (default), the framework treats this object as caller-owned and will + ! not deallocate it during reset/teardown. + logical, private :: framework_owns_me = .false. + contains + ! Required hashable method + procedure :: key => ccp_properties_get_key + ! Informational methods + procedure :: is_instantiated => ccp_is_instantiated + procedure :: standard_name => ccp_get_standard_name + procedure :: long_name => ccp_get_long_name + procedure :: diagnostic_name => ccp_get_diagnostic_name + procedure :: units => ccp_get_units + procedure :: is_layer_var => ccp_is_layer_var + procedure :: is_interface_var => ccp_is_interface_var + procedure :: is_2d_var => ccp_is_2d_var + procedure :: vertical_dimension => ccp_get_vertical_dimension + procedure :: const_index => ccp_const_index + procedure :: is_advected => ccp_is_advected + procedure :: is_thermo_active => ccp_is_thermo_active + procedure :: is_water_species => ccp_is_water_species + procedure :: equivalent => ccp_is_equivalent + procedure :: is_mass_mixing_ratio => ccp_is_mass_mixing_ratio + procedure :: is_volume_mixing_ratio => ccp_is_volume_mixing_ratio + procedure :: is_number_concentration => ccp_is_number_concentration + procedure :: is_dry => ccp_is_dry + procedure :: is_moist => ccp_is_moist + procedure :: is_wet => ccp_is_wet + procedure :: minimum => ccp_min_val + procedure :: molar_mass => ccp_molar_mass + procedure :: default_value => ccp_default_value + procedure :: has_default => ccp_has_default + procedure :: is_match => ccp_is_match + ! Copy method (be sure to update this anytime fields are added) + procedure :: copyconstituent + generic :: assignment(=) => copyconstituent + ! Methods that change state (XXgoldyXX: make private?) + procedure :: instantiate => ccp_instantiate + procedure :: deallocate => ccp_deallocate + procedure :: set_const_index => ccp_set_const_index + procedure :: set_thermo_active => ccp_set_thermo_active + procedure :: set_water_species => ccp_set_water_species + procedure :: set_minimum => ccp_set_min_val + procedure :: set_molar_mass => ccp_set_molar_mass + ! Ownership flag for framework-side cleanup (see field comment above) + procedure :: is_framework_owned => ccp_is_framework_owned + procedure :: set_framework_owned => ccp_set_framework_owned + end type ccpp_constituent_properties_t + + !! \section arg_table_ccpp_constituent_prop_ptr_t + !! \htmlinclude ccpp_constituent_prop_ptr_t.html + !! + type, public :: ccpp_constituent_prop_ptr_t + type(ccpp_constituent_properties_t), private, pointer :: prop => null() + contains + ! Informational methods + procedure :: standard_name => ccpt_get_standard_name + procedure :: long_name => ccpt_get_long_name + procedure :: diagnostic_name => ccpt_get_diagnostic_name + procedure :: units => ccpt_get_units + procedure :: is_layer_var => ccpt_is_layer_var + procedure :: is_interface_var => ccpt_is_interface_var + procedure :: is_2d_var => ccpt_is_2d_var + procedure :: vertical_dimension => ccpt_get_vertical_dimension + procedure :: const_index => ccpt_const_index + procedure :: is_advected => ccpt_is_advected + procedure :: is_thermo_active => ccpt_is_thermo_active + procedure :: is_water_species => ccpt_is_water_species + procedure :: is_mass_mixing_ratio => ccpt_is_mass_mixing_ratio + procedure :: is_volume_mixing_ratio => ccpt_is_volume_mixing_ratio + procedure :: is_number_concentration => ccpt_is_number_concentration + procedure :: is_dry => ccpt_is_dry + procedure :: is_moist => ccpt_is_moist + procedure :: is_wet => ccpt_is_wet + procedure :: minimum => ccpt_min_val + procedure :: molar_mass => ccpt_molar_mass + procedure :: default_value => ccpt_default_value + procedure :: has_default => ccpt_has_default + ! ccpt_set: Set the internal pointer + procedure :: set => ccpt_set + ! Methods that change state (XXgoldyXX: make private?) + procedure :: deallocate => ccpt_deallocate + procedure :: set_const_index => ccpt_set_const_index + procedure :: set_thermo_active => ccpt_set_thermo_active + procedure :: set_water_species => ccpt_set_water_species + procedure :: set_minimum => ccpt_set_min_val + procedure :: set_molar_mass => ccpt_set_molar_mass + end type ccpp_constituent_prop_ptr_t + + !! \section arg_table_ccpp_model_constituents_t + !! \htmlinclude ccpp_model_constituents_t.html + !! + type, public :: ccpp_model_constituents_t + ! A ccpp_model_constituents_t object holds all the metadata and field + ! data for a model run's constituents along with data and methods + ! to initialize and access the data. + !!XXgoldyXX: To do: allow accessor functions as CCPP local variable + !! names so that members can be private. + integer :: num_layer_vars = 0 + integer :: num_advected_vars = 0 + integer, private :: num_layers = 0 + type(ccpp_hash_table_t), private :: hash_table + logical, private :: table_locked = .false. + logical, private :: data_locked = .false. + ! These fields are public to allow for efficient (i.e., no copying) + ! usage even though it breaks object independence + real(kind=kind_phys), allocatable :: vars_layer(:, :, :) + real(kind=kind_phys), allocatable :: vars_layer_tend(:, :, :) + real(kind=kind_phys), allocatable :: vars_minvalue(:) + ! An array containing all the constituent metadata + ! Each element contains a pointer to a constituent from the hash table + type(ccpp_constituent_prop_ptr_t), allocatable :: const_metadata(:) + contains + ! Return .true. if a constituent matches pattern + procedure, private :: is_match => ccp_model_const_is_match + ! Return a constituent from the hash table + procedure, private :: find_const => ccp_model_const_find_const + ! Are both the properties table and data array locked (i.e., ready to be used)? + procedure :: locked => ccp_model_const_locked + ! Is the properties table locked (i.e., ready to be used)? + procedure :: const_props_locked => ccp_model_const_props_locked + ! Is the data array locked (i.e., ready to be used)? + procedure :: const_data_locked => ccp_model_const_data_locked + ! Is it okay to add new metadata fields? + procedure :: okay_to_add => ccp_model_const_okay_to_add + ! Add a constituent's metadata to the master hash table + procedure :: new_field => ccp_model_const_add_metadata + ! Initialize hash table + procedure :: initialize_table => ccp_model_const_initialize + ! Freeze hash table and set constituents properties + procedure :: lock_table => ccp_model_const_table_lock + ! Freeze and initialize constituent field arrays + procedure :: lock_data => ccp_model_const_data_lock + ! Empty (reset) the entire object + procedure :: reset => ccp_model_const_reset + ! Query number of constituents matching pattern + procedure :: num_constituents => ccp_model_const_num_match + ! Return index of constituent matching standard name + procedure :: const_index => ccp_model_const_index + ! Return metadata matching standard name + procedure :: field_metadata => ccp_model_const_metadata + ! Gather constituent fields matching pattern + procedure :: copy_in => ccp_model_const_copy_in_3d + ! Update constituent fields matching pattern + procedure :: copy_out => ccp_model_const_copy_out_3d + ! Return pointer to constituent array (for use by host model) + procedure :: field_data_ptr => ccp_field_data_ptr + ! Return pointer to advected constituent array (for use by host model) + procedure :: advected_constituents_ptr => ccp_advected_data_ptr + ! Return pointer to constituent properties array (for use by host model) + procedure :: constituent_props_ptr => ccp_constituent_props_ptr + end type ccpp_model_constituents_t + + ! Private interfaces + private to_str + private initialize_errvars + private append_errvars + private handle_allocate_error + private check_var_bounds + +contains + + !######################################################################## + ! + ! CCPP_CONSTITUENT_PROPERTIES_T (constituent metadata) methods + ! + !######################################################################## + + subroutine copyconstituent(outconst, inconst) + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(inout) :: outconst + type(ccpp_constituent_properties_t), intent(in) :: inconst + + outconst%var_std_name = inconst%var_std_name + outconst%var_long_name = inconst%var_long_name + outconst%var_diag_name = inconst%var_diag_name + outconst%vert_dim = inconst%vert_dim + outconst%const_ind = inconst%const_ind + outconst%advected = inconst%advected + outconst%const_type = inconst%const_type + outconst%const_water = inconst%const_water + outconst%min_val = inconst%min_val + outconst%const_default_value = inconst%const_default_value + outconst%molar_mass_val = inconst%molar_mass_val + outconst%thermo_active = inconst%thermo_active + outconst%water_species = inconst%water_species + outconst%var_units = inconst%var_units + outconst%const_water = inconst%const_water + end subroutine copyconstituent + + !####################################################################### + + character(len=10) function to_str(val) + ! return default integer as a left justified string + + ! Dummy argument + integer, intent(in) :: val + + write(to_str, '(i0)') val + + end function to_str + + !####################################################################### + + subroutine initialize_errvars(errcode, errmsg) + ! Initialize error variables, if present + + ! Dummy arguments + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + if (present(errcode)) then + errcode = 0 + end if + if (present(errmsg)) then + errmsg = '' + end if + end subroutine initialize_errvars + + !####################################################################### + + subroutine append_errvars(errcode_val, errmsg_val, subname, errcode, errmsg, caller) + ! Append to error variables, if present + + ! Dummy arguments + integer, intent(in) :: errcode_val + character(len=*), intent(in) :: errmsg_val + character(len=*), intent(in) :: subname + integer, optional, intent(inout) :: errcode + character(len=*), optional, intent(inout) :: errmsg + character(len=*), optional, intent(in) :: caller + ! Local variable + integer :: emsg_len + + if (present(errcode)) then + errcode = errcode + errcode_val + end if + if (present(errmsg)) then + emsg_len = len_trim(errmsg) + if (emsg_len > 0) then + errmsg(emsg_len + 1:) = '; ' + end if + emsg_len = len_trim(errmsg) + if (present(caller)) then + errmsg(emsg_len + 1:) = trim(caller) // " " // trim(errmsg_val) + else + errmsg(emsg_len + 1:) = trim(subname) // " " // trim(errmsg_val) + end if + end if + end subroutine append_errvars + + !####################################################################### + + subroutine handle_allocate_error(astat, fieldname, subname, errcode, errmsg) + ! Generate an error message if indicates an allocation failure + + ! Dummy arguments + integer, intent(in) :: astat + character(len=*), intent(in) :: fieldname + character(len=*), intent(in) :: subname + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + call initialize_errvars(errcode, errmsg) + if (astat /= 0) then + call append_errvars(astat, "Error allocating ccpp_constituent_properties_t object component " // & + trim(fieldname) // ", error code = " // to_str(astat), subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine handle_allocate_error + + !####################################################################### + + subroutine check_var_bounds(var, var_bound, varname, subname, errcode, errmsg) + ! Generate an error message if indicates an allocation failure + + ! Dummy arguments + integer, intent(in) :: var + integer, intent(in) :: var_bound + character(len=*), intent(in) :: varname + character(len=*), intent(in) :: subname + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + call initialize_errvars(errcode, errmsg) + if (var > var_bound) then + call append_errvars(1, trim(varname) // " exceeds its upper bound, " // & + to_str(var_bound), subname, errcode=errcode, errmsg=errmsg) + end if + end subroutine check_var_bounds + + !####################################################################### + + function ccp_properties_get_key(hashable) + ! Return the constituent properties class key (var_std_name) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: hashable + character(len=:), allocatable :: ccp_properties_get_key + + ccp_properties_get_key = hashable%var_std_name + + end function ccp_properties_get_key + + !####################################################################### + + logical function ccp_is_instantiated(this, errcode, errmsg) + ! Return .true. iff is instantiated + ! If is *not* instantiated and and/or is present, + ! fill these fields with an error status + ! If *is* instantiated and and/or is present, + ! clear these fields. + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + character(len=*), parameter :: subname = 'ccp_is_instantiated' + + ccp_is_instantiated = allocated(this%var_std_name) + call initialize_errvars(errcode, errmsg) + if (.not. ccp_is_instantiated) then + call append_errvars(1, "ccpp_constituent_properties_t object is not initialized", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end function ccp_is_instantiated + + !####################################################################### + + subroutine ccp_instantiate(this, std_name, long_name, diag_name, units, & + vertical_dim, advected, default_value, min_value, molar_mass, water_species, & + mixing_ratio_type, errcode, errmsg) + ! Initialize all fields in + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(inout) :: this + character(len=*), intent(in) :: std_name + character(len=*), intent(in) :: long_name + character(len=*), intent(in) :: diag_name + character(len=*), intent(in) :: units + character(len=*), intent(in) :: vertical_dim + logical, optional, intent(in) :: advected + real(kind=kind_phys), optional, intent(in) :: default_value + real(kind=kind_phys), optional, intent(in) :: min_value + real(kind=kind_phys), optional, intent(in) :: molar_mass + logical, optional, intent(in) :: water_species + character(len=*), optional, intent(in) :: mixing_ratio_type + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + + if (this%is_instantiated()) then + errcode = 1 + write(errmsg, *) 'ccpp_constituent_properties_t object, ', & + trim(std_name), ', is already initialized as ', this%var_std_name + else + errcode = 0 + errmsg = '' + this%var_std_name = trim(std_name) + end if + if (errcode == 0) then + this%var_long_name = trim(long_name) + this%var_diag_name = trim(diag_name) + this%var_units = trim(units) + this%vert_dim = trim(vertical_dim) + if (present(advected)) then + this%advected = advected + else + this%advected = .false. + end if + if (present(default_value)) then + this%const_default_value = default_value + end if + if (present(min_value)) then + this%min_val = min_value + end if + if (present(molar_mass)) then + this%molar_mass_val = molar_mass + end if + if (present(water_species)) then + this%water_species = water_species + end if + end if + if (errcode == 0) then + if (index(this%var_std_name, "volume_mixing_ratio") > 0) then + this%const_type = volume_mixing_ratio + else if (index(this%var_std_name, "number_concentration") > 0) then + this%const_type = number_concentration + else + this%const_type = mass_mixing_ratio + end if + end if + if (errcode == 0) then + ! Determine if this mixing ratio is dry, moist, or "wet". + ! If a type was provided, use that (if it's valid) + if (present(mixing_ratio_type)) then + if (trim(mixing_ratio_type) == 'wet') then + this%const_water = wet_mixing_ratio + else if (trim(mixing_ratio_type) == 'moist') then + this%const_water = moist_mixing_ratio + else if (trim(mixing_ratio_type) == 'dry') then + this%const_water = dry_mixing_ratio + else + errcode = 1 + write(errmsg, *) 'ccp_instantiate: invalid mixing ratio type. ', & + 'Must be one of: "wet", "moist", or "dry". Got: "', & + trim(mixing_ratio_type), '"' + end if + else + ! Otherwise, parse it from the standard name + if (index(this%var_std_name, "wrt_moist_air_and_condensed_water") > 0) then + this%const_water = wet_mixing_ratio + else if (index(this%var_std_name, "wrt_moist_air") > 0) then + this%const_water = moist_mixing_ratio + else + this%const_water = dry_mixing_ratio + end if + end if + end if + if (errcode /= 0) then + call this%deallocate() + end if + end subroutine ccp_instantiate + + !####################################################################### + + subroutine ccp_deallocate(this) + ! Deallocate memory associated with this constituent property object + + ! Dummy argument + class(ccpp_constituent_properties_t), intent(inout) :: this + + if (allocated(this%var_std_name)) then + deallocate(this%var_std_name) + end if + if (allocated(this%var_long_name)) then + deallocate(this%var_long_name) + end if + if (allocated(this%var_diag_name)) then + deallocate(this%var_diag_name) + end if + if (allocated(this%vert_dim)) then + deallocate(this%vert_dim) + end if + this%const_ind = int_unassigned + this%advected = .false. + this%const_type = int_unassigned + this%const_water = int_unassigned + this%const_default_value = kphys_unassigned + this%framework_owns_me = .false. + + end subroutine ccp_deallocate + + !####################################################################### + + logical function ccp_is_framework_owned(this) result(owned) + ! Return .true. if this object's storage was transferred to the framework + ! (via set_framework_owned) and should be deallocated during teardown. + + class(ccpp_constituent_properties_t), intent(in) :: this + + owned = this%framework_owns_me + + end function ccp_is_framework_owned + + !####################################################################### + + subroutine ccp_set_framework_owned(this, value) + ! Mark this object as owned by the framework (value=.true.) when the + ! caller has allocated it on the heap and is transferring ownership. + ! Call before passing to ccpp_model_constituents_t%new_field. + + class(ccpp_constituent_properties_t), intent(inout) :: this + logical, intent(in) :: value + + this%framework_owns_me = value + + end subroutine ccp_set_framework_owned + + !####################################################################### + + subroutine ccp_get_standard_name(this, std_name, errcode, errmsg) + ! Return this constituent's standard name + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + character(len=*), intent(out) :: std_name + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + std_name = this%var_std_name + else + std_name = '' + end if + + end subroutine ccp_get_standard_name + + !####################################################################### + + subroutine ccp_get_long_name(this, long_name, errcode, errmsg) + ! Return this constituent's long name (description) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + character(len=*), intent(out) :: long_name + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + long_name = this%var_long_name + else + long_name = '' + end if + + end subroutine ccp_get_long_name + + !####################################################################### + + subroutine ccp_get_diagnostic_name(this, diag_name, errcode, errmsg) + ! Return this constituent's diagnostic name + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + character(len=*), intent(out) :: diag_name + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + diag_name = this%var_diag_name + else + diag_name = '' + end if + + end subroutine ccp_get_diagnostic_name + + !####################################################################### + + subroutine ccp_get_units(this, units, errcode, errmsg) + ! Return this constituent's units + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + character(len=*), intent(out) :: units + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + units = this%var_units + else + units = '' + end if + + end subroutine ccp_get_units + + !####################################################################### + + subroutine ccp_get_vertical_dimension(this, vert_dim, errcode, errmsg) + ! Return the standard name of this constituent's vertical dimension + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + character(len=*), intent(out) :: vert_dim + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + vert_dim = this%vert_dim + else + vert_dim = '' + end if + + end subroutine ccp_get_vertical_dimension + + !####################################################################### + + logical function ccp_is_layer_var(this) result(is_layer) + ! Return .true. iff this constituent has a layer vertical dimension + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + ! Local variable + character(len=dimname_len) :: dimname + + call this%vertical_dimension(dimname) + is_layer = trim(dimname) == 'vertical_layer_dimension' + + end function ccp_is_layer_var + + !####################################################################### + + logical function ccp_is_interface_var(this) result(is_interface) + ! Return .true. iff this constituent has a interface vertical dimension + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + ! Local variable + character(len=dimname_len) :: dimname + + call this%vertical_dimension(dimname) + is_interface = trim(dimname) == 'vertical_interface_dimension' + + end function ccp_is_interface_var + + !####################################################################### + + logical function ccp_is_2d_var(this) result(is_2d) + ! Return .true. iff this constituent has a 2d vertical dimension + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + ! Local variable + character(len=dimname_len) :: dimname + + call this%vertical_dimension(dimname) + is_2d = len_trim(dimname) == 0 + + end function ccp_is_2d_var + + !####################################################################### + + integer function ccp_const_index(this, errcode, errmsg) + ! Return this constituent's array index (or -1 of not assigned) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + ccp_const_index = this%const_ind + else + ccp_const_index = int_unassigned + end if + + end function ccp_const_index + + !####################################################################### + + subroutine ccp_set_const_index(this, index, errcode, errmsg) + ! Set this constituent's index in the master constituent array + ! It is an error to try to set an index if it is already set + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(inout) :: this + integer, intent(in) :: index + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + character(len=*), parameter :: subname = 'ccp_set_const_index' + + if (this%is_instantiated(errcode, errmsg)) then + if (this%const_ind == int_unassigned) then + this%const_ind = index + else + call append_errvars(1, "ccpp_constituent_properties_t const index " // & + "is already set", subname, errcode=errcode, errmsg=errmsg) + end if + end if + + end subroutine ccp_set_const_index + + !####################################################################### + + subroutine ccp_set_thermo_active(this, thermo_flag, errcode, errmsg) + ! Set whether this constituent is thermodynamically active, which + ! means that certain physics schemes will use this constitutent + ! when calculating thermodynamic quantities (e.g. enthalpy). + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(inout) :: this + logical, intent(in) :: thermo_flag + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + !Set thermodynamically active flag for this constituent: + if (this%is_instantiated(errcode, errmsg)) then + this%thermo_active = thermo_flag + end if + + end subroutine ccp_set_thermo_active + + !####################################################################### + + subroutine ccp_set_water_species(this, water_flag, errcode, errmsg) + ! Set whether this constituent is a water species, which means + ! that this constituent represents a particular phase or type + ! of water in the atmosphere. + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(inout) :: this + logical, intent(in) :: water_flag + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + !Set water species flag for this constituent: + if (this%is_instantiated(errcode, errmsg)) then + this%water_species = water_flag + end if + + end subroutine ccp_set_water_species + + !####################################################################### + + subroutine ccp_is_thermo_active(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + logical, intent(out) :: val_out + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + !If instantiated then check if constituent is + !thermodynamically active, otherwise return false: + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%thermo_active + else + val_out = .false. + end if + end subroutine ccp_is_thermo_active + + !####################################################################### + + subroutine ccp_is_water_species(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + logical, intent(out) :: val_out + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + !If instantiated then check if constituent is + !a water species, otherwise return false: + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%water_species + else + val_out = .false. + end if + end subroutine ccp_is_water_species + + !####################################################################### + + subroutine ccp_is_advected(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + logical, intent(out) :: val_out + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%advected + else + val_out = .false. + end if + end subroutine ccp_is_advected + + !####################################################################### + + subroutine ccp_is_equivalent(this, oconst, equiv, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + type(ccpp_constituent_properties_t), intent(in) :: oconst + logical, intent(out) :: equiv + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg) .and. & + oconst%is_instantiated(errcode, errmsg)) then + equiv = (trim(this%var_std_name) == trim(oconst%var_std_name)) .and. & + (trim(this%var_long_name) == trim(oconst%var_long_name)) .and. & + (trim(this%var_diag_name) == trim(oconst%var_diag_name)) .and. & + (trim(this%vert_dim) == trim(oconst%vert_dim)) .and. & + (trim(this%var_units) == trim(oconst%var_units)) .and. & + (this%advected .eqv. oconst%advected) .and. & + (this%const_default_value == oconst%const_default_value) .and. & + (this%min_val == oconst%min_val) .and. & + (this%molar_mass_val == oconst%molar_mass_val) .and. & + (this%thermo_active .eqv. oconst%thermo_active) .and. & + (this%const_water == oconst%const_water) .and. & + (this%water_species .eqv. oconst%water_species) + else + equiv = .false. + end if + + end subroutine ccp_is_equivalent + + !######################################################################## + + subroutine ccp_is_mass_mixing_ratio(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%const_type == mass_mixing_ratio + else + val_out = .false. + end if + end subroutine ccp_is_mass_mixing_ratio + + !######################################################################## + + subroutine ccp_is_volume_mixing_ratio(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%const_type == volume_mixing_ratio + else + val_out = .false. + end if + end subroutine ccp_is_volume_mixing_ratio + + !######################################################################## + + subroutine ccp_is_number_concentration(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%const_type == number_concentration + else + val_out = .false. + end if + end subroutine ccp_is_number_concentration + + !######################################################################## + + subroutine ccp_is_dry(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%const_water == dry_mixing_ratio + else + val_out = .false. + end if + + end subroutine ccp_is_dry + + !######################################################################## + + subroutine ccp_is_moist(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%const_water == moist_mixing_ratio + else + val_out = .false. + end if + + end subroutine ccp_is_moist + + !######################################################################## + + subroutine ccp_is_wet(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%const_water == wet_mixing_ratio + else + val_out = .false. + end if + + end subroutine ccp_is_wet + + !######################################################################## + + subroutine ccp_min_val(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + real(kind=kind_phys), intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%min_val + else + val_out = kphys_unassigned + end if + + end subroutine ccp_min_val + + !######################################################################## + + subroutine ccp_set_min_val(this, min_value, errcode, errmsg) + ! Set the minimum value of this particular constituent. + ! If this subroutine is never used then the minimum + ! value defaults to zero. + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(inout) :: this + real(kind=kind_phys), intent(in) :: min_value + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + !Set minimum allowed value for this constituent: + if (this%is_instantiated(errcode, errmsg)) then + this%min_val = min_value + end if + + end subroutine ccp_set_min_val + + !######################################################################## + + subroutine ccp_molar_mass(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + real(kind=kind_phys), intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%molar_mass_val + else + val_out = kphys_unassigned + end if + + end subroutine ccp_molar_mass + + !######################################################################## + + subroutine ccp_set_molar_mass(this, molar_mass, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(inout) :: this + real(kind=kind_phys), intent(in) :: molar_mass + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + this%molar_mass_val = molar_mass + end if + + end subroutine ccp_set_molar_mass + + !######################################################################## + + subroutine ccp_default_value(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + real(kind=kind_phys), intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%const_default_value + else + val_out = kphys_unassigned + end if + + end subroutine ccp_default_value + + !######################################################################## + + subroutine ccp_has_default(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccp_has_default' + + if (this%is_instantiated(errcode, errmsg)) then + val_out = this%const_default_value /= kphys_unassigned + else + val_out = .false. + end if + + end subroutine ccp_has_default + + !######################################################################## + + logical function ccp_is_match(this, comp_props) result(is_match) + ! Return .true. iff the constituent's properties match the checked + ! attributes of another constituent properties object + ! Since this is a private function, error checking for locked status + ! is *not* performed. + + ! Dummy arguments + class(ccpp_constituent_properties_t), intent(in) :: this + type(ccpp_constituent_properties_t), intent(in) :: comp_props + ! Local variable + logical :: val, comp_val + character(len=stdname_len) :: char_val, char_comp_val + + ! By default, every constituent is a match + is_match = .true. + ! Check: advected, thermo_active, water_species, units + call this%is_advected(val) + call comp_props%is_advected(comp_val) + if (val .neqv. comp_val) then + is_match = .false. + return + end if + + call this%is_thermo_active(val) + call comp_props%is_thermo_active(comp_val) + if (val .neqv. comp_val) then + is_match = .false. + return + end if + + call this%is_water_species(val) + call comp_props%is_water_species(comp_val) + if (val .neqv. comp_val) then + is_match = .false. + return + end if + + call this%units(char_val) + call comp_props%units(char_comp_val) + if (trim(char_val) /= trim(char_comp_val)) then + is_match = .false. + return + end if + + end function ccp_is_match + + !######################################################################## + ! + ! CCPP_MODEL_CONSTITUENTS_T (constituent field data) methods + ! + !######################################################################## + + logical function ccp_model_const_locked(this, errcode, errmsg, warn_func) + ! Return .true. iff is locked (i.e., ready to use) + ! Optionally fill out and if object not initialized + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(in) :: this + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + character(len=*), optional, intent(in) :: warn_func + ! Local variable + character(len=*), parameter :: subname = 'ccp_model_const_locked' + + call initialize_errvars(errcode, errmsg) + ccp_model_const_locked = .false. + ! Use an initialized hash table as double check + if (this%hash_table%is_initialized()) then + ccp_model_const_locked = this%table_locked .and. this%data_locked + if ((.not. (this%table_locked .and. this%data_locked)) .and. & + present(errmsg) .and. present(warn_func)) then + ! Write a warning as a courtesy to calling function but do not set + ! errcode (let caller decide). + write(errmsg, *) trim(warn_func), & + ' WARNING: Model constituents not ready to use' + end if + else + call append_errvars(1, "WARNING: Model constituents not initialized", & + subname, errcode=errcode, errmsg=errmsg, caller=warn_func) + end if + + end function ccp_model_const_locked + + !######################################################################## + + logical function ccp_model_const_props_locked(this, errcode, errmsg, warn_func) + ! Return .true. iff 's constituent properties are ready to use + ! Optionally fill out and if object not initialized + ! Dummy arguments + class(ccpp_model_constituents_t), intent(in) :: this + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + character(len=*), optional, intent(in) :: warn_func + ! Local variable + character(len=*), parameter :: subname = 'ccp_model_const_table_locked' + + call initialize_errvars(errcode, errmsg) + ccp_model_const_props_locked = .false. + ! Use an initialized hash table as double check + if (this%hash_table%is_initialized()) then + ccp_model_const_props_locked = this%table_locked + if (.not. this%table_locked .and. & + present(errmsg) .and. present(warn_func)) then + ! Write a warning as a courtesy to calling function but do not set + ! errcode (let caller decide). + write(errmsg, *) trim(warn_func), & + ' WARNING: Model constituent properties not ready to use' + end if + else + call append_errvars(1, & + "WARNING: Model constituent properties not initialized", & + subname, errcode=errcode, errmsg=errmsg, caller=warn_func) + end if + + end function ccp_model_const_props_locked + + !######################################################################## + + logical function ccp_model_const_data_locked(this, errcode, errmsg, warn_func) + ! Return .true. iff 's data are ready to use + ! Optionally fill out and if object not initialized + ! Dummy arguments + class(ccpp_model_constituents_t), intent(in) :: this + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + character(len=*), optional, intent(in) :: warn_func + ! Local variable + character(len=*), parameter :: subname = 'ccp_model_const_data_locked' + + call initialize_errvars(errcode, errmsg) + ccp_model_const_data_locked = .false. + ! Use an initialized hash table as double check + if (this%hash_table%is_initialized()) then + ccp_model_const_data_locked = this%data_locked + if (.not. this%data_locked .and. & + present(errmsg) .and. present(warn_func)) then + ! Write a warning as a courtesy to calling function but do not set + ! errcode (let caller decide). + write(errmsg, *) trim(warn_func), & + ' WARNING: Model constituent data not ready to use' + end if + else + call append_errvars(1, & + "WARNING: Model constituent data not initialized", & + subname, errcode=errcode, errmsg=errmsg, caller=warn_func) + end if + + end function ccp_model_const_data_locked + + !######################################################################## + + logical function ccp_model_const_okay_to_add(this, errcode, errmsg, & + warn_func) + ! Return .true. iff is initialized and not locked + ! Optionally fill out and if the conditions + ! are not met. + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(inout) :: this + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + character(len=*), optional, intent(in) :: warn_func + ! Local variable + character(len=*), parameter :: subname = 'ccp_model_const_okay_to_add' + + ccp_model_const_okay_to_add = this%hash_table%is_initialized() + if (ccp_model_const_okay_to_add) then + ccp_model_const_okay_to_add = .not. (this%const_props_locked(errcode=errcode, & + errmsg=errmsg, warn_func=subname) .or. this%const_data_locked(errcode=errcode, & + errmsg=errmsg, warn_func=subname)) + if (.not. ccp_model_const_okay_to_add) then + call append_errvars(1, & + "WARNING: Model constituents are locked", & + subname, errcode=errcode, errmsg=errmsg, caller=warn_func) + end if + else + call append_errvars(1, & + "WARNING: Model constituents not initialized", & + subname, errcode=errcode, errmsg=errmsg, caller=warn_func) + end if + + end function ccp_model_const_okay_to_add + + !######################################################################## + + subroutine ccp_model_const_add_metadata(this, field_data, errcode, errmsg) + ! Add a constituent's metadata to the master hash table + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(inout) :: this + type(ccpp_constituent_properties_t), target, intent(in) :: field_data + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variables + character(len=errmsg_len) :: error + character(len=*), parameter :: subname = 'ccp_model_const_add_metadata' + type(ccpp_constituent_properties_t), pointer :: cprop => null() + character(len=stdname_len) :: standard_name + logical :: match + + if (this%okay_to_add(errcode=errcode, errmsg=errmsg, & + warn_func=subname)) then + error = '' + ! Check to see if standard name is already in the table + call field_data%standard_name(standard_name, errcode, errmsg) + cprop => this%find_const(standard_name) + if (associated(cprop)) then + ! Standard name already in table, let's see if the existing constituent is the same + match = cprop%is_match(field_data) + if (match) then + ! Existing constituent is a match - no need to throw an error, just don't add + return + else + ! Existing constituent is not a match - this is an error + call append_errvars(1, "ERROR: Trying to add constituent " // & + trim(standard_name) // " but an incompatible" // & + " constituent with this name already exists", subname, & + errcode=errcode, errmsg=errmsg) + return + end if + end if + call this%hash_table%add_hash_key(field_data, error) + if (len_trim(error) > 0) then + call append_errvars(1, trim(error), subname, errcode=errcode, errmsg=errmsg) + else + ! If we get here we are successful, add to variable count + if (field_data%is_layer_var()) then + this%num_layer_vars = this%num_layer_vars + 1 + else + if (present(errmsg)) then + call field_data%vertical_dimension(error, & + errcode=errcode, errmsg=errmsg) + if (errcode /= 0) then + call append_errvars(1, & + "ERROR: Unknown vertical dimension, '" // & + trim(error) // "'", subname, & + errcode=errcode, errmsg=errmsg) + end if + end if + end if + end if + else + call append_errvars(1, "WARNING: Model constituents are locked", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccp_model_const_add_metadata + + !######################################################################## + + subroutine ccp_model_const_initialize(this, num_elements) + ! Initialize hash table, is total number of elements + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(inout) :: this + integer, intent(in) :: num_elements + ! Local variable + integer :: tbl_size + + ! Clear any data + call this%reset() + ! Figure a log base 2 for initializing hash table + tbl_size = num_elements * 10 ! Hash padding + tbl_size = int((log(real(tbl_size, kind_phys)) / log(2.0_kind_phys)) + & + 1.0_kind_phys) + ! Initialize hash table + call this%hash_table%initialize(tbl_size) + this%table_locked = .false. + + end subroutine ccp_model_const_initialize + + !######################################################################## + + function ccp_model_const_find_const(this, standard_name, errcode, errmsg) & + result(cprop) + ! Return a constituent with key, , from the hash table + ! must be locked to execute this function + ! Since this is a private function, error checking for locked status + ! is *not* performed. + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(in) :: this + character(len=*), intent(in) :: standard_name + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + type(ccpp_constituent_properties_t), pointer :: cprop + ! Local variables + class(ccpp_hashable_t), pointer :: hval + character(len=errmsg_len) :: error + character(len=*), parameter :: subname = 'ccp_model_const_find_const' + + nullify(cprop) + + hval => this%hash_table%table_value(standard_name, errmsg=error) + if (len_trim(error) > 0) then + call append_errvars(1, trim(error), subname, & + errcode=errcode, errmsg=errmsg) + else + select type (hval) + type is (ccpp_constituent_properties_t) + cprop => hval + class default + call append_errvars(1, "ERROR: Bad hash table value " // & + trim(standard_name), subname, errcode=errcode, errmsg=errmsg) + end select + end if + + end function ccp_model_const_find_const + + !######################################################################## + + subroutine ccp_model_const_table_lock(this, errcode, errmsg) + ! Freeze hash table and initialize constituent properties + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(inout) :: this + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variables + integer :: index_const + integer :: index_advect + integer :: num_vars + integer :: astat + integer :: errcode_local + logical :: check + type(ccpp_hash_iterator_t) :: hiter + class(ccpp_hashable_t), pointer :: hval + type(ccpp_constituent_properties_t), pointer :: cprop + character(len=dimname_len) :: dimname + character(len=*), parameter :: subname = 'ccp_model_const_table_lock' + + astat = 0 + errcode_local = 0 + if (this%const_props_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then + call append_errvars(1, & + "WARNING: Model constituents properties already locked, ignoring", & + subname, errcode=errcode, errmsg=errmsg) + errcode_local = 1 + else + ! Make sure everything is really initialized + call this%reset(clear_hash_table=.false.) + this%num_advected_vars = 0 + ! Allocate the constituent array + num_vars = this%hash_table%num_values() + allocate(this%const_metadata(num_vars), stat=astat) + call handle_allocate_error(astat, 'const_metadata', & + subname, errcode=errcode, errmsg=errmsg) + ! We want to pack the advected constituents at the beginning of + ! the field array so we need to know how many there are + if (astat == 0) then + call hiter%initialize(this%hash_table) + do + if (hiter%valid()) then + hval => hiter%value() + select type (hval) + type is (ccpp_constituent_properties_t) + cprop => hval + call cprop%is_advected(check) + if (check) then + this%num_advected_vars = this%num_advected_vars + 1 + end if + end select + call hiter%next() + else + exit + end if + end do + ! Sanity check on num_advect + if (this%num_advected_vars > num_vars) then + call append_errvars(1, "ERROR: num_advected_vars index " // & + to_str(this%num_advected_vars) // & + " out of bounds " // to_str(num_vars), & + subname, errcode=errcode, errmsg=errmsg) + errcode_local = 1 + end if + end if + index_advect = 0 + index_const = this%num_advected_vars + ! Iterate through the hash table to find entries + if (errcode_local == 0) then + call hiter%initialize(this%hash_table) + do + if (hiter%valid()) then + hval => hiter%value() + select type (hval) + type is (ccpp_constituent_properties_t) + cprop => hval + call cprop%is_advected(check) + if (check) then + index_advect = index_advect + 1 + if (index_advect > this%num_advected_vars) then + call append_errvars(1, "ERROR: const a index " // & + to_str(index_advect) // " out of bounds " // & + to_str(this%num_advected_vars), & + subname, errcode=errcode, errmsg=errmsg) + errcode_local = errcode_local + 1 + exit + end if + call cprop%set_const_index(index_advect, & + errcode=errcode, errmsg=errmsg) + call this%const_metadata(index_advect)%set(cprop) + else + index_const = index_const + 1 + if (index_const > num_vars) then + call append_errvars(1, "ERROR: const v index " // & + to_str(index_const) // " out of bounds " // & + to_str(num_vars), subname, errcode=errcode, & + errmsg=errmsg) + errcode_local = errcode_local + 1 + exit + end if + call cprop%set_const_index(index_const, & + errcode=errcode, errmsg=errmsg) + call this%const_metadata(index_const)%set(cprop) + end if + ! Make sure this is a layer variable + if (.not. cprop%is_layer_var()) then + call cprop%vertical_dimension(dimname, & + errcode=errcode, errmsg=errmsg) + call append_errvars(1, "ERROR: Bad vertical dimension, '" // & + trim(dimname), subname, errcode=errcode, errmsg=errmsg) + errcode_local = errcode_local + 1 + exit + end if + class default + call append_errvars(1, "ERROR: Bad hash table value", & + subname, errcode=errcode, errmsg=errmsg) + errcode_local = errcode_local + 1 + exit + end select + call hiter%next() + else + exit + end if + end do + ! Some size sanity checks + if (index_const /= this%hash_table%num_values()) then + call append_errvars(1, "ERROR: Too few constituents " // & + to_str(index_const) // " found in hash table " // & + to_str(this%hash_table%num_values()), subname, & + errcode=errcode, errmsg=errmsg) + errcode_local = errcode_local + 1 + end if + if (index_advect /= this%num_advected_vars) then + call append_errvars(1, "ERROR: Too few advected constituents " // & + to_str(index_const) // " found in hash table " // & + to_str(this%hash_table%num_values()), subname, & + errcode=errcode, errmsg=errmsg) + errcode_local = errcode_local + 1 + end if + if (present(errcode)) then + if (errcode /= 0) then + errcode_local = 1 + end if + end if + if (errcode_local == 0) then + this%table_locked = .true. + end if + end if + end if + + end subroutine ccp_model_const_table_lock + + !######################################################################## + + subroutine ccp_model_const_data_lock(this, ncols, num_layers, errcode, errmsg) + ! Freeze hash table and initialize constituent arrays + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(inout) :: this + integer, intent(in) :: ncols + integer, intent(in) :: num_layers + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variables + integer :: astat, index, errcode_local + real(kind=kind_phys) :: default_value + real(kind=kind_phys) :: minvalue + character(len=*), parameter :: subname = 'ccp_model_const_data_lock' + + errcode_local = 0 + if (this%const_data_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then + call append_errvars(1, & + "WARNING: Model constituent data already locked, ignoring", & + subname, errcode=errcode, errmsg=errmsg) + errcode_local = errcode_local + 1 + else if (.not. this%const_props_locked(errcode=errcode, errmsg=errmsg, & + warn_func=subname)) then + call append_errvars(1, & + "WARNING: Model constituent properties not yet locked, ignoring", & + subname, errcode=errcode, errmsg=errmsg) + errcode_local = errcode_local + 1 + else + allocate(this%vars_layer(ncols, num_layers, this%hash_table%num_values()), & + stat=astat) + call handle_allocate_error(astat, 'vars_layer', & + subname, errcode=errcode, errmsg=errmsg) + errcode_local = astat + if (astat == 0) then + allocate(this%vars_layer_tend(ncols, num_layers, this%hash_table%num_values()), & + stat=astat) + call handle_allocate_error(astat, 'vars_layer_tend', & + subname, errcode=errcode, errmsg=errmsg) + errcode_local = astat + end if + if (astat == 0) then + allocate(this%vars_minvalue(this%hash_table%num_values()), stat=astat) + call handle_allocate_error(astat, 'vars_minvalue', & + subname, errcode=errcode, errmsg=errmsg) + errcode_local = astat + end if + ! Initialize tendencies to 0 + this%vars_layer_tend(:, :, :) = 0._kind_phys + if (errcode_local == 0) then + this%num_layers = num_layers + do index = 1, this%hash_table%num_values() + !Set all constituents to their default values: + call this%const_metadata(index)%default_value(default_value, & + errcode, errmsg) + this%vars_layer(:, :, index) = default_value + + ! Also set the minimum allowed value array + call this%const_metadata(index)%minimum(minvalue, errcode, & + errmsg) + this%vars_minvalue(index) = minvalue + end do + end if + if (present(errcode)) then + if (errcode /= 0) then + errcode_local = 1 + end if + end if + if (errcode_local == 0) then + this%data_locked = .true. + end if + end if + + end subroutine ccp_model_const_data_lock + + !######################################################################## + + subroutine ccp_model_const_reset(this, clear_hash_table) + ! Empty (reset) the entire object + ! Optionally do not clear the hash table (and its data) + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(inout) :: this + logical, optional, intent(in) :: clear_hash_table + ! Local variables + logical :: clear_table + integer :: index + + if (present(clear_hash_table)) then + clear_table = clear_hash_table + else + clear_table = .true. + end if + if (allocated(this%vars_layer)) then + deallocate(this%vars_layer) + end if + if (allocated(this%vars_minvalue)) then + deallocate(this%vars_minvalue) + end if + if (allocated(this%vars_layer_tend)) then + deallocate(this%vars_layer_tend) + end if + if (allocated(this%const_metadata)) then + if (clear_table) then + do index = 1, size(this%const_metadata, 1) + call this%const_metadata(index)%deallocate() + end do + end if + deallocate(this%const_metadata) + end if + if (clear_table) then + this%num_layer_vars = 0 + this%num_advected_vars = 0 + this%num_layers = 0 + call this%hash_table%clear() + end if + + end subroutine ccp_model_const_reset + + !######################################################################## + + logical function ccp_model_const_is_match(this, index, advected, & + thermo_active, water_species) result(is_match) + ! Return .true. iff the constituent at matches a pattern + ! Each (optional) property which is present represents something + ! which is required as part of a match. + ! Since this is a private function, error checking for locked status + ! is *not* performed. + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(in) :: this + integer, intent(in) :: index + logical, optional, intent(in) :: advected + logical, optional, intent(in) :: thermo_active + logical, optional, intent(in) :: water_species + ! Local variable + logical :: check + + ! By default, every constituent is a match + is_match = .true. + if (present(advected)) then + call this%const_metadata(index)%is_advected(check) + if (advected .neqv. check) then + is_match = .false. + end if + end if + + if (present(thermo_active)) then + call this%const_metadata(index)%is_thermo_active(check) + if (thermo_active .neqv. check) then + is_match = .false. + end if + end if + + if (present(water_species)) then + call this%const_metadata(index)%is_water_species(check) + if (water_species .neqv. check) then + is_match = .false. + end if + end if + + end function ccp_model_const_is_match + + !######################################################################## + + subroutine ccp_model_const_num_match(this, nmatch, advected, thermo_active, & + water_species, errcode, errmsg) + ! Query number of constituents matching pattern + ! Each (optional) property which is present represents something + ! which is required as part of a match. + ! must be locked to execute this function + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(in) :: this + integer, intent(out) :: nmatch + logical, optional, intent(in) :: advected + logical, optional, intent(in) :: thermo_active + logical, optional, intent(in) :: water_species + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variables + integer :: index + character(len=*), parameter :: subname = "ccp_model_const_num_match" + + nmatch = 0 + if (this%const_props_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then + do index = 1, size(this%const_metadata) + if (this%is_match(index, advected=advected, thermo_active=thermo_active, & + water_species=water_species)) then + nmatch = nmatch + 1 + end if + end do + end if + + end subroutine ccp_model_const_num_match + + !######################################################################## + + subroutine ccp_model_const_index(this, index, standard_name, errcode, errmsg) + ! Return index of metadata matching . + ! must be locked to execute this function + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(in) :: this + character(len=*), intent(in) :: standard_name + integer, intent(out) :: index + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variables + type(ccpp_constituent_properties_t), pointer :: cprop => null() + character(len=*), parameter :: subname = "ccp_model_const_index" + + if (this%const_props_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then + cprop => this%find_const(standard_name) + if (associated(cprop)) then + index = cprop%const_index() + else + index = int_unassigned + end if + else + index = int_unassigned + end if + + end subroutine ccp_model_const_index + + !######################################################################## + + subroutine ccp_model_const_metadata(this, standard_name, const_data, & + errcode, errmsg) + ! Return metadata matching standard name + ! must be locked to execute this function + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(in) :: this + character(len=*), intent(in) :: standard_name + type(ccpp_constituent_properties_t), intent(out) :: const_data + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variables + type(ccpp_constituent_properties_t), pointer :: cprop => null() + character(len=*), parameter :: subname = "ccp_model_const_metadata" + + if (this%const_props_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then + cprop => this%find_const(standard_name, errcode=errcode, errmsg=errmsg) + if (associated(cprop)) then + const_data = cprop + end if + end if + + end subroutine ccp_model_const_metadata + + !######################################################################## + + subroutine ccp_model_const_copy_in_3d(this, const_array, advected, & + thermo_active, water_species, errcode, errmsg) + ! Gather constituent fields matching pattern + ! Each (optional) property which is present represents something + ! which is required as part of a match. + ! must be locked to execute this function + + ! Dummy arguments + class(ccpp_model_constituents_t), intent(in) :: this + real(kind=kind_phys), intent(out) :: const_array(:, :, :) + logical, optional, intent(in) :: advected + logical, optional, intent(in) :: thermo_active + logical, optional, intent(in) :: water_species + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variables + integer :: index ! const_metadata index + integer :: cindex ! const_array index + integer :: fld_ind ! const field index + integer :: max_cind ! Size of const_array + integer :: num_levels ! Levels of const_array + character(len=stdname_len) :: std_name + character(len=*), parameter :: subname = "ccp_model_const_copy_in_3d" + + if (this%locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then + cindex = 0 + max_cind = size(const_array, 3) + num_levels = size(const_array, 2) + do index = 1, size(this%const_metadata) + if (this%is_match(index, advected=advected, & + thermo_active=thermo_active, & + water_species=water_species)) then + ! See if we have room for another constituent + cindex = cindex + 1 + if (cindex > max_cind) then + call append_errvars(1, & + ": Too many constituents for ", & + subname, errcode=errcode, errmsg=errmsg) + exit + end if + ! Copy this constituent's field data to + call this%const_metadata(index)%const_index(fld_ind) + if (fld_ind /= index) then + call this%const_metadata(index)%standard_name(std_name) + call append_errvars(1, ": ERROR: " // & + "bad field index, " // to_str(fld_ind) // & + " for '" // trim(std_name) // "', should have been " // & + to_str(index), subname, errcode=errcode, errmsg=errmsg) + exit + else if (this%const_metadata(index)%is_layer_var()) then + if (this%num_layers == num_levels) then + const_array(:, :, cindex) = this%vars_layer(:, :, fld_ind) + else + call this%const_metadata(index)%standard_name(std_name) + call append_errvars(1, ": ERROR: " // & + "Wrong number of vertical levels for '" // & + trim(std_name) // "', " // to_str(num_levels) // & + ", expected " // to_str(this%num_layers), & + subname, errcode=errcode, errmsg=errmsg) + exit + end if + else + call this%const_metadata(index)%standard_name(std_name) + call append_errvars(1, ": Unsupported var type," // & + " wrong number of vertical levels for '" // & + trim(std_name) // "', " // to_str(num_levels) // & + ", expected" // to_str(this%num_layers), & + subname, errcode=errcode, errmsg=errmsg) + exit + end if + end if + end do + end if + + end subroutine ccp_model_const_copy_in_3d + + !######################################################################## + + subroutine ccp_model_const_copy_out_3d(this, const_array, advected, & + thermo_active, water_species, errcode, errmsg) + ! Update constituent fields matching pattern + ! Each (optional) property which is present represents something + ! which is required as part of a match. + ! must be locked to execute this function + + ! Dummy argument + class(ccpp_model_constituents_t), intent(inout) :: this + real(kind=kind_phys), intent(in) :: const_array(:, :, :) + logical, optional, intent(in) :: advected + logical, optional, intent(in) :: thermo_active + logical, optional, intent(in) :: water_species + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variables + integer :: index ! const_metadata index + integer :: cindex ! const_array index + integer :: fld_ind ! const field index + integer :: max_cind ! Size of const_array + integer :: num_levels ! Levels of const_array + character(len=stdname_len) :: std_name + character(len=*), parameter :: subname = "ccp_model_const_copy_out_3d" + + if (this%locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then + cindex = 0 + max_cind = size(const_array, 3) + num_levels = size(const_array, 2) + do index = 1, size(this%const_metadata) + if (this%is_match(index, advected=advected, & + thermo_active=thermo_active, & + water_species=water_species)) then + ! See if we have room for another constituent + cindex = cindex + 1 + if (cindex > max_cind) then + call append_errvars(1, & + ": Too many constituents for ", & + subname, errcode=errcode, errmsg=errmsg) + exit + end if + ! Copy this field of to to constituent's field data + call this%const_metadata(index)%const_index(fld_ind) + if (fld_ind /= index) then + call this%const_metadata(index)%standard_name(std_name) + call append_errvars(1, ": ERROR: " // & + "bad field index, " // to_str(fld_ind) // & + " for '" // trim(std_name) // "', should have been" // & + to_str(index), subname, errcode=errcode, errmsg=errmsg) + exit + else if (this%const_metadata(index)%is_layer_var()) then + if (this%num_layers == num_levels) then + this%vars_layer(:, :, fld_ind) = const_array(:, :, cindex) + else + call this%const_metadata(index)%standard_name(std_name) + call append_errvars(1, & + ": Wrong number of vertical levels for '" // & + trim(std_name) // "', " // to_str(num_levels) // & + ", expected" // to_str(this%num_layers), & + subname, errcode=errcode, errmsg=errmsg) + exit + end if + else + call this%const_metadata(index)%standard_name(std_name) + call append_errvars(1, ": Unsupported var type," // & + " wrong number of vertical levels for'" // & + trim(std_name) // "', " // to_str(num_levels) // & + ", expected " // to_str(this%num_layers), & + subname, errcode=errcode, errmsg=errmsg) + exit + end if + end if + end do + end if + + end subroutine ccp_model_const_copy_out_3d + + !######################################################################## + + function ccp_field_data_ptr(this) result(const_ptr) + ! Return pointer to constituent array (for use by host model) + + ! Dummy arguments + class(ccpp_model_constituents_t), target, intent(inout) :: this + real(kind=kind_phys), pointer :: const_ptr(:, :, :) + ! Local variables + integer :: errcode + character(len=errmsg_len) :: errmsg + character(len=*), parameter :: subname = 'ccp_field_data_ptr' + + if (this%locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then + const_ptr => this%vars_layer + else + ! We don't want output variables in a function so just nullify + ! See note above about creating a 'last_error' method + nullify(const_ptr) + end if + + end function ccp_field_data_ptr + + !######################################################################## + + function ccp_advected_data_ptr(this) result(const_ptr) + ! Return pointer to advected constituent array (for use by host model) + + ! Dummy arguments + class(ccpp_model_constituents_t), target, intent(inout) :: this + real(kind=kind_phys), pointer :: const_ptr(:, :, :) + ! Local variables + integer :: errcode + character(len=errmsg_len) :: errmsg + character(len=*), parameter :: subname = 'ccp_advected_data_ptr' + + if (this%locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then + const_ptr => this%vars_layer(:, :, 1:this%num_advected_vars) + else + ! We don't want output variables in a function so just nullify + ! See note above about creating a 'last_error' method + nullify(const_ptr) + end if + + end function ccp_advected_data_ptr + + function ccp_constituent_props_ptr(this) result(const_ptr) + ! Return pointer to constituent properties array (for use by host model) + + ! Dummy arguments + class(ccpp_model_constituents_t), target, intent(inout) :: this + type(ccpp_constituent_prop_ptr_t), pointer :: const_ptr(:) + ! Local variables + integer :: errcode + character(len=errmsg_len) :: errmsg + character(len=*), parameter :: subname = 'ccp_constituent_props_ptr' + + if (this%const_props_locked(errcode=errcode, errmsg=errmsg, warn_func=subname)) then + const_ptr => this%const_metadata + else + ! We don't want output variables in a function so just nullify + ! See note above about creating a 'last_error' method + nullify(const_ptr) + end if + + end function ccp_constituent_props_ptr + + !######################################################################## + + !##################################### + ! ccpp_constituent_prop_ptr_t methods + !##################################### + + !####################################################################### + + subroutine ccpt_get_standard_name(this, std_name, errcode, errmsg) + ! Return this constituent's standard name + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + character(len=*), intent(out) :: std_name + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_get_standard_name' + + if (associated(this%prop)) then + call this%prop%standard_name(std_name, errcode, errmsg) + else + std_name = '' + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_get_standard_name + + !####################################################################### + + subroutine ccpt_get_long_name(this, long_name, errcode, errmsg) + ! Return this constituent's long name (description) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + character(len=*), intent(out) :: long_name + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_get_long_name' + + if (associated(this%prop)) then + call this%prop%long_name(long_name, errcode, errmsg) + else + long_name = '' + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_get_long_name + + !####################################################################### + + subroutine ccpt_get_diagnostic_name(this, diag_name, errcode, errmsg) + ! Return this constituent's diagnostic name + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + character(len=*), intent(out) :: diag_name + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_get_diagnostic_name' + + if (associated(this%prop)) then + call this%prop%diagnostic_name(diag_name, errcode, errmsg) + else + diag_name = '' + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_get_diagnostic_name + + !####################################################################### + + subroutine ccpt_get_units(this, units, errcode, errmsg) + ! Return this constituent's units + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + character(len=*), intent(out) :: units + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_get_units' + + if (associated(this%prop)) then + call this%prop%units(units, errcode, errmsg) + else + units = '' + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_get_units + + !####################################################################### + + subroutine ccpt_get_vertical_dimension(this, vert_dim, errcode, errmsg) + ! Return the standard name of this constituent's vertical dimension + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + character(len=*), intent(out) :: vert_dim + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_get_vertical_dimension' + + if (associated(this%prop)) then + if (this%prop%is_instantiated(errcode, errmsg)) then + call this%prop%vertical_dimension(vert_dim, errcode, errmsg) + end if + else + vert_dim = '' + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_get_vertical_dimension + + !####################################################################### + + logical function ccpt_is_layer_var(this) result(is_layer) + ! Return .true. iff this constituent has a layer vertical dimension + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + ! Local variables + character(len=dimname_len) :: dimname + character(len=*), parameter :: subname = 'ccpt_is_layer_var' + + if (associated(this%prop)) then + call this%prop%vertical_dimension(dimname) + is_layer = trim(dimname) == 'vertical_layer_dimension' + else + is_layer = .false. + end if + + end function ccpt_is_layer_var + + !####################################################################### + + logical function ccpt_is_interface_var(this) result(is_interface) + ! Return .true. iff this constituent has a interface vertical dimension + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + ! Local variables + character(len=dimname_len) :: dimname + character(len=*), parameter :: subname = 'ccpt_is_interface_var' + + if (associated(this%prop)) then + call this%prop%vertical_dimension(dimname) + is_interface = trim(dimname) == 'vertical_interface_dimension' + else + is_interface = .false. + end if + + end function ccpt_is_interface_var + + !####################################################################### + + logical function ccpt_is_2d_var(this) result(is_2d) + ! Return .true. iff this constituent has a 2d vertical dimension + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + ! Local variables + character(len=dimname_len) :: dimname + character(len=*), parameter :: subname = 'ccpt_is_2d_var' + + if (associated(this%prop)) then + call this%prop%vertical_dimension(dimname) + is_2d = len_trim(dimname) == 0 + else + is_2d = .false. + end if + + end function ccpt_is_2d_var + + !####################################################################### + + subroutine ccpt_const_index(this, index, errcode, errmsg) + ! Return this constituent's master index (or -1 of not assigned) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + integer, intent(out) :: index + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_const_index' + + if (associated(this%prop)) then + index = this%prop%const_index(errcode, errmsg) + else + index = int_unassigned + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_const_index + + !####################################################################### + + subroutine ccpt_is_thermo_active(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + logical, intent(out) :: val_out + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_is_thermo_active' + + if (associated(this%prop)) then + call this%prop%is_thermo_active(val_out, errcode, errmsg) + else + val_out = .false. + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_is_thermo_active + + !####################################################################### + + subroutine ccpt_is_water_species(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + logical, intent(out) :: val_out + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_is_water_species' + + if (associated(this%prop)) then + call this%prop%is_water_species(val_out, errcode, errmsg) + else + val_out = .false. + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_is_water_species + + !####################################################################### + + subroutine ccpt_is_advected(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + logical, intent(out) :: val_out + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_is_advected' + + if (associated(this%prop)) then + call this%prop%is_advected(val_out, errcode, errmsg) + else + val_out = .false. + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_is_advected + + !######################################################################## + + subroutine ccpt_is_mass_mixing_ratio(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_is_mass_mixing_ratio' + + if (associated(this%prop)) then + call this%prop%is_mass_mixing_ratio(val_out, errcode, errmsg) + else + val_out = .false. + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_is_mass_mixing_ratio + + !######################################################################## + + subroutine ccpt_is_volume_mixing_ratio(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_is_volume_mixing_ratio' + + if (associated(this%prop)) then + call this%prop%is_volume_mixing_ratio(val_out, errcode, errmsg) + else + val_out = .false. + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_is_volume_mixing_ratio + + !######################################################################## + + subroutine ccpt_is_number_concentration(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_is_number_concentration' + + if (associated(this%prop)) then + call this%prop%is_number_concentration(val_out, errcode, errmsg) + else + val_out = .false. + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_is_number_concentration + + !######################################################################## + + subroutine ccpt_is_dry(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_is_dry' + + if (associated(this%prop)) then + call this%prop%is_dry(val_out, errcode, errmsg) + else + val_out = .false. + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_is_dry + + !######################################################################## + + subroutine ccpt_is_moist(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_is_moist' + + if (associated(this%prop)) then + call this%prop%is_moist(val_out, errcode, errmsg) + else + val_out = .false. + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_is_moist + + !######################################################################## + + subroutine ccpt_is_wet(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_is_wet' + + if (associated(this%prop)) then + call this%prop%is_wet(val_out, errcode, errmsg) + else + val_out = .false. + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_is_wet + + !######################################################################## + + subroutine ccpt_min_val(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + real(kind=kind_phys), intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_min_val' + + if (associated(this%prop)) then + call this%prop%minimum(val_out, errcode, errmsg) + else + val_out = kphys_unassigned + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_min_val + + !######################################################################## + + subroutine ccpt_set_min_val(this, min_value, errcode, errmsg) + ! Set the minimum value of this particular constituent. + ! If this subroutine is never used then the minimum + ! value defaults to zero. + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(inout) :: this + real(kind=kind_phys), intent(in) :: min_value + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_set_min_val' + + !Set minimum value for this constituent: + if (associated(this%prop)) then + call this%prop%set_minimum(min_value, errcode, errmsg) + else + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_set_min_val + + !######################################################################## + + subroutine ccpt_molar_mass(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + real(kind=kind_phys), intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_molar_mass' + + if (associated(this%prop)) then + call this%prop%molar_mass(val_out, errcode, errmsg) + else + val_out = kphys_unassigned + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_molar_mass + + !######################################################################## + + subroutine ccpt_set_molar_mass(this, molar_mass, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(inout) :: this + real(kind=kind_phys), intent(in) :: molar_mass + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_set_molar_mass' + + if (associated(this%prop)) then + call this%prop%set_molar_mass(molar_mass, errcode, errmsg) + else + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_set_molar_mass + + !######################################################################## + + subroutine ccpt_default_value(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + real(kind=kind_phys), intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_default_value' + + if (associated(this%prop)) then + call this%prop%default_value(val_out, errcode, errmsg) + else + val_out = kphys_unassigned + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_default_value + + !######################################################################## + + subroutine ccpt_has_default(this, val_out, errcode, errmsg) + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(in) :: this + logical, intent(out) :: val_out + integer, intent(out) :: errcode + character(len=*), intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_has_default' + + if (associated(this%prop)) then + call this%prop%has_default(val_out, errcode, errmsg) + else + val_out = .false. + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_has_default + + !######################################################################## + + subroutine ccpt_set(this, const_ptr, errcode, errmsg) + ! Set the pointer to , however, an error is recorded if + ! the pointer is already set. + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(inout) :: this + type(ccpp_constituent_properties_t), pointer :: const_ptr + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variables + character(len=stdname_len) :: stdname + character(len=errmsg_len) :: errmsg2 + character(len=*), parameter :: subname = 'ccpt_set' + + call initialize_errvars(errcode, errmsg) + if (associated(this%prop)) then + call this%standard_name(stdname, errcode=errcode, errmsg=errmsg2) + if (errcode == 0) then + write(errmsg2, *) "Pointer already allocated as '", & + trim(stdname), "'" + end if + errcode = errcode + 1 + call append_errvars(1, trim(errmsg2), subname, errcode=errcode, & + errmsg=errmsg) + else + this%prop => const_ptr + end if + + end subroutine ccpt_set + + !######################################################################## + + subroutine ccpt_deallocate(this) + ! Release the wrapper's reference to its constituent property object. + ! If the framework owns the object (i.e., the caller of + ! ccpp_model_constituents_t%new_field allocated it and flipped its + ! framework_owns_me flag via set_framework_owned), free its internal + ! storage and deallocate the object itself. Otherwise, the object is + ! caller-owned and we only drop our pointer to it. + + ! Dummy argument + class(ccpp_constituent_prop_ptr_t), intent(inout) :: this + + if (associated(this%prop)) then + if (this%prop%is_framework_owned()) then + call this%prop%deallocate() + deallocate(this%prop) + end if + nullify(this%prop) + end if + + end subroutine ccpt_deallocate + + !####################################################################### + + subroutine ccpt_set_const_index(this, index, errcode, errmsg) + ! Set this constituent's index in the master constituent array + ! It is an error to try to set an index if it is already set + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(inout) :: this + integer, intent(in) :: index + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_set_const_index' + + if (associated(this%prop)) then + if (this%prop%is_instantiated(errcode, errmsg)) then + if (this%prop%const_ind == int_unassigned) then + this%prop%const_ind = index + else + call append_errvars(1, "ccpp_constituent_prop_ptr_t " // & + "const index is already set", & + subname, errcode=errcode, errmsg=errmsg) + end if + end if + else + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_set_const_index + + !####################################################################### + + subroutine ccpt_set_thermo_active(this, thermo_flag, errcode, errmsg) + ! Set whether this constituent is thermodynamically active, which + ! means that certain physics schemes will use this constitutent + ! when calculating thermodynamic quantities (e.g. enthalpy). + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(inout) :: this + logical, intent(in) :: thermo_flag + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_set_thermo_active' + + if (associated(this%prop)) then + if (this%prop%is_instantiated(errcode, errmsg)) then + this%prop%thermo_active = thermo_flag + end if + else + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_set_thermo_active + + !####################################################################### + + subroutine ccpt_set_water_species(this, water_flag, errcode, errmsg) + ! Set whether this constituent is a water species, which means + ! that this constituent represents a particular phase or type + ! of water in the atmosphere. + + ! Dummy arguments + class(ccpp_constituent_prop_ptr_t), intent(inout) :: this + logical, intent(in) :: water_flag + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + ! Local variable + character(len=*), parameter :: subname = 'ccpt_set_water_species' + + if (associated(this%prop)) then + if (this%prop%is_instantiated(errcode, errmsg)) then + this%prop%water_species = water_flag + end if + else + call append_errvars(1, ": invalid constituent pointer", & + subname, errcode=errcode, errmsg=errmsg) + end if + + end subroutine ccpt_set_water_species + +end module ccpp_constituent_prop_mod diff --git a/capgen-ng/src/ccpp_constituent_prop_mod.meta b/capgen-ng/src/ccpp_constituent_prop_mod.meta new file mode 100644 index 00000000..657f3d8e --- /dev/null +++ b/capgen-ng/src/ccpp_constituent_prop_mod.meta @@ -0,0 +1,63 @@ +######################################################################## + +[ccpp-table-properties] + name = ccpp_constituent_properties_t + type = ddt + +[ccpp-arg-table] + name = ccpp_constituent_properties_t + type = ddt + +######################################################################## + +[ccpp-table-properties] + name = ccpp_constituent_prop_ptr_t + type = ddt + +[ccpp-arg-table] + name = ccpp_constituent_prop_ptr_t + type = ddt + +######################################################################## +[ccpp-table-properties] + name = ccpp_model_constituents_t + type = ddt + +[ccpp-arg-table] + name = ccpp_model_constituents_t + type = ddt +[ num_layer_vars ] + standard_name = number_of_ccpp_constituents + long_name = Number of constituents managed by CCPP Framework + units = count + dimensions = () + type = integer +[ num_advected_vars ] + standard_name = number_of_ccpp_advected_constituents + long_name = Number of advected constituents managed by CCPP Framework + units = count + dimensions = () + type = integer +[ vars_layer ] + standard_name = ccpp_constituents + long_name = Array of constituents managed by CCPP Framework + units = none + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + type = real | kind = kind_phys +[ vars_layer_tend ] + standard_name = ccpp_constituent_tendencies + long_name = Array of constituent tendencies managed by CCPP Framework + units = none + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + type = real | kind = kind_phys +[ const_metadata ] + standard_name = ccpp_constituent_properties + units = None + type = ccpp_constituent_prop_ptr_t + dimensions = (number_of_ccpp_constituents) +[ vars_minvalue ] + standard_name = ccpp_constituent_minimum_values + units = kg kg-1 + type = real | kind = kind_phys + dimensions = (number_of_ccpp_constituents) + protected = True diff --git a/capgen-ng/src/ccpp_hash_table.F90 b/capgen-ng/src/ccpp_hash_table.F90 new file mode 100644 index 00000000..9f175a3a --- /dev/null +++ b/capgen-ng/src/ccpp_hash_table.F90 @@ -0,0 +1,520 @@ +!!XXgoldyXX: To do, statistics output +module ccpp_hash_table + + use ccpp_hashable, only: ccpp_hashable_t + + implicit none + private + + ! + ! Constants used in hashing function gen_hash_key. + ! + + integer, parameter :: gen_hash_key_offset = 21467 ! z'000053db' + + integer, parameter :: tbl_max_idx = 15 + integer, parameter, dimension(0:tbl_max_idx) :: tbl_gen_hash_key = & + (/ 61, 59, 53, 47, 43, 41, 37, 31, 29, 23, 17, 13, 11, 7, 3, 1 /) + + integer, parameter :: table_factor_size = 8 ! Table size / # entries + integer, parameter :: table_overflow_factor = 4 ! # entries / Overflow size + + type :: table_entry_t + ! Any table entry contains a key and a value + class(ccpp_hashable_t), pointer :: entry_value => null() + type(table_entry_t), pointer :: next => null() + contains + final :: finalize_table_entry + end type table_entry_t + + type, public :: ccpp_hash_table_t + ! ccpp_hash_table_t contains all information to build and use a hash table + ! It also keeps track of statistics such as collision frequency and size + integer, private :: table_size = -1 + integer, private :: key_offset = gen_hash_key_offset + type(table_entry_t), private, allocatable :: table(:) + ! Statistics + integer, private :: num_keys = 0 + integer, private :: num_key_collisions = 0 + integer, private :: max_collision = 0 + contains + procedure :: is_initialized => hash_table_is_initialized + procedure :: initialize => hash_table_initialize_table + procedure :: key_hash => hash_table_key_hash + procedure :: add_hash_key => hash_table_add_hash_key + procedure :: table_value => hash_table_table_value + procedure :: num_values => hash_table_num_values + procedure :: clear => hash_table_clear_table + end type ccpp_hash_table_t + + type, public :: ccpp_hash_iterator_t + ! ccpp_hash_iterator contains information allowing iteration through all + ! entries in a hash table + integer, private :: index = 0 + type(table_entry_t), private, pointer :: table_entry => null() + type(ccpp_hash_table_t), private, pointer :: hash_table => null() + contains + procedure :: initialize => hash_iterator_initialize + procedure :: key => hash_iterator_key + procedure :: next => hash_iterator_next_entry + procedure :: valid => hash_iterator_is_valid + procedure :: value => hash_iterator_value + end type ccpp_hash_iterator_t + + !! Private interfaces + private :: have_error ! Has a called routine detected an error? + private :: clear_optstring ! Clear a string, if present + +contains + + !####################################################################### + ! + ! Hash table methods + ! + !####################################################################### + + logical function have_error(errmsg) + ! Return .true. iff is present and contains text + + ! Dummy argument + character(len=*), optional, intent(in) :: errmsg + + have_error = present(errmsg) + if (have_error) then + have_error = len_trim(errmsg) > 0 + end if + end function have_error + + !####################################################################### + + subroutine clear_optstring(str) + ! clear if it is present + + ! Dummy argument + character(len=*), optional, intent(inout) :: str + + if (present(str)) then + str = '' + end if + end subroutine clear_optstring + + !####################################################################### + + elemental subroutine finalize_table_entry(te) + + ! Dummy argument + type(table_entry_t), intent(inout) :: te + ! Local variable + type(table_entry_t), pointer :: temp + + if (associated(te%entry_value)) then + nullify(te%entry_value) ! We may not own the memory + temp => te%next + nullify(te%next) + if (associated(temp)) then + deallocate(temp) + nullify(temp) + end if + end if + + end subroutine finalize_table_entry + + !####################################################################### + + logical function hash_table_is_initialized(this) + ! Return .true. iff is an initialized hash table + + ! Dummy argument + class(ccpp_hash_table_t) :: this + + hash_table_is_initialized = allocated(this%table) + + end function hash_table_is_initialized + + !####################################################################### + + subroutine hash_table_initialize_table(this, tbl_size, key_off) + ! Initialize this table. + + ! Dummy arguments + class(ccpp_hash_table_t) :: this + integer, intent(in) :: tbl_size ! new table size + integer, optional, intent(in) :: key_off ! key offset + + ! Clear this table so it can be initialized + if (allocated(this%table)) then + deallocate(this%table) + end if + this%num_keys = 0 + this%num_key_collisions = 0 + this%max_collision = 0 + ! Avoid too-large tables + this%table_size = ishft(1, min(tbl_size, bit_size(1) - 2)) + allocate(this%table(this%table_size)) + if (present(key_off)) then + this%key_offset = key_off + end if + end subroutine hash_table_initialize_table + + !####################################################################### + + integer function hash_table_key_hash(this, string, errmsg) result(hash_key) + ! + !----------------------------------------------------------------------- + ! + ! Purpose: Generate a hash key on the interval [0 .. tbl_hash_pri_sz-1] + ! given a character string. + ! + ! Algorithm is a variant of perl's internal hashing function. + ! + !----------------------------------------------------------------------- + ! + ! + ! Dummy Arguments: + ! + class(ccpp_hash_table_t) :: this + character(len=*), intent(in) :: string + character(len=*), optional, intent(out) :: errmsg + character(len=*), parameter :: subname = 'HASH_TABLE_KEY_HASH' + ! + ! Local. + ! + integer :: hash + integer :: index + integer :: ind_fact + integer :: hash_fact + + hash = this%key_offset + ind_fact = 0 + do index = 1, len_trim(string) + ind_fact = ind_fact + 1 + if (ind_fact > tbl_max_idx) then + ind_fact = 1 + end if + hash_fact = tbl_gen_hash_key(ind_fact) + hash = ieor(hash, (ichar(string(index:index)) * hash_fact)) + end do + + hash_key = iand(hash, this%table_size - 1) + 1 + if ((hash_key < 1) .or. (hash_key > this%table_size)) then + if (present(errmsg)) then + write(errmsg, '(2a,2(i0,a))') subname, ' ERROR: Key Hash, ', & + hash_key, ' out of bounds, [1, ', this%table_size, ']' + else + write(6, '(2a,2(i0,a))') subname, ' ERROR: Key Hash, ', & + hash_key, ' out of bounds, [1, ', this%table_size, ']' + stop 1 + end if + end if + + end function hash_table_key_hash + + !####################################################################### + + function hash_table_table_value(this, key, errmsg) result(tbl_val) + ! + !----------------------------------------------------------------------- + ! + ! Purpose: Return the the key value of + ! + ! If the object is not found, return NULL + ! + !----------------------------------------------------------------------- + ! + ! Dummy Arguments: + ! + class(ccpp_hash_table_t) :: this + character(len=*), intent(in) :: key + character(len=*), optional, intent(out) :: errmsg + class(ccpp_hashable_t), pointer :: tbl_val + ! + ! Local. + ! + integer :: hash_key + type(table_entry_t), pointer :: next_ptr + character(len=*), parameter :: subname = 'HASH_TABLE_TABLE_INDEX' + + call clear_optstring(errmsg) + nullify(tbl_val) + hash_key = this%key_hash(key, errmsg=errmsg) + if (have_error(errmsg)) then + errmsg = trim(errmsg) // ', called from ' // subname + else if (associated(this%table(hash_key)%entry_value)) then + if (this%table(hash_key)%entry_value%key() == trim(key)) then + tbl_val => this%table(hash_key)%entry_value + else + next_ptr => this%table(hash_key)%next + do + if (associated(next_ptr)) then + if (associated(next_ptr%entry_value)) then + if (next_ptr%entry_value%key() == trim(key)) then + tbl_val => next_ptr%entry_value + exit + end if + end if + next_ptr => next_ptr%next + else + exit + end if + end do + end if + end if + + if ((.not. associated(tbl_val)) .and. present(errmsg)) then + if (.not. have_error(errmsg)) then ! Still need to test for empty + write(errmsg, *) subname, ": No entry for '", trim(key), "'" + end if + end if + + end function hash_table_table_value + + !####################################################################### + + subroutine hash_table_add_hash_key(this, newval, errmsg) + ! + !----------------------------------------------------------------------- + ! + ! Purpose: Add to this hash table using its key + ! Its key must not be an empty string + ! It is an error to try to add a key more than once + ! + ! + !----------------------------------------------------------------------- + + ! Dummy arguments: + class(ccpp_hash_table_t) :: this + class(ccpp_hashable_t), target :: newval + character(len=*), optional, intent(out) :: errmsg + ! Local variables + integer :: hash_ind + integer :: ovflw_len + character(len=:), allocatable :: newkey + type(table_entry_t), pointer :: next_ptr + type(table_entry_t), pointer :: new_entry + character(len=*), parameter :: subname = 'HASH_TABLE_ADD_HASH_KEY' + + call clear_optstring(errmsg) + nullify(new_entry) + newkey = newval%key() + hash_ind = this%key_hash(newkey, errmsg=errmsg) + ! Check for this entry + if (have_error(errmsg)) then + errmsg = trim(errmsg) // ', called from ' // subname + else if (associated(this%table_value(newkey))) then + if (present(errmsg)) then + write(errmsg, *) subname, " ERROR: key, '", newkey, & + "' already in table" + end if + else + if (associated(this%table(hash_ind)%entry_value)) then + ! We have a collision, make a new entry + allocate(new_entry) + new_entry%entry_value => newval + ! Now, find a spot + if (associated(this%table(hash_ind)%next)) then + ovflw_len = 1 + next_ptr => this%table(hash_ind)%next + do + if (associated(next_ptr%next)) then + ovflw_len = ovflw_len + 1 + next_ptr => next_ptr%next + else + exit + end if + end do + ovflw_len = ovflw_len + 1 + next_ptr%next => new_entry + else + this%num_key_collisions = this%num_key_collisions + 1 + this%table(hash_ind)%next => new_entry + ovflw_len = 1 + end if + nullify(new_entry) + this%max_collision = max(this%max_collision, ovflw_len) + else + this%table(hash_ind)%entry_value => newval + end if + this%num_keys = this%num_keys + 1 + end if + + end subroutine hash_table_add_hash_key + + !####################################################################### + + integer function hash_table_num_values(this) result(numval) + ! + !----------------------------------------------------------------------- + ! + ! Purpose: Return the number of populated table values + ! + !----------------------------------------------------------------------- + + ! Dummy argument: + class(ccpp_hash_table_t) :: this + + numval = this%num_keys + + end function hash_table_num_values + + !####################################################################### + + subroutine hash_table_clear_table(this) + ! + !----------------------------------------------------------------------- + ! + ! Purpose: Deallocate the hash table and all of its entries + ! + !----------------------------------------------------------------------- + + ! Dummy argument: + class(ccpp_hash_table_t) :: this + + ! Clear all the table entries + if (this%is_initialized()) then + if (allocated(this%table)) then + ! This should deallocate the entire chain of entries + deallocate(this%table) + end if + end if + this%table_size = -1 + this%num_keys = 0 + this%num_key_collisions = 0 + this%max_collision = 0 + + end subroutine hash_table_clear_table + + !####################################################################### + ! + ! Hash iterator methods + ! + !####################################################################### + + subroutine hash_iterator_initialize(this, hash_table) + ! Initialize a hash_table iterator to the first value in the hash table + ! Note that the table_entry pointer is only used for the "next" field + ! in the hash table (entry itself is not a pointer). + + ! Dummy arguments + class(ccpp_hash_iterator_t) :: this + class(ccpp_hash_table_t), target :: hash_table + + this%hash_table => hash_table + this%index = 0 + nullify(this%table_entry) + do + this%index = this%index + 1 + if (associated(hash_table%table(this%index)%entry_value)) then + exit + else if (this%index > hash_table%table_size) then + this%index = 0 + end if + end do + end subroutine hash_iterator_initialize + + !####################################################################### + + function hash_iterator_key(this) result(key) + ! Return the key for this hash iterator entry + + ! Dummy arguments + class(ccpp_hash_iterator_t) :: this + character(len=:), allocatable :: key + + if (this%valid()) then + if (associated(this%table_entry)) then + key = this%table_entry%entry_value%key() + else + key = this%hash_table%table(this%index)%entry_value%key() + end if + else + key = '' + end if + + end function hash_iterator_key + + !####################################################################### + + subroutine hash_iterator_next_entry(this) + ! Set the iterator to the next valid hash table value + + ! Dummy argument + class(ccpp_hash_iterator_t) :: this + ! Local variable + logical :: has_table_entry + logical :: has_table_next + + if (this%index > 0) then + ! We have initialized this table, so look for next entry + has_table_entry = associated(this%table_entry) + if (has_table_entry) then + has_table_next = associated(this%table_entry%next) + else + has_table_next = .false. + end if + if (has_table_next) then + this%table_entry => this%table_entry%next + else if ((.not. has_table_entry) .and. & + associated(this%hash_table%table(this%index)%next)) then + this%table_entry => this%hash_table%table(this%index)%next + else + do + if (this%index >= this%hash_table%table_size) then + this%index = 0 + nullify(this%table_entry) + exit + else + this%index = this%index + 1 + nullify(this%table_entry) + associate(t_entry => this%hash_table%table(this%index)) + if (associated(t_entry%entry_value)) then + exit + end if + end associate + end if + end do + end if + else + ! This is an invalid iterator state + nullify(this%table_entry) + end if + + end subroutine hash_iterator_next_entry + + !####################################################################### + + logical function hash_iterator_is_valid(this) result(valid) + ! Return .true. iff this iterator is in a valid (active entry) state + + ! Dummy arguments + class(ccpp_hash_iterator_t) :: this + + valid = .false. + if ((this%index > 0) .and. & + (this%index <= this%hash_table%table_size)) then + valid = .true. + end if + + end function hash_iterator_is_valid + + !####################################################################### + + function hash_iterator_value(this) result(val) + ! Return the value or this hash iterator entry + + ! Dummy arguments + class(ccpp_hash_iterator_t) :: this + class(ccpp_hashable_t), pointer :: val + + if (this%valid()) then + if (associated(this%table_entry)) then + val => this%table_entry%entry_value + else + val => this%hash_table%table(this%index)%entry_value + end if + else + nullify(val) + end if + + end function hash_iterator_value + +end module ccpp_hash_table diff --git a/capgen-ng/src/ccpp_hashable.F90 b/capgen-ng/src/ccpp_hashable.F90 new file mode 100644 index 00000000..21a70902 --- /dev/null +++ b/capgen-ng/src/ccpp_hashable.F90 @@ -0,0 +1,98 @@ +module ccpp_hashable + + implicit none + private + + ! Public interfaces + public :: new_hashable_char + public :: new_hashable_int + + type, abstract, public :: ccpp_hashable_t + ! The hashable type is a base type that contains a hash key. + contains + procedure(ccpp_hashable_get_key), deferred :: key + end type ccpp_hashable_t + + type, public, extends(ccpp_hashable_t) :: ccpp_hashable_char_t + character(len=:), private, allocatable :: name + contains + procedure :: key => ccpp_hashable_char_get_key + end type ccpp_hashable_char_t + + type, public, extends(ccpp_hashable_t) :: ccpp_hashable_int_t + integer, private :: value + contains + procedure :: key => ccpp_hashable_int_get_key + procedure :: val => ccpp_hashable_int_get_val + end type ccpp_hashable_int_t + + ! Abstract interface for key procedure of ccpp_hashable_t class + abstract interface + function ccpp_hashable_get_key(hashable) + import :: ccpp_hashable_t + class(ccpp_hashable_t), intent(in) :: hashable + character(len=:), allocatable :: ccpp_hashable_get_key + end function ccpp_hashable_get_key + end interface + +contains + + !####################################################################### + + subroutine new_hashable_char(name_in, new_obj) + character(len=*), intent(in) :: name_in + type(ccpp_hashable_char_t), pointer :: new_obj + + if (associated(new_obj)) then + deallocate(new_obj) + end if + allocate(new_obj) + new_obj%name = name_in + end subroutine new_hashable_char + + !####################################################################### + + function ccpp_hashable_char_get_key(hashable) + ! Return the hashable char class key (name) + class(ccpp_hashable_char_t), intent(in) :: hashable + character(len=:), allocatable :: ccpp_hashable_char_get_key + + ccpp_hashable_char_get_key = hashable%name + end function ccpp_hashable_char_get_key + + !####################################################################### + + subroutine new_hashable_int(val_in, new_obj) + integer, intent(in) :: val_in + type(ccpp_hashable_int_t), pointer :: new_obj + + if (associated(new_obj)) then + deallocate(new_obj) + end if + allocate(new_obj) + new_obj%value = val_in + end subroutine new_hashable_int + + !####################################################################### + + function ccpp_hashable_int_get_key(hashable) + ! Return the hashable int class key (value ==> string) + class(ccpp_hashable_int_t), intent(in) :: hashable + character(len=:), allocatable :: ccpp_hashable_int_get_key + + character(len=32) :: key_str + + write(key_str, '(i0)') hashable%val() + ccpp_hashable_int_get_key = trim(key_str) + end function ccpp_hashable_int_get_key + + !####################################################################### + + integer function ccpp_hashable_int_get_val(hashable) + ! Return the hashable int class value + class(ccpp_hashable_int_t), intent(in) :: hashable + + ccpp_hashable_int_get_val = hashable%value + end function ccpp_hashable_int_get_val + +end module ccpp_hashable diff --git a/capgen-ng/src/ccpp_scheme_utils.F90 b/capgen-ng/src/ccpp_scheme_utils.F90 new file mode 100644 index 00000000..d4de6499 --- /dev/null +++ b/capgen-ng/src/ccpp_scheme_utils.F90 @@ -0,0 +1,122 @@ +module ccpp_scheme_utils + + ! Module of utilities available to CCPP schemes + + use ccpp_constituent_prop_mod, only: ccpp_model_constituents_t, & + int_unassigned + + implicit none + private + + !! Public interfaces + public :: ccpp_initialize_constituent_ptr ! Used by framework to initialize + public :: ccpp_constituent_index ! Lookup index constituent by name + public :: ccpp_constituent_indices ! Lookup indices of consitutents by name + + !! Private module variables & interfaces + + ! initialized set to .true. once hash table pointer is initialized + logical :: initialized = .false. + type(ccpp_model_constituents_t), pointer :: constituent_obj => null() + + private :: check_initialization + private :: status_ok + +contains + + subroutine check_initialization(caller, errcode, errmsg) + ! Dummy arguments + character(len=*), intent(in) :: caller + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + if (initialized) then + if (present(errcode)) then + errcode = 0 + end if + if (present(errmsg)) then + errmsg = '' + end if + else + if (present(errcode)) then + errcode = 1 + end if + if (present(errmsg)) then + errmsg = trim(caller) // ' FAILED, module not initialized' + end if + end if + end subroutine check_initialization + + logical function status_ok(errcode) + ! Dummy argument + integer, optional, intent(in) :: errcode + + if (present(errcode)) then + status_ok = (errcode == 0) .and. initialized + else + status_ok = initialized + end if + + end function status_ok + + subroutine ccpp_initialize_constituent_ptr(const_obj) + ! Dummy arguments + type(ccpp_model_constituents_t), pointer, intent(in) :: const_obj + + if (.not. initialized) then + constituent_obj => const_obj + initialized = .true. + end if + end subroutine ccpp_initialize_constituent_ptr + + subroutine ccpp_constituent_index(standard_name, const_index, errcode, errmsg) + ! Dummy arguments + character(len=*), intent(in) :: standard_name + integer, intent(out) :: const_index + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + ! Local variable + character(len=*), parameter :: subname = 'ccpp_constituent_index' + + call check_initialization(caller=subname, errcode=errcode, errmsg=errmsg) + if (status_ok(errcode)) then + call constituent_obj%const_index(const_index, standard_name, & + errcode, errmsg) + else + const_index = int_unassigned + end if + end subroutine ccpp_constituent_index + + subroutine ccpp_constituent_indices(standard_names, const_inds, errcode, errmsg) + ! Dummy arguments + character(len=*), intent(in) :: standard_names(:) + integer, intent(out) :: const_inds(:) + integer, optional, intent(out) :: errcode + character(len=*), optional, intent(out) :: errmsg + + ! Local variables + integer :: indx + character(len=*), parameter :: subname = 'ccpp_constituent_indices' + + const_inds = int_unassigned + call check_initialization(caller=subname, errcode=errcode, errmsg=errmsg) + if (status_ok(errcode)) then + if (size(const_inds) < size(standard_names)) then + errcode = 1 + write(errmsg, '(3a)') subname, ": const_inds array too small. ", & + "Must be greater than or equal to the size of standard_names" + else + do indx = 1, size(standard_names) + ! For each std name in , find the const. index + call constituent_obj%const_index(const_inds(indx), & + standard_names(indx), errcode, errmsg) + if (errcode /= 0) then + exit + end if + end do + end if + end if + end subroutine ccpp_constituent_indices + +end module ccpp_scheme_utils diff --git a/end-to-end-tests/CMakeLists.txt b/end-to-end-tests/CMakeLists.txt new file mode 100644 index 00000000..8e5945fc --- /dev/null +++ b/end-to-end-tests/CMakeLists.txt @@ -0,0 +1,82 @@ +cmake_minimum_required(VERSION 3.18) + +project(ccpp_framework_end_to_end_tests + VERSION 8.0.0 + LANGUAGES Fortran) + +enable_testing() +include(cmake/ccpp_capgen.cmake) + +option(OPENMP "Enable OpenMP support for the framework" ON) +message(STATUS "OpenMP ${OPENMP}") + +set(CCPP_VERBOSITY "2" CACHE STRING "Verbosity level of output (default: 0)") + +# Set appropriate flags to help with debugging test issues +if(${CMAKE_Fortran_COMPILER_ID} STREQUAL "GNU") + ADD_COMPILE_OPTIONS(-fcheck=all) + ADD_COMPILE_OPTIONS(-fbacktrace) + ADD_COMPILE_OPTIONS(-ffpe-trap=zero) + ADD_COMPILE_OPTIONS(-finit-real=nan) + ADD_COMPILE_OPTIONS(-ggdb) + ADD_COMPILE_OPTIONS(-ffree-line-length-none) + ADD_COMPILE_OPTIONS(-cpp) +elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL "Intel") + ADD_COMPILE_OPTIONS(-fpe0) + ADD_COMPILE_OPTIONS(-warn) + ADD_COMPILE_OPTIONS(-traceback) + ADD_COMPILE_OPTIONS(-debug extended) + ADD_COMPILE_OPTIONS(-fpp) + ADD_COMPILE_OPTIONS(-diag-disable=10448) +elseif(${CMAKE_Fortran_COMPILER_ID} STREQUAL "IntelLLVM") + ADD_COMPILE_OPTIONS(-fpe0) + ADD_COMPILE_OPTIONS(-warn) + ADD_COMPILE_OPTIONS(-traceback) + ADD_COMPILE_OPTIONS(-debug full) + ADD_COMPILE_OPTIONS(-fpp) +elseif (${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVIDIA" OR ${CMAKE_Fortran_COMPILER_ID} STREQUAL "NVHPC") + ADD_COMPILE_OPTIONS(-Mnoipa) + ADD_COMPILE_OPTIONS(-traceback) + ADD_COMPILE_OPTIONS(-Mfree) + ADD_COMPILE_OPTIONS(-Ktrap=fp) + ADD_COMPILE_OPTIONS(-Mpreprocess) +else() + message (WARNING "This program may not be able to be compiled with compiler :${CMAKE_Fortran_COMPILER_ID}") +endif() + +#------------------------------------------------------------------------------ +# Set MPI flags for Fortran with MPI F08 interface +find_package(MPI COMPONENTS Fortran REQUIRED) +if(NOT MPI_Fortran_HAVE_F08_MODULE) + message(FATAL_ERROR "MPI implementation does not support the Fortran 2008 mpi_f08 interface") +endif() + +#------------------------------------------------------------------------------ +# Set OpenMP flags for C/C++/Fortran +if(OPENMP) + find_package(OpenMP REQUIRED) +endif() + +#------------------------------------------------------------------------------ +# Set a default build type if none was specified +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + message(STATUS "Setting build type to 'Debug' as none was specified.") + set(CMAKE_BUILD_TYPE Debug CACHE STRING "Choose the type of build." FORCE) +endif() + +#------------------------------------------------------------------------------ +# Run simple tests first +add_subdirectory(chunked_data) +add_subdirectory(opt_arg) + +#------------------------------------------------------------------------------ +# Run intermediate tests next +add_subdirectory(nested_suite) +add_subdirectory(ddthost) +add_subdirectory(instances) + +#------------------------------------------------------------------------------ +# Run most complex tests last +add_subdirectory(capgen_ng) +add_subdirectory(var_compat) +add_subdirectory(advection) diff --git a/end-to-end-tests/README.md.tobeupdated b/end-to-end-tests/README.md.tobeupdated new file mode 100644 index 00000000..a9f05bd7 --- /dev/null +++ b/end-to-end-tests/README.md.tobeupdated @@ -0,0 +1,16 @@ +# How to build the chunked data test + +1. Set compiler environment as appropriate for your system +2. Run the following commands: +``` +cd test_prebuild/test_chunked_data/ +#rm -fr build +mkdir build +../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build +cd build +cmake .. 2>&1 | tee log.cmake +make 2>&1 | tee log.make +./test_chunked_data.x +# On systems where linking against the MPI library requires a parallel launcher, +# use 'mpirun -np 1 ./test_chunked_data.x' or 'srun -n 1 ./test_chunked_data.x' etc. +``` diff --git a/end-to-end-tests/advection/CMakeLists.txt b/end-to-end-tests/advection/CMakeLists.txt new file mode 100644 index 00000000..11661ead --- /dev/null +++ b/end-to-end-tests/advection/CMakeLists.txt @@ -0,0 +1,71 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "cld_liq" "cld_ice" "apply_constituent_tendencies" "const_indices") +set(HOST_FILES "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "cld_suite.xml") +set(HOST "test_host") + +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES}) + +# Run ccpp_capgen +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +# Add extra files needed for testing +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_advection.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_advection_host_integration.F90 +) +target_link_libraries(test_advection.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_advection.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_advection.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_advection + COMMAND test_advection.x) diff --git a/end-to-end-tests/advection/README.md b/end-to-end-tests/advection/README.md new file mode 100644 index 00000000..c460e13a --- /dev/null +++ b/end-to-end-tests/advection/README.md @@ -0,0 +1,10 @@ +# Advection Test + +Contains tests to exercise the capabilities of the constituents object, including: +- Adding run-time constituents from the host via a register phase +- Adding run-time constituents from schemes via a register phase + - Also tests that trying to add a constituent outside of the register phase errors as expected +- Passing around and modifying the constituent array +- Accessing and modifying a constituent tendency variable +- Passing around the constituent tendency array +- Dimensions are case-insensitive diff --git a/end-to-end-tests/advection/advection_test_reports.py b/end-to-end-tests/advection/advection_test_reports.py new file mode 100644 index 00000000..4fbe8e68 --- /dev/null +++ b/end-to-end-tests/advection/advection_test_reports.py @@ -0,0 +1,127 @@ +#! /usr/bin/env python3 +""" +----------------------------------------------------------------------- + Description: Test advection database report python interface + + Assumptions: + + Command line arguments: build_dir database_filepath + + Usage: python test_reports +----------------------------------------------------------------------- +""" +import os +import unittest + +from test_stub import BaseTests + +_BUILD_DIR = os.path.join(os.path.abspath(os.environ['BUILD_DIR']), "test", "advection_test") +_DATABASE = os.path.abspath(os.path.join(_BUILD_DIR, "ccpp", "datatable.xml")) + +_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +_FRAMEWORK_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, os.pardir)) +_SCRIPTS_DIR = os.path.abspath(os.path.join(_FRAMEWORK_DIR, "scripts")) + +# Check data +_HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] +_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_cld_suite_cap.F90")] +_UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), + os.path.join(_FRAMEWORK_DIR, "src", + "ccpp_constituent_prop_mod.F90"), + os.path.join(_FRAMEWORK_DIR, "src", + "ccpp_scheme_utils.F90"), + os.path.join(_FRAMEWORK_DIR, "src", "ccpp_hashable.F90"), + os.path.join(_FRAMEWORK_DIR, "src", "ccpp_hash_table.F90")] +_CCPP_FILES = _UTILITY_FILES + _HOST_FILES + _SUITE_FILES +_DEPENDENCIES = [""] +_PROCESS_LIST = [""] +_MODULE_LIST = ["cld_ice", "cld_liq", "const_indices", "apply_constituent_tendencies"] +_SUITE_LIST = ["cld_suite"] +_REQUIRED_VARS_CLD = ["ccpp_error_code", "ccpp_error_message", + "horizontal_loop_begin", "horizontal_loop_end", + "surface_air_pressure", "temperature", + "tendency_of_cloud_liquid_dry_mixing_ratio", + "time_step_for_physics", "water_temperature_at_freezing", + "water_vapor_specific_humidity", + "cloud_ice_dry_mixing_ratio", + "cloud_liquid_dry_mixing_ratio", + "ccpp_constituents", + "ccpp_constituent_tendencies", + "number_of_ccpp_constituents", + "dynamic_constituents_for_cld_ice", + "dynamic_constituents_for_cld_liq", + "test_banana_constituent_indices", "test_banana_name", + "banana_array_dim", + "test_banana_name_array", + "test_banana_constituent_index", + # Added by --debug option + "horizontal_dimension", + "vertical_layer_dimension"] +_INPUT_VARS_CLD = ["surface_air_pressure", "temperature", + "horizontal_loop_begin", "horizontal_loop_end", + "time_step_for_physics", "water_temperature_at_freezing", + "water_vapor_specific_humidity", + "cloud_ice_dry_mixing_ratio", + "cloud_liquid_dry_mixing_ratio", + "tendency_of_cloud_liquid_dry_mixing_ratio", + "ccpp_constituents", + "ccpp_constituent_tendencies", + "number_of_ccpp_constituents", + "banana_array_dim", + "test_banana_name_array", "test_banana_name", + # Added by --debug option + "horizontal_dimension", + "vertical_layer_dimension"] +_OUTPUT_VARS_CLD = ["ccpp_error_code", "ccpp_error_message", + "water_vapor_specific_humidity", "temperature", + "tendency_of_cloud_liquid_dry_mixing_ratio", + "cloud_ice_dry_mixing_ratio", + "ccpp_constituents", + "ccpp_constituent_tendencies", + "cloud_liquid_dry_mixing_ratio", + "dynamic_constituents_for_cld_ice", + "dynamic_constituents_for_cld_liq", + "dynamic_constituents_for_cld_liq", + "test_banana_constituent_indices", + "test_banana_constituent_index"] + + +class TestAdvectionHostDataTables(unittest.TestCase, BaseTests.TestHostDataTables): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + +class CommandLineAdvectionHostDatafileRequiredFiles(unittest.TestCase, BaseTests.TestHostCommandLineDataFiles): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" + + +class TestCapgenCldSuite(unittest.TestCase, BaseTests.TestSuite): + database = _DATABASE + required_vars = _REQUIRED_VARS_CLD + input_vars = _INPUT_VARS_CLD + output_vars = _OUTPUT_VARS_CLD + suite_name = "cld_suite" + + +class CommandLineCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuiteCommandLine): + database = _DATABASE + required_vars = _REQUIRED_VARS_CLD + input_vars = _INPUT_VARS_CLD + output_vars = _OUTPUT_VARS_CLD + suite_name = "cld_suite" + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" diff --git a/end-to-end-tests/advection/apply_constituent_tendencies.F90 b/end-to-end-tests/advection/apply_constituent_tendencies.F90 new file mode 100644 index 00000000..63a1881c --- /dev/null +++ b/end-to-end-tests/advection/apply_constituent_tendencies.F90 @@ -0,0 +1,39 @@ +module apply_constituent_tendencies + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: apply_constituent_tendencies_run + +contains + + !> \section arg_table_apply_constituent_tendencies_run Argument Table + !!! \htmlinclude apply_constituent_tendencies_run.html + subroutine apply_constituent_tendencies_run(const_tend, const, errcode, errmsg) + ! Dummy arguments + real(kind=kind_phys), intent(inout) :: const_tend(:, :, :) ! constituent tendency array + real(kind=kind_phys), intent(inout) :: const(:, :, :) ! constituent state array + integer, intent(out) :: errcode + character(len=512), intent(out) :: errmsg + + ! Local variables + integer :: klev, jcnst, icol + + errcode = 0 + errmsg = '' + + do icol = 1, size(const_tend, 1) + do klev = 1, size(const_tend, 2) + do jcnst = 1, size(const_tend, 3) + const(icol, klev, jcnst) = const(icol, klev, jcnst) + const_tend(icol, klev, jcnst) + end do + end do + end do + + const_tend = 0._kind_phys + + end subroutine apply_constituent_tendencies_run + +end module apply_constituent_tendencies diff --git a/end-to-end-tests/advection/apply_constituent_tendencies.meta b/end-to-end-tests/advection/apply_constituent_tendencies.meta new file mode 100644 index 00000000..ac02e5e4 --- /dev/null +++ b/end-to-end-tests/advection/apply_constituent_tendencies.meta @@ -0,0 +1,36 @@ +##################################################################### +[ccpp-table-properties] + name = apply_constituent_tendencies + type = scheme +[ccpp-arg-table] + name = apply_constituent_tendencies_run + type = scheme +[ const_tend ] + standard_name = ccpp_constituent_tendencies + long_name = ccpp constituent tendencies + units = none + type = real | kind = kind_phys + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + intent = inout +[ const ] + standard_name = ccpp_constituents + long_name = ccpp constituents + units = none + type = real | kind = kind_phys + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + intent = inout +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + type = integer + dimensions = () + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + type = character | kind = len=512 + dimensions = () + intent = out +######################################################### diff --git a/end-to-end-tests/advection/cld_ice.F90 b/end-to-end-tests/advection/cld_ice.F90 new file mode 100644 index 00000000..bf19b979 --- /dev/null +++ b/end-to-end-tests/advection/cld_ice.F90 @@ -0,0 +1,125 @@ +! Test parameterization with advected species +! + +module cld_ice + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: cld_ice_register + public :: cld_ice_init + public :: cld_ice_run + public :: cld_ice_final + + real(kind=kind_phys), private :: tcld = huge(1.0_kind_phys) + +contains + + !> \section arg_table_cld_ice_register Argument Table + !! \htmlinclude arg_table_cld_ice_register.html + !! + subroutine cld_ice_register(dyn_const_ice, errmsg, errcode) + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const_ice(:) + integer, intent(out) :: errcode + character(len=512), intent(out) :: errmsg + + errmsg = '' + errcode = 0 + allocate(dyn_const_ice(2), stat=errcode) + if (errcode /= 0) then + errmsg = 'Error allocating dyn_const in cld_ice_dynamic_constituents' + return + end if + call dyn_const_ice(1)%instantiate(std_name='dyn_const1', long_name='dyn const1', & + diag_name='DYNCONST1', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + min_value=1000._kind_phys, water_species=.true., mixing_ratio_type='wet', & + errcode=errcode, errmsg=errmsg) + call dyn_const_ice(2)%instantiate(std_name='dyn_const2_wrt_moist_air', long_name='dyn const2', & + diag_name='DYNCONST2', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + water_species=.false., errcode=errcode, errmsg=errmsg) + + end subroutine cld_ice_register + + !> \section arg_table_cld_ice_run Argument Table + !! \htmlinclude arg_table_cld_ice_run.html + !! + subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & + errmsg, errflg) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(inout) :: temp(:, :) + real(kind=kind_phys), intent(inout) :: qv(:, :) + real(kind=kind_phys), intent(in) :: ps(:) + real(kind=kind_phys), intent(inout) :: cld_ice_array(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: icol + integer :: ilev + real(kind=kind_phys) :: frz + + errmsg = '' + errflg = 0 + + ! Apply state-of-the-art thermodynamics :) + do icol = 1, ncol + do ilev = 1, size(temp, 2) + if (temp(icol, ilev) < tcld) then + frz = max(qv(icol, ilev) - 0.5_kind_phys, 0.0_kind_phys) + cld_ice_array(icol, ilev) = cld_ice_array(icol, ilev) + frz + qv(icol, ilev) = qv(icol, ilev) - frz + if (frz > 0.0_kind_phys) then + temp(icol, ilev) = temp(icol, ilev) + 1.0_kind_phys + end if + end if + end do + end do + + end subroutine cld_ice_run + + !> \section arg_table_cld_ice_init Argument Table + !! \htmlinclude arg_table_cld_ice_init.html + !! + subroutine cld_ice_init(tfreeze, errmsg, errflg) + + real(kind=kind_phys), intent(in) :: tfreeze + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + tcld = tfreeze - 20.0_kind_phys + + end subroutine cld_ice_init + + !> \section arg_table_cld_ice_final Argument Table + !! \htmlinclude arg_table_cld_ice_final.html + !! + + !> @{ + !! This routine does nothing, but it tests if blank + !! lines and doxygen comments between metadata hooks + !! and the subroutine are parsed correctly. + !! @{ + + subroutine cld_ice_final(errmsg, errflg) + + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + end subroutine cld_ice_final + + !! @} + !! @} + +end module cld_ice diff --git a/end-to-end-tests/advection/cld_ice.meta b/end-to-end-tests/advection/cld_ice.meta new file mode 100644 index 00000000..bd3cf24a --- /dev/null +++ b/end-to-end-tests/advection/cld_ice.meta @@ -0,0 +1,136 @@ +# cld_ice is a scheme that produces a cloud ice amount +[ccpp-table-properties] + name = cld_ice + type = scheme + +[ccpp-arg-table] + name = cld_ice_register + type = scheme +[ dyn_const_ice ] + standard_name = dynamic_constituents_for_cld_ice + units = none + dimensions = (:) + allocatable = True + type = ccpp_constituent_properties_t + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_ice_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp ] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ qv ] + standard_name = water_vapor_specific_humidity + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) + intent = in +[ cld_ice_array ] + standard_name = cloud_ice_dry_mixing_ratio + advected = .true. + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_ice_init + type = scheme +[ tfreeze ] + standard_name = water_temperature_at_freezing + long_name = Freezing temperature of water at sea level + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_ice_final + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/advection/cld_liq.F90 b/end-to-end-tests/advection/cld_liq.F90 new file mode 100644 index 00000000..d019f152 --- /dev/null +++ b/end-to-end-tests/advection/cld_liq.F90 @@ -0,0 +1,107 @@ +! Test parameterization with advected species +! + +module cld_liq + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + private + + public :: cld_liq_register + public :: cld_liq_init + public :: cld_liq_run + +contains + + !> \section arg_table_cld_liq_register Argument Table + !! \htmlinclude arg_table_cld_liq_register.html + !! + subroutine cld_liq_register(dyn_const, errmsg, errflg) + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + allocate(dyn_const(2), stat=errflg) + if (errflg /= 0) then + errmsg = 'Error allocating dyn_const in cld_liq_register' + return + end if + call dyn_const(1)%instantiate(std_name="dyn_const3_wrt_moist_air_and_condensed_water", long_name='dyn const3', & + diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + water_species=.true., mixing_ratio_type='dry', & + errcode=errflg, errmsg=errmsg) + call dyn_const(2)%instantiate(std_name="cloud_liquid_dry_mixing_ratio", long_name='Cloud liquid dry mixing ratio', & + diag_name='CLDLIQ', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + ! Defer setting water_species later in the test + !water_species=.true., + mixing_ratio_type='dry', & + errcode=errflg, errmsg=errmsg) + + end subroutine cld_liq_register + + !> \section arg_table_cld_liq_run Argument Table + !! \htmlinclude arg_table_cld_liq_run.html + !! + subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & + cld_liq_array, cld_liq_tend, errmsg, errflg) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(in) :: tcld + real(kind=kind_phys), intent(inout) :: temp(:, :) + real(kind=kind_phys), intent(inout) :: qv(:, :) + real(kind=kind_phys), intent(in) :: ps(:) + real(kind=kind_phys), intent(inout) :: cld_liq_array(:, :) + real(kind=kind_phys), intent(out) :: cld_liq_tend(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: icol + integer :: ilev + real(kind=kind_phys) :: cond + + errmsg = '' + errflg = 0 + + ! Apply state-of-the-art thermodynamics :) + do icol = 1, ncol + do ilev = 1, size(temp, 2) + cld_liq_array(icol, ilev) = max(0.0_kind_phys, cld_liq_array(icol, ilev)) + if ((qv(icol, ilev) > 0.0_kind_phys) .and. & + (temp(icol, ilev) <= tcld)) then + cond = min(qv(icol, ilev), 0.1_kind_phys) + cld_liq_tend(icol, ilev) = cond + qv(icol, ilev) = qv(icol, ilev) - cond + if (cond > 0.0_kind_phys) then + temp(icol, ilev) = temp(icol, ilev) + (cond * 5.0_kind_phys) + end if + end if + end do + end do + + end subroutine cld_liq_run + + !> \section arg_table_cld_liq_init Argument Table + !! \htmlinclude arg_table_cld_liq_init.html + !! + subroutine cld_liq_init(tfreeze, tcld, errmsg, errflg) + + real(kind=kind_phys), intent(in) :: tfreeze + real(kind=kind_phys), intent(out) :: tcld + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + tcld = tfreeze - 20.0_kind_phys + + end subroutine cld_liq_init + +end module cld_liq diff --git a/end-to-end-tests/advection/cld_liq.meta b/end-to-end-tests/advection/cld_liq.meta new file mode 100644 index 00000000..db43c5ef --- /dev/null +++ b/end-to-end-tests/advection/cld_liq.meta @@ -0,0 +1,135 @@ +# cld_liq is a scheme that produces a cloud liquid amount +[ccpp-table-properties] + name = cld_liq + type = scheme +[ccpp-arg-table] + name = cld_liq_register + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_cld_liq + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out + allocatable = true +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_liq_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ tcld] + standard_name = minimum_temperature_for_cloud_liquid + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ temp ] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_LAYER_dimension) + type = real + kind = kind_phys + intent = inout +[ qv ] + standard_name = water_vapor_specific_humidity + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = hPa + dimensions = (horizontal_dimension) + intent = in +[ cld_liq_array ] + standard_name = cloud_liquid_dry_mixing_ratio + diagnostic_name = CLDLIQ + advected = .true. + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = inout +[ cld_liq_tend ] + standard_name = tendency_of_cloud_liquid_dry_mixing_ratio + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_liq_init + type = scheme +[ tfreeze] + standard_name = water_temperature_at_freezing + long_name = Freezing temperature of water at sea level + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ tcld] + standard_name = minimum_temperature_for_cloud_liquid + units = K + dimensions = () + type = real | kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/advection/cld_suite.xml b/end-to-end-tests/advection/cld_suite.xml new file mode 100644 index 00000000..fac613e8 --- /dev/null +++ b/end-to-end-tests/advection/cld_suite.xml @@ -0,0 +1,11 @@ + + + + + const_indices + cld_liq + apply_constituent_tendencies + cld_ice + apply_constituent_tendencies + + diff --git a/end-to-end-tests/advection/cld_suite_error.xml b/end-to-end-tests/advection/cld_suite_error.xml new file mode 100644 index 00000000..80acac91 --- /dev/null +++ b/end-to-end-tests/advection/cld_suite_error.xml @@ -0,0 +1,9 @@ + + + + + dlc_liq + cld_liq + cld_ice + + diff --git a/end-to-end-tests/advection/const_indices.F90 b/end-to-end-tests/advection/const_indices.F90 new file mode 100644 index 00000000..bc3b46a7 --- /dev/null +++ b/end-to-end-tests/advection/const_indices.F90 @@ -0,0 +1,95 @@ +! Test collection of constituent indices +! + +module const_indices + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: const_indices_init + public :: const_indices_run + +contains + + !> \section arg_table_const_indices_run Argument Table + !! \htmlinclude arg_table_const_indices_run.html + !! + subroutine const_indices_run(const_std_name, num_consts, test_stdname_array, & + const_index, const_inds, errmsg, errflg) + use ccpp_constituent_prop_mod, only: int_unassigned + use ccpp_scheme_utils, only: ccpp_constituent_index + use ccpp_scheme_utils, only: ccpp_constituent_indices + + character(len=*), intent(in) :: const_std_name + integer, intent(in) :: num_consts + character(len=*), intent(in) :: test_stdname_array(:) + integer, intent(out) :: const_index + integer, intent(out) :: const_inds(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: indx + integer :: test_indx + + errmsg = '' + errflg = 0 + + ! Find the constituent index for + call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) + if (errflg == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) + end if + ! Check that a non-registered constituent is detectable but + ! does not cause an error + if (errflg == 0) then + call ccpp_constituent_index('unobtainium', test_indx, errflg, errmsg) + if (test_indx /= int_unassigned) then + if (errflg == 0) then + ! Do not add an error if one is already reported + errflg = 2 + write(errmsg, '(2a,i0,a,i0)') "ccpp_constituent_index called for ", & + "'unobtainium' returned an index of ", test_indx, ", not ", & + int_unassigned + end if + end if + end if + + end subroutine const_indices_run + + !> \section arg_table_const_indices_init Argument Table + !! \htmlinclude arg_table_const_indices_init.html + !! + subroutine const_indices_init(const_std_name, num_consts, test_stdname_array, & + const_index, const_inds, errmsg, errflg) + use ccpp_scheme_utils, only: ccpp_constituent_index, & + ccpp_constituent_indices + + character(len=*), intent(in) :: const_std_name + integer, intent(in) :: num_consts + character(len=*), intent(in) :: test_stdname_array(:) + integer, intent(out) :: const_index + integer, intent(out) :: const_inds(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: indx + + errmsg = '' + errflg = 0 + + ! Find the constituent index for + call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) + if (errflg == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) + end if + + end subroutine const_indices_init + + !! @} + !! @} + +end module const_indices diff --git a/end-to-end-tests/advection/const_indices.meta b/end-to-end-tests/advection/const_indices.meta new file mode 100644 index 00000000..a4cc98e2 --- /dev/null +++ b/end-to-end-tests/advection/const_indices.meta @@ -0,0 +1,108 @@ +# const_indices just returns some constituent indices as a test +[ccpp-table-properties] + name = const_indices + type = scheme +[ccpp-arg-table] + name = const_indices_run + type = scheme +[ const_std_name ] + standard_name = test_banana_name + type = character | kind = len=* + units = 1 + dimensions = () + protected = true + intent = in +[ num_consts ] + standard_name = banana_array_dim + long_name = Size of test_banana_name_array + units = 1 + dimensions = () + type = integer + intent = in +[ test_stdname_array ] + standard_name = test_banana_name_array + type = character | kind = len=* + units = count + dimensions = (banana_array_dim) + intent = in +[ const_index ] + standard_name = test_banana_constituent_index + long_name = Constituent index + units = 1 + dimensions = () + type = integer + intent = out +[ const_inds ] + standard_name = test_banana_constituent_indices + long_name = Array of constituent indices + units = 1 + dimensions = (banana_array_dim) + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = const_indices_init + type = scheme +[ const_std_name ] + standard_name = test_banana_name + type = character | kind = len=* + units = 1 + dimensions = () + protected = true + intent = in +[ num_consts ] + standard_name = banana_array_dim + long_name = Size of test_banana_name_array + units = 1 + dimensions = () + type = integer + intent = in +[ test_stdname_array ] + standard_name = test_banana_name_array + type = character | kind = len=* + units = count + dimensions = (banana_array_dim) + intent = in +[ const_index ] + standard_name = test_banana_constituent_index + long_name = Constituent index + units = 1 + dimensions = () + type = integer + intent = out +[ const_inds ] + standard_name = test_banana_constituent_indices + long_name = Array of constituent indices + units = 1 + dimensions = (banana_array_dim) + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/advection/dlc_liq.F90 b/end-to-end-tests/advection/dlc_liq.F90 new file mode 100644 index 00000000..20ff4b7b --- /dev/null +++ b/end-to-end-tests/advection/dlc_liq.F90 @@ -0,0 +1,41 @@ +! Test parameterization with a runtime constituents +! properties object outside of the register phase + +module dlc_liq + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + private + + public :: dlc_liq_init + +contains + + !> \section arg_table_dlc_liq_init Argument Table + !! \htmlinclude arg_table_dlc_liq_init.html + !! + subroutine dlc_liq_init(dyn_const, errmsg, errflg) + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + character(len=256) :: stdname + + errmsg = '' + errflg = 0 + allocate(dyn_const(1), stat=errflg) + if (errflg /= 0) then + errmsg = 'Error allocating dyn_const in dlc_liq_init' + return + end if + call dyn_const(1)%instantiate(std_name="dyn_const3", long_name='dyn const3', & + diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + errcode=errflg, errmsg=errmsg) + call dyn_const(1)%standard_name(stdname, errcode=errflg, errmsg=errmsg) + + end subroutine dlc_liq_init + +end module dlc_liq diff --git a/end-to-end-tests/advection/dlc_liq.meta b/end-to-end-tests/advection/dlc_liq.meta new file mode 100644 index 00000000..fedb6243 --- /dev/null +++ b/end-to-end-tests/advection/dlc_liq.meta @@ -0,0 +1,29 @@ +# dlc_liq is a scheme that has a ccpp_constituent_properties_t variable +# outside of the register phase +[ccpp-table-properties] + name = dlc_liq + type = scheme +[ccpp-arg-table] + name = dlc_liq_init + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_dlc_liq + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out + allocatable = true +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/advection/test_advection_host_integration.F90 b/end-to-end-tests/advection/test_advection_host_integration.F90 new file mode 100644 index 00000000..f1f73576 --- /dev/null +++ b/end-to-end-tests/advection/test_advection_host_integration.F90 @@ -0,0 +1,79 @@ +program test + use test_prog, only: test_host, & + suite_info, & + cm, & + cs + + implicit none + + character(len=cs), target :: test_parts1(1) + character(len=cm), target :: test_invars1(11) + character(len=cm), target :: test_outvars1(13) + character(len=cm), target :: test_reqvars1(18) + + type(suite_info) :: test_suites(1) + logical :: run_okay + + test_parts1 = (/ 'physics '/) + test_invars1 = (/ & + 'banana_array_dim ', & + 'cloud_ice_dry_mixing_ratio ', & + 'cloud_liquid_dry_mixing_ratio ', & + 'surface_air_pressure ', & + 'temperature ', & + 'time_step_for_physics ', & + 'water_temperature_at_freezing ', & + 'ccpp_constituent_tendencies ', & + 'ccpp_constituents ', & + 'number_of_ccpp_constituents ', & + 'water_vapor_specific_humidity ' /) + test_outvars1 = (/ & + 'ccpp_error_message ', & + 'ccpp_error_code ', & + 'temperature ', & + 'water_vapor_specific_humidity ', & + 'cloud_liquid_dry_mixing_ratio ', & + 'ccpp_constituent_tendencies ', & + 'ccpp_constituents ', & + 'dynamic_constituents_for_cld_liq ', & + 'dynamic_constituents_for_cld_ice ', & + 'tendency_of_cloud_liquid_dry_mixing_ratio', & + 'test_banana_constituent_index ', & + 'test_banana_constituent_indices ', & + 'cloud_ice_dry_mixing_ratio ' /) + test_reqvars1 = (/ & + 'banana_array_dim ', & + 'surface_air_pressure ', & + 'temperature ', & + 'time_step_for_physics ', & + 'cloud_liquid_dry_mixing_ratio ', & + 'tendency_of_cloud_liquid_dry_mixing_ratio', & + 'cloud_ice_dry_mixing_ratio ', & + 'dynamic_constituents_for_cld_liq ', & + 'dynamic_constituents_for_cld_ice ', & + 'water_temperature_at_freezing ', & + 'ccpp_constituent_tendencies ', & + 'ccpp_constituents ', & + 'number_of_ccpp_constituents ', & + 'test_banana_constituent_index ', & + 'test_banana_constituent_indices ', & + 'water_vapor_specific_humidity ', & + 'ccpp_error_message ', & + 'ccpp_error_code ' /) + + ! Setup expected test suite info + test_suites(1)%suite_name = 'cld_suite' + test_suites(1)%suite_parts => test_parts1 + test_suites(1)%suite_input_vars => test_invars1 + test_suites(1)%suite_output_vars => test_outvars1 + test_suites(1)%suite_required_vars => test_reqvars1 + + call test_host(run_okay, test_suites) + + if (run_okay) then + stop 0 + else + stop -1 + end if + +end program test diff --git a/end-to-end-tests/advection/test_host.F90 b/end-to-end-tests/advection/test_host.F90 new file mode 100644 index 00000000..f02456e3 --- /dev/null +++ b/end-to-end-tests/advection/test_host.F90 @@ -0,0 +1,1172 @@ +module test_prog + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + private + + public test_host + + ! Public data and interfaces + integer, public, parameter :: cs = 16 + integer, public, parameter :: cm = 41 + + !> \section arg_table_suite_info Argument Table + !! \htmlinclude arg_table_suite_info.html + !! + type, public :: suite_info + character(len=cs) :: suite_name = '' + character(len=cs), pointer :: suite_parts(:) => null() + character(len=cm), pointer :: suite_input_vars(:) => null() + character(len=cm), pointer :: suite_output_vars(:) => null() + character(len=cm), pointer :: suite_required_vars(:) => null() + end type suite_info + + type(ccpp_constituent_properties_t), private, target, allocatable :: host_constituents(:) + + private :: check_suite + private :: advect_constituents ! Move data around + private :: check_errflg + +contains + + subroutine check_errflg(subname, errflg, errmsg, errflg_final) + ! If errflg is not zero, print an error message + character(len=*), intent(in) :: subname + integer, intent(in) :: errflg + character(len=*), intent(in) :: errmsg + + integer, intent(out) :: errflg_final + + if (errflg /= 0) then + write(6, '(a,i0,4a)') "Error ", errflg, " from ", trim(subname), & + ':', trim(errmsg) + !Notify test script that a failure occurred: + errflg_final = -1 !Notify test script that a failure occured + end if + + end subroutine check_errflg + + logical function check_suite(test_suite) + use ccpp_static_api, only: ccpp_physics_suite_part_list + use ccpp_static_api, only: ccpp_physics_suite_variables + use test_utils, only: check_list + + ! Dummy argument + type(suite_info), intent(in) :: test_suite + ! Local variables + logical :: check + integer :: errflg + character(len=512) :: errmsg + character(len=128), allocatable :: test_list(:) + + check_suite = .true. + ! First, check the suite parts + call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_parts, 'part names', & + suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the input variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.true., output_vars=.false.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_input_vars, & + 'input variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the output variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.false., output_vars=.true.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_output_vars, & + 'output variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check all required variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_required_vars, & + 'required variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + end function check_suite + + subroutine advect_constituents() + use test_host_mod, only: phys_state, & + ncnst + use test_host_mod, only: twist_array + + ! Local variables + integer :: q_ind ! Constituent index + + do q_ind = 1, ncnst ! Skip checks, they were done in constituents_in + call twist_array(phys_state%q(:, :, q_ind)) + end do + end subroutine advect_constituents + + !> \section arg_table_test_host Argument Table + !! \htmlinclude arg_table_test_host.html + !! + subroutine test_host(retval, test_suites) + + use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t + use test_host_mod, only: num_time_steps + use test_host_mod, only: init_data, & + compare_data + use test_host_mod, only: ncols, & + pver + use test_host_data, only: num_consts, & + std_name_array, & + const_std_name + use test_host_data, only: check_constituent_indices + use ccpp_static_api, only: ccpp_deallocate_dynamic_constituents + use ccpp_static_api, only: ccpp_register_constituents + use ccpp_static_api, only: ccpp_is_scheme_constituent + use ccpp_static_api, only: ccpp_initialize_constituents + use ccpp_static_api, only: ccpp_number_constituents + use ccpp_static_api, only: ccpp_constituents_array + use ccpp_static_api, only: ccpp_register + use ccpp_static_api, only: ccpp_init + use ccpp_static_api, only: ccpp_physics_init + use ccpp_static_api, only: ccpp_physics_timestep_init + use ccpp_static_api, only: ccpp_physics_run + use ccpp_static_api, only: ccpp_physics_timestep_final + use ccpp_static_api, only: ccpp_physics_final + use ccpp_static_api, only: ccpp_final + use ccpp_static_api, only: ccpp_physics_suite_list + use ccpp_static_api, only: ccpp_const_get_index + use ccpp_static_api, only: ccpp_model_const_properties + use test_utils, only: check_list + + type(suite_info), intent(in) :: test_suites(:) + logical, intent(out) :: retval + + logical :: check + integer :: col_start, col_end + integer :: index, sind + integer :: index_liq, index_ice + integer :: index_dyn1, index_dyn2, index_dyn3 + integer :: time_step + integer :: num_suites + integer :: num_advected ! Num advected species + logical :: const_log + logical :: is_constituent + logical :: has_default + integer :: test_scalar_const_index + integer :: test_const_indices(num_consts) + character(len=128), allocatable :: suite_names(:) + character(len=256) :: const_str + character(len=512) :: errmsg + character(len=512) :: expected_error + integer :: errflg + integer :: errflg_final ! Used to notify testing script of test failure + real(kind=kind_phys), pointer :: const_ptr(:, :, :) + real(kind=kind_phys) :: default_value + real(kind=kind_phys) :: check_value + type(ccpp_constituent_prop_ptr_t), pointer :: const_props(:) + character(len=*), parameter :: subname = 'test_host' + + ! Initialized "final" error flag used to report a failure to the larged + ! testing script: + errflg_final = 0 + + ! Gather and test the inspection routines + num_suites = size(test_suites) + call ccpp_physics_suite_list(suite_names) + retval = check_list(suite_names, test_suites(:)%suite_name, & + 'suite names') + write(6, *) 'Available suites are:' + do index = 1, size(suite_names) + do sind = 1, num_suites + if (trim(test_suites(sind)%suite_name) == & + trim(suite_names(index))) then + exit + end if + end do + write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & + ' = test_suites(', sind, ')' + end do + if (retval) then + do sind = 1, num_suites + check = check_suite(test_suites(sind)) + retval = retval .and. check + end do + end if + !!! Return here if any check failed + if (.not. retval) then + return + end if + + errflg = 0 + errmsg = '' + + ! Check that is_scheme_constituent works as expected + call ccpp_is_scheme_constituent('specific_humidity', & + is_constituent, errflg, errmsg) + call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & + errmsg, errflg_final) + ! specific_humidity should not be an existing constituent + if (is_constituent) then + write(6, *) "ERROR: specific humidity is already a constituent" + errflg_final = -1 ! Notify test script that a failure occurred + end if + call ccpp_is_scheme_constituent('cloud_ice_dry_mixing_ratio', & + is_constituent, errflg, errmsg) + call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & + errmsg, errflg_final) + ! cloud_ice_dry_mixing_ratio should be an existing constituent + if (.not. is_constituent) then + write(6, *) "ERROR: cloud_ice_dry_mixing ratio not found in ", & + "host cap constituent list" + errflg_final = -1 ! Notify test script that a failure occurred + end if + + ! Use the suite information to call the register phase + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_register(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in register of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + + ! Register the constituents to find out what needs advecting + ! DO A COUPLE OF TESTS FIRST + + ! First confirm the correct error occurs if you try to add an + ! incompatible constituent with the same standard name + expected_error = 'ccp_model_const_add_metadata ERROR: Trying to add ' //& + 'constituent specific_humidity but an incompatible ' // & + 'constituent with this name already exists' + allocate(host_constituents(2)) + call host_constituents(1)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errflg, errmsg=errmsg) + call host_constituents(2)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errflg, errmsg=errmsg) + call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) + if (errflg == 0) then + call ccpp_register_constituents(host_constituents, & + errmsg=errmsg, errflg=errflg) + end if + ! Check the error + if (errflg == 0) then + write(6, '(2a)') 'ERROR register_constituents: expected this error: ', & + trim(expected_error) + else + if (trim(errmsg) /= trim(expected_error)) then + write(6, '(4a)') 'ERROR register_constituents: expected this error: ', & + trim(expected_error), ' Got: ', trim(errmsg) + end if + end if + + ! Now try again but with a compatible constituent - should be ignored when + ! the constituents object is created + ! Use the suite information to call the register phase + errflg = 0 + call ccpp_deallocate_dynamic_constituents() + deallocate(host_constituents) + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_register(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in register of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + allocate(host_constituents(3)) + call host_constituents(1)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errflg, errmsg=errmsg) + call host_constituents(2)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errflg, errmsg=errmsg) + call host_constituents(3)%instantiate( & + std_name='cloud_ice_dry_mixing_ratio', & + long_name='Cloud ice dry mixing ratio', & + diag_name='CLDICE', & + units='kg kg-1', & + vertical_dim='vertical_layer_dimension', & + advected=.true., & + default_value=0._kind_phys, & + !water_species=.true., & + mixing_ratio_type='dry', & + errcode=errflg, errmsg=errmsg) + + call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) + if (errflg == 0) then + call ccpp_register_constituents(host_constituents, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(2a)') 'ERROR register_constituents: ', trim(errmsg) + retval = .false. + return + end if + ! Check number of advected constituents + if (errflg == 0) then + call ccpp_number_constituents(num_advected, errmsg=errmsg, & + errflg=errflg) + call check_errflg(subname // ".num_advected", errflg, errmsg, errflg_final) + end if + if (num_advected /= 6) then + write(6, '(a,i0)') "ERROR: num advected constituents = ", num_advected + retval = .false. + return + end if + ! Initialize constituent data + call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, errflg=errflg, errmsg=errmsg) + + ! Stop tests here if initialization failed (as all other tests will likely + ! fail as well: + if (errflg /= 0) then + retval = .false. + return + end if + + ! Initialize our 'data' + const_ptr => ccpp_constituents_array() + + ! Check if the specific humidity index can be found: + call ccpp_const_get_index('specific_humidity', const_index=index, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_specific_humidity", errflg, errmsg, & + errflg_final) + + ! Check if the cloud liquid index can be found: + call ccpp_const_get_index(stdname='cloud_liquid_dry_mixing_ratio', & + const_index=index_liq, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_cld_liq", errflg, errmsg, & + errflg_final) + + ! Check if the cloud ice index can be found: + call ccpp_const_get_index(stdname='cloud_ice_dry_mixing_ratio', & + const_index=index_ice, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_cld_ice", errflg, errmsg, & + errflg_final) + + ! Check if the dynamic constituents indices can be found + call ccpp_const_get_index(stdname='dyn_const1', const_index=index_dyn1, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_dyn_const1", errflg, errmsg, & + errflg_final) + call ccpp_const_get_index(stdname='dyn_const2_wrt_moist_air', const_index=index_dyn2, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_dyn_const2", errflg, errmsg, & + errflg_final) + call ccpp_const_get_index(stdname='dyn_const3_wrt_moist_air_and_condensed_water', const_index=index_dyn3, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_dyn_const3", errflg, errmsg, & + errflg_final) + + ! Load up the test array indices + call ccpp_const_get_index(stdname=const_std_name, const_index=test_scalar_const_index, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // "." // const_std_name, errflg, errmsg, & + errflg_final) + do sind = 1, num_consts + call ccpp_const_get_index(stdname=std_name_array(sind), & + const_index=test_const_indices(sind), errflg=errflg, errmsg=errmsg) + call check_errflg(subname // "." // std_name_array(sind), errflg, errmsg, & + errflg_final) + end do + + ! Stop tests here if the index checks failed, as all other tests will + ! likely fail as well: + if (errflg_final /= 0) then + retval = .false. + return + end if + + call init_data(const_ptr, index, index_liq, index_ice, index_dyn3) + + ! Check some constituent properties + ! ++++++++++++++++++++++++++++++++++ + + const_props => ccpp_model_const_properties() + + ! Standard name: + call const_props(index)%standard_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get standard_name for specific_humidity, index = ", & + index, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'specific_humidity') then + write(6, *) "ERROR: standard name, '", trim(const_str), & + "' should be 'specific_humidity'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check standard name for a dynamic constituent + call const_props(index_dyn2)%standard_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get standard_name for dyn_const2, index = ", & + index_dyn2, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'dyn_const2_wrt_moist_air') then + write(6, *) "ERROR: standard name, '", trim(const_str), & + "' should be 'dyn_const2_wrt_moist_air'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Long name: + call const_props(index_liq)%long_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get long_name for cld_liq index = ", & + index_liq, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'Cloud liquid dry mixing ratio') then + write(6, *) "ERROR: long name, '", trim(const_str), & + "' should be 'Cloud liquid dry mixing ratio'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check long name for a dynamic constituent + call const_props(index_dyn1)%long_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get long_name for dyn_const1 index = ", & + index_dyn1, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'dyn const1') then + write(6, *) "ERROR: long name, '", trim(const_str), & + "' should be 'dyn const1'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Diagnostic name: + call const_props(index_liq)%diagnostic_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get diagnostic name for cld_liq index = ", & + index_liq, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'CLDLIQ') then + write(6, *) "ERROR: diagnostic name, '", trim(const_str), & + "' should be 'CLDLIQ'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check default diagnostic name is set correctly + call const_props(index_ice)%diagnostic_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get diagnostic name for cld_ice index = ", & + index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'CLDICE') then + write(6, *) "ERROR: diagnostic name, '", trim(const_str), & + "' should be 'CLDICE'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check diagnostic name of a dynamic constituent + call const_props(index_dyn2)%diagnostic_name(const_str, errflg, & + errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get diagnostic name for dyn_const2 index = ", & + index_dyn2, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'DYNCONST2') then + write(6, *) "ERROR: diagnostic name, '", trim(const_str), & + "' should be 'DYNCONST2'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Mass mixing ratio: + call const_props(index_ice)%is_mass_mixing_ratio(const_log, errflg, & + errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get mass mixing ratio prop for cld_ice index = ", & + index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: cloud ice is not a mass mixing_ratio" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check mass mixing ratio for a dynamic constituent + call const_props(index_dyn2)%is_mass_mixing_ratio(const_log, errflg, & + errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get mass mixing ratio prop for dyn_const2 index = ", & + index_dyn2, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const2 is not a mass mixing_ratio" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Dry mixing ratio: + call const_props(index_ice)%is_dry(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get dry prop for cld_ice index = ", index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: cloud ice mass_mixing_ratio is not dry" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check wet mixing ratio for dynamic constituent 1 + call const_props(index_dyn1)%is_dry(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get dry prop for dyn_const1 index = ", index_dyn1, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (const_log) then + write(6, *) "ERROR: dyn_const1 is dry and should be wet" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + call const_props(index_dyn1)%is_wet(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get wet prop for dyn_const1 index = ", index_dyn1, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const1 is not wet but should be" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check moist mixing ratio for dynamic constituent 2 + call const_props(index_dyn2)%is_dry(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get dry prop for dyn_const2 index = ", index_dyn2, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (const_log) then + write(6, *) "ERROR: dyn_const2 is dry and should be moist" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + call const_props(index_dyn2)%is_moist(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get moist prop for dyn_const2 index = ", index_dyn2, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const2 is not moist but should be" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check dry mixing ratio for dynamic constituent 3 + call const_props(index_dyn3)%is_dry(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get dry prop for dyn_const3 index = ", index_dyn3, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const3 is not dry and should be" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! ------------------- + + ! ------------------- + ! minimum value tests: + ! ------------------- + + ! Check that a constituent's minimum value defaults to zero: + call const_props(index_dyn2)%minimum(check_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get minimum value for dyn_const2 index = ", index_dyn2, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (check_value /= 0._kind_phys) then ! Should be zero + write(6, *) "ERROR: 'minimum' should default to zero for all ", & + "constituents unless set by host model or scheme metadata." + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that a constituent instantiated with a specified minimum value + ! actually contains that minimum value property: + call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get minimum value for dyn_const1 index = ", index_dyn1, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (check_value /= 1000._kind_phys) then !Should be 1000 + write(6, *) "ERROR: 'minimum' should give a value of 1000 ", & + "for dyn_const1, as was set during instantiation." + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that setting a constituent's minimum value works + ! as expected: + call const_props(index_dyn1)%set_minimum(1._kind_phys, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to set minimum value for dyn_const1 index = ", index_dyn1, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + " trying to get minimum value for dyn_const1 index = ", & + index_dyn1, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errflg == 0) then + if (check_value /= 1._kind_phys) then ! Should now be one + write(6, *) "ERROR: 'set_minimum' did not set constituent", & + " minimum value correctly." + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! ---------------------- + ! molecular weight tests: + ! ---------------------- + + ! Check that a constituent instantiated with a specified molecular + ! weight actually contains that molecular weight property value: + call const_props(index)%molar_mass(check_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get molecular weight for specific humidity index = ", & + index, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (check_value /= 2000._kind_phys) then ! Should be 2000 + write(6, *) "ERROR: 'molar_mass' should give a value of 2000 ", & + "for specific humidity, as was set during instantiation." + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that setting a constituent's molecular weight works + ! as expected: + call const_props(index_ice)%set_molar_mass(1._kind_phys, errflg, & + errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to set molecular weight for cld_ice index = ", index_ice, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + call const_props(index_ice)%molar_mass(check_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + " trying to get molecular weight for cld_ice index = ", & + index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errflg == 0) then + if (check_value /= 1._kind_phys) then ! Should be equal to one + write(6, *) "ERROR: 'set_molar_mass' did not set constituent", & + " molecular weight value correctly." + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! ------------------- + ! thermo-active tests: + ! ------------------- + + ! Check that being thermodynamically active defaults to False: + call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get thermo_active prop for cld_ice index = ", index_ice, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (check) then ! Should be False + write(6, *) "ERROR: 'is_thermo_active' should default to False ", & + "for all constituents unless set by host model." + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that setting a constituent to be thermodynamically active works + ! as expected: + call const_props(index_ice)%set_thermo_active(.true., errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to set thermo_active prop for cld_ice index = ", index_ice, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + " trying to get thermo_active prop for cld_ice index = ", & + index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errflg == 0) then + if (.not. check) then ! Should now be True + write(6, *) "ERROR: 'set_thermo_active' did not set", & + " thermo_active constituent property correctly." + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! ------------------- + + ! ------------------- + ! water-species tests: + ! ------------------- + + ! Check that being a water species defaults to False: + call const_props(index_liq)%is_water_species(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get water_species prop for cld_liq index = ", index_liq, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (check) then ! Should be False + write(6, *) "ERROR: 'is_water_species' should default to False ", & + "for all constituents unless set by host model." + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that setting a constituent to be a water species works + ! as expected: + call const_props(index_liq)%set_water_species(.true., errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to set water_species prop for cld_liq index = ", index_liq, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + call const_props(index_liq)%is_water_species(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + " trying to get water_species prop for cld_liq index = ", & + index_liq, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errflg == 0) then + if (.not. check) then ! Should now be True + write(6, *) "ERROR: 'set_water_species' did not set", & + " water_species constituent property correctly." + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that setting a constituent to be a water species via the + ! instantiate call works as expected + call const_props(index_dyn1)%is_water_species(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + "trying to get water_species prop for dyn_const1 index = ", & + index_dyn1, trim(errmsg) + end if + if (errflg == 0) then + if (.not. check) then ! Should now be True + write(6, *) "ERROR: 'water_species=.true. did not set", & + " water_species constituent property correctly" + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + call const_props(index_dyn2)%is_water_species(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + "trying to get water_species prop for dyn_const2 index = ", & + index_dyn2, trim(errmsg) + end if + if (errflg == 0) then + if (check) then ! Should now be False + write(6, *) "ERROR: 'water_species=.false. did not set", & + " water_species constituent property correctly" + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! ------------------- + + ! Check that setting a constituent's default value works as expected + call const_props(index_liq)%has_default(has_default, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to check for default for cld_liq index = ", index_liq, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (.not. has_default) then + write(6, *) "ERROR: cloud_liquid_dry_mixing_ratio should have default but doesn't" + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + call const_props(index_ice)%has_default(has_default, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to check for default for cld_ice index = ", index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (.not. has_default) then + write(6, *) "ERROR: cloud ice_dry_mixing_ratio should have default but doesn't" + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + call const_props(index_ice)%default_value(default_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to grab default for cld_ice index = ", index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (default_value /= 0.0_kind_phys) then + write(6, *) "ERROR: cloud ice mass_mixing_ratio default is ", default_value, & + " but should be 0.0" + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! ++++++++++++++++++++++++++++++++++ + + ! Set error flag to the "final" value, because any error + ! above will likely result in a large number of failures + ! below: + errflg = errflg_final + + ! Call ccpp_init + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_init(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + + ! Call ccpp_physics_init + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + + ! Check indices + call check_constituent_indices(test_scalar_const_index, test_const_indices, & + errmsg, errflg) + call check_errflg(subname // " check suite indices", errflg, errmsg, & + errflg_final) + + ! Loop over time steps + do time_step = 1, num_time_steps + ! Initialize the timestep + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + end if + end if + end do + + do col_start = 1, ncols, 5 + if (errflg /= 0) then + continue + end if + col_end = min(col_start + 4, ncols) + + do sind = 1, num_suites + do index = 1, size(test_suites(sind)%suite_parts) + if (errflg == 0) then + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + col_start=col_start, col_end=col_end, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(5a)') trim(test_suites(sind)%suite_name), & + '/', trim(test_suites(sind)%suite_parts(index)),& + ': ', trim(errmsg) + exit + end if + end if + end do + end do + end do + ! Check indices + call check_constituent_indices(test_scalar_const_index, test_const_indices, & + errmsg, errflg) + call check_errflg(subname // " check suite indices", errflg, errmsg, & + errflg_final) + + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + exit + end if + end do + + ! Run "dycore" + if (errflg == 0) then + call advect_constituents() + end if + end do ! End time step loop + + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & + 'Exiting...' + exit + end if + end if + end do + + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_final(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_final, ', & + 'Exiting...' + exit + end if + end if + end do + + call ccpp_deallocate_dynamic_constituents() + deallocate(host_constituents) + + if (errflg == 0) then + ! Run finished without error, check answers + if (compare_data(num_advected)) then + write(6, *) 'Answers are correct!' + errflg = 0 + else + write(6, *) 'Answers are not correct!' + errflg = -1 + end if + end if + + ! Make sure "final" flag is non-zero if "errflg" is: + if (errflg /= 0) then + errflg_final = -1 ! Notify test script that a failure occured + end if + + ! Set return value to False if any errors were found: + retval = errflg_final == 0 + + end subroutine test_host + +end module test_prog diff --git a/end-to-end-tests/advection/test_host.meta b/end-to-end-tests/advection/test_host.meta new file mode 100644 index 00000000..ab33172f --- /dev/null +++ b/end-to-end-tests/advection/test_host.meta @@ -0,0 +1,70 @@ +[ccpp-table-properties] + name = suite_info + type = ddt +[ccpp-arg-table] + name = suite_info + type = ddt + +[ccpp-table-properties] + name = test_host + type = control +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/end-to-end-tests/advection/test_host_data.F90 b/end-to-end-tests/advection/test_host_data.F90 new file mode 100644 index 00000000..f360ad79 --- /dev/null +++ b/end-to-end-tests/advection/test_host_data.F90 @@ -0,0 +1,96 @@ +module test_host_data + + use ccpp_kinds, only: kind_phys + + implicit none + + !> \section arg_table_physics_state Argument Table + !! \htmlinclude arg_table_physics_state.html + type physics_state + real(kind=kind_phys), allocatable :: ps(:) ! surface pressure + real(kind=kind_phys), allocatable :: temp(:, :) ! temperature + real(kind=kind_phys), dimension(:, :, :), pointer :: q => null() ! constituent array + end type physics_state + + !> \section arg_table_test_host_data Argument Table + !! \htmlinclude arg_table_test_host_data.html + integer, public, parameter :: num_consts = 3 + character(len=32), public, parameter :: std_name_array(num_consts) = (/ & + 'specific_humidity ', & + 'cloud_liquid_dry_mixing_ratio', & + 'cloud_ice_dry_mixing_ratio ' /) + character(len=32), public, parameter :: const_std_name = std_name_array(1) + + integer :: const_inds(num_consts) = -1 ! test array access from suite + integer :: const_index = -1 ! test scalar access from suite + + public :: allocate_physics_state + public :: check_constituent_indices + +contains + + subroutine check_constituent_indices(test_index, test_indices, errmsg, errflg) + ! Check constituent indices against what was found by suite + ! indices are passed in rather than looked up to avoid a dependency loop + ! Dummy arguments + integer, intent(in) :: test_index ! scalar const index from host + integer, intent(in) :: test_indices(:) ! array_test_indices from host + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! Local variable + integer :: indx + integer :: emstrt + + errflg = 0 + errmsg = '' + if (test_index /= const_index) then + emstrt = len_trim(errmsg) + 1 + write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_index_check for ', & + const_std_name, test_index, ' /= ', const_index + errflg = errflg + 1 + end if + do indx = 1, num_consts + if (test_indices(indx) /= const_inds(indx)) then + emstrt = len_trim(errmsg) + 1 + if (len_trim(errmsg) > 0) then + write(errmsg(emstrt:), '(", ")') + emstrt = emstrt + 2 + end if + write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_indices_check for ', & + std_name_array(indx), test_indices(indx), ' /= ', const_inds(indx) + errflg = errflg + 1 + end if + end do + + ! Reset for next test + const_index = -1 + const_inds = -1 + + end subroutine check_constituent_indices + + subroutine allocate_physics_state(cols, levels, constituents, state) + integer, intent(in) :: cols + integer, intent(in) :: levels + real(kind=kind_phys), pointer :: constituents(:, :, :) + type(physics_state), intent(out) :: state + + if (allocated(state%ps)) then + deallocate(state%ps) + end if + allocate(state%ps(cols)) + state%ps = 0.0_kind_phys + if (allocated(state%temp)) then + deallocate(state%temp) + end if + allocate(state%temp(cols, levels)) + if (associated(state%q)) then + ! Do not deallocate (we do not own this array) + nullify(state%q) + end if + ! Point to the advected constituents array + state%q => constituents + + end subroutine allocate_physics_state + +end module test_host_data diff --git a/end-to-end-tests/advection/test_host_data.meta b/end-to-end-tests/advection/test_host_data.meta new file mode 100644 index 00000000..960ce33e --- /dev/null +++ b/end-to-end-tests/advection/test_host_data.meta @@ -0,0 +1,67 @@ +[ccpp-table-properties] + name = physics_state + type = ddt +[ccpp-arg-table] + name = physics_state + type = ddt +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) +[ Temp ] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys +[ q ] + standard_name = constituent_mixing_ratio + type = real + kind = kind_phys + units = kg kg-1 moist or dry air depending on type + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) +[ q(:,:,index_of_water_vapor_specific_humidity) ] + standard_name = water_vapor_specific_humidity + type = real + kind = kind_phys + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + +[ccpp-table-properties] + name = test_host_data + type = host +[ccpp-arg-table] + name = test_host_data + type = host +[ num_consts ] + standard_name = banana_array_dim + long_name = Size of test_banana_name_array + units = 1 + dimensions = () + type = integer +[ std_name_array ] + standard_name = test_banana_name_array + type = character | kind = len=32 + units = count + dimensions = (banana_array_dim) + protected = true +[ const_std_name ] + standard_name = test_banana_name + type = character | kind = len=32 + units = 1 + dimensions = () + protected = true +[ const_inds ] + standard_name = test_banana_constituent_indices + long_name = Array of constituent indices + units = 1 + dimensions = (banana_array_dim) + protected = true + type = integer +[ const_index ] + standard_name = test_banana_constituent_index + long_name = Constituent index + units = 1 + dimensions = () + type = integer diff --git a/end-to-end-tests/advection/test_host_mod.F90 b/end-to-end-tests/advection/test_host_mod.F90 new file mode 100644 index 00000000..5099b9c1 --- /dev/null +++ b/end-to-end-tests/advection/test_host_mod.F90 @@ -0,0 +1,176 @@ +module test_host_mod + + use ccpp_kinds, only: kind_phys + use test_host_data, only: physics_state, & + allocate_physics_state + + implicit none + public + + integer, parameter :: num_time_steps = 2 + real(kind=kind_phys), parameter :: tolerance = 1.0e-13_kind_phys + + !> \section arg_table_test_host_mod Argument Table + !! \htmlinclude arg_table_test_host_mod.html + !! + integer, parameter :: ncols = 10 + integer, parameter :: pver = 5 + integer, parameter :: pverp = pver + 1 + integer, protected :: ncnst = -1 + integer, protected :: index_qv = -1 + real(kind=kind_phys) :: dt + real(kind=kind_phys), parameter :: tfreeze = 273.15_kind_phys + type(physics_state) :: phys_state + integer :: num_model_times = -1 + integer, allocatable :: model_times(:) + + public :: init_data + public :: compare_data + public :: twist_array + + real(kind=kind_phys), private, allocatable :: check_vals(:, :, :) + real(kind=kind_phys), private :: check_temp(ncols, pver) + integer, private :: ind_liq = -1 + integer, private :: ind_ice = -1 + +contains + + subroutine init_data(constituent_array, index_qv_use, index_liq, index_ice, index_dyn) + + ! Dummy arguments + real(kind=kind_phys), pointer :: constituent_array(:, :, :) ! From host & suites + integer, intent(in) :: index_qv_use + integer, intent(in) :: index_liq + integer, intent(in) :: index_ice + integer, intent(in) :: index_dyn + + ! Local variables + integer :: col + integer :: lev + integer :: cind + integer :: itime + real(kind=kind_phys) :: qmax + real(kind=kind_phys), parameter :: inc = 0.1_kind_phys + + ! Allocate and initialize state + ! Temperature starts above freezing and decreases to -30C + ! water vapor is initialized in odd columns to different amounts + ncnst = size(constituent_array, 3) + call allocate_physics_state(ncols, pver, constituent_array, phys_state) + index_qv = index_qv_use + ind_liq = index_liq + ind_ice = index_ice + allocate(check_vals(ncols, pver, ncnst)) + check_vals(:, :, :) = 0.0_kind_phys + check_vals(:, :, index_dyn) = 1.0_kind_phys + do lev = 1, pver + phys_state%temp(:, lev) = tfreeze + (10.0_kind_phys * (lev - 3)) + qmax = real(lev, kind_phys) + do col = 1, ncols + if (mod(col, 2) == 1) then + phys_state%q(col, lev, index_qv) = qmax + else + phys_state%q(col, lev, index_qv) = 0.0_kind_phys + end if + end do + end do + check_vals(:, :, index_qv) = phys_state%q(:, :, index_qv) + check_temp(:, :) = phys_state%temp(:, :) + ! Do timestep 1 + do col = 1, ncols, 2 + check_temp(col, 1) = check_temp(col, 1) + 0.5_kind_phys + check_vals(col, 1, index_qv) = check_vals(col, 1, index_qv) - inc + check_vals(col, 1, ind_liq) = check_vals(col, 1, ind_liq) + inc + end do + do itime = 1, num_time_steps + do cind = 1, ncnst + call twist_array(check_vals(:, :, cind)) + end do + end do + + end subroutine init_data + + subroutine twist_array(array) + ! Dummy argument + real(kind=kind_phys), intent(inout) :: array(:, :) + + ! Local variables + integer :: icol, ilev ! Field coordinates + integer :: idir ! 'w' sign + integer :: levb, leve ! Starting and ending level indices + real(kind=kind_phys) :: last_val, next_val + + idir = 1 + leve = (pver * mod(ncols, 2)) + mod(ncols - 1, 2) + last_val = array(ncols, leve) + do icol = 1, ncols + levb = ((pver * (1 - idir)) + (1 + idir)) / 2 + leve = ((pver * (1 + idir)) + (1 - idir)) / 2 + do ilev = levb, leve, idir + next_val = array(icol, ilev) + array(icol, ilev) = last_val + last_val = next_val + end do + idir = -1 * idir + end do + + end subroutine twist_array + + logical function compare_data(ncnst) + + integer, intent(in) :: ncnst + + integer :: col + integer :: lev + integer :: cind + logical :: need_header + real(kind=kind_phys) :: check + real(kind=kind_phys) :: denom + + compare_data = .true. + + need_header = .true. + do lev = 1, pver + do col = 1, ncols + check = check_temp(col, lev) + if (abs((phys_state%temp(col, lev) - check) / check) > & + tolerance) then + if (need_header) then + write(6, '(" COL LEV T MIDPOINTS EXPECTED")') + need_header = .false. + end if + write(6, '(2i5,2(3x,es15.7))') col, lev, & + phys_state%temp(col, lev), check + compare_data = .false. + end if + end do + end do + ! Check constituents + need_header = .true. + do cind = 1, ncnst + do lev = 1, pver + do col = 1, ncols + check = check_vals(col, lev, cind) + if (check < tolerance) then + denom = 1.0_kind_phys + else + denom = check + end if + if (abs((phys_state%q(col, lev, cind) - check) / denom) > & + tolerance) then + if (need_header) then + write(6, '(2(2x,a),3x,a,10x,a,14x,a)') & + 'COL', 'LEV', 'C#', 'Q', 'EXPECTED' + need_header = .false. + end if + write(6, '(3i5,2(3x,es15.7))') col, lev, cind, & + phys_state%q(col, lev, cind), check + compare_data = .false. + end if + end do + end do + end do + + end function compare_data + +end module test_host_mod diff --git a/end-to-end-tests/advection/test_host_mod.meta b/end-to-end-tests/advection/test_host_mod.meta new file mode 100644 index 00000000..6c3d15eb --- /dev/null +++ b/end-to-end-tests/advection/test_host_mod.meta @@ -0,0 +1,64 @@ +[ccpp-table-properties] + name = test_host_mod + type = host +[ccpp-arg-table] + name = test_host_mod + type = host +[ ncols] + standard_name = horizontal_dimension + units = count + type = integer + protected = True + dimensions = () +[ pver ] + standard_name = vertical_layer_dimension + units = count + type = integer + protected = True + dimensions = () +[ pverP ] + standard_name = vertical_interface_dimension + type = integer + units = count + protected = True + dimensions = () +[ ncnst ] + standard_name = number_of_tracers + type = integer + units = count + protected = True + dimensions = () +[ index_qv ] + standard_name = index_of_water_vapor_specific_humidity + units = index + type = integer + protected = True + dimensions = () +[ dt ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real | kind = kind_phys +[ tfreeze ] + standard_name = water_temperature_at_freezing + long_name = Freezing temperature of water at sea level + units = K + dimensions = () + type = real | kind = kind_phys +[ phys_state ] + standard_name = physics_state_derived_type + long_name = Physics State DDT + type = physics_state + dimensions = () +[ num_model_times ] + standard_name = number_of_model_times + type = integer + units = count + dimensions = () +[ model_times ] + standard_name = model_times + units = seconds + dimensions = (number_of_model_times) + type = integer + allocatable = True diff --git a/end-to-end-tests/capgen_ng/CMakeLists.txt b/end-to-end-tests/capgen_ng/CMakeLists.txt new file mode 100644 index 00000000..6236c598 --- /dev/null +++ b/end-to-end-tests/capgen_ng/CMakeLists.txt @@ -0,0 +1,95 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "setup_coeffs" "temp_set" "temp_adjust" "temp_calc_adjust" "make_ddt" "environ_conditions") +set(HOST_FILES "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "ddt_suite.xml" "temp_suite.xml") +set(KIND_TYPE "kind_phys=REAL64") +set(HOST "test_host") + +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +# Fortran files are not all in one directory +set(SCHEME_FORTRAN_FILES "") +foreach(sfile ${SCHEME_FILES}) + find_file(fort_file "${sfile}.F90" NO_CACHE + HINTS ${CMAKE_CURRENT_SOURCE_DIR} + HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir1 + HINTS ${CMAKE_CURRENT_SOURCE_DIR}/source_dir2 + HINTS ${CMAKE_CURRENT_SOURCE_DIR}/adjust) + list(APPEND SCHEME_FORTRAN_FILES ${fort_file}) + unset(fort_file) +endforeach() +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES}) + +# Run ccpp_capgen_ng +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + KIND_TYPES ${KIND_TYPES} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_capgen_ng.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_capgen_host_integration.F90 +) +target_link_libraries(test_capgen_ng.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_capgen_ng.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_capgen_ng.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_capgen_ng_omp1 + COMMAND test_capgen_ng.x) + +add_test(NAME test_capgen_ng_omp2 + COMMAND test_capgen_ng.x) + +set_tests_properties(test_capgen_ng_omp1 + PROPERTIES + ENVIRONMENT "OMP_NUM_THREADS=1" +) + +set_tests_properties(test_capgen_ng_omp2 + PROPERTIES + ENVIRONMENT "OMP_NUM_THREADS=2" +) diff --git a/end-to-end-tests/capgen_ng/README.md b/end-to-end-tests/capgen_ng/README.md new file mode 100644 index 00000000..a56bfcdf --- /dev/null +++ b/end-to-end-tests/capgen_ng/README.md @@ -0,0 +1,14 @@ +# Capgen Test + +Contains tests for overall capgen capabilities such as: +- Multiple suites +- Multiple groups +- General DDT usage +- DDT with undocumented DDT member variable +- Dimensions with `ccpp_constant_one:N` and just `N` +- Non-standard dimensions (not just horizontal and vertical) (including integer dimensions) +- Variables that should be promoted to suite level +- Dimensions that are set in the register phase and used to allocate module-level + interstitial variables +- Threading + diff --git a/end-to-end-tests/capgen_ng/adjust/temp_kinds.F90 b/end-to-end-tests/capgen_ng/adjust/temp_kinds.F90 new file mode 100644 index 00000000..3fb4cca4 --- /dev/null +++ b/end-to-end-tests/capgen_ng/adjust/temp_kinds.F90 @@ -0,0 +1,12 @@ +! Define a new Fortran kind for use within +! various temp_* test files. + +module temp_kinds + + implicit none + private + + integer, public, parameter :: temp_r8 = selected_real_kind(12) !8-byte real + integer, public, parameter :: temp_i8 = selected_int_kind(13) !8-byte integer + +end module temp_kinds diff --git a/end-to-end-tests/capgen_ng/capgen_test_reports.py b/end-to-end-tests/capgen_ng/capgen_test_reports.py new file mode 100644 index 00000000..c168827d --- /dev/null +++ b/end-to-end-tests/capgen_ng/capgen_test_reports.py @@ -0,0 +1,151 @@ +#! /usr/bin/env python3 +""" +----------------------------------------------------------------------- + Description: Test capgen database report python interface + + Assumptions: + + Command line arguments: build_dir database_filepath + + Usage: python test_reports +----------------------------------------------------------------------- +""" +import os +import unittest + +from test_stub import BaseTests + +_BUILD_DIR = os.path.join(os.path.abspath(os.environ['BUILD_DIR']), "test", "capgen_test") +_DATABASE = os.path.abspath(os.path.join(_BUILD_DIR, "ccpp", "datatable.xml")) + +_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +_FRAMEWORK_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, os.pardir)) +_SCRIPTS_DIR = os.path.join(_FRAMEWORK_DIR, "scripts") +_SRC_DIR = os.path.join(_FRAMEWORK_DIR, "src") + +# Check data +_HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] +_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_ddt_suite_cap.F90"), + os.path.join(_BUILD_DIR, "ccpp", "ccpp_temp_suite_cap.F90")] +_UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), + os.path.join(_SRC_DIR, "ccpp_constituent_prop_mod.F90"), + os.path.join(_SRC_DIR, "ccpp_scheme_utils.F90"), + os.path.join(_SRC_DIR, "ccpp_hashable.F90"), + os.path.join(_SRC_DIR, "ccpp_hash_table.F90")] +_CCPP_FILES = _UTILITY_FILES + \ + [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90"), + os.path.join(_BUILD_DIR, "ccpp", "ccpp_ddt_suite_cap.F90"), + os.path.join(_BUILD_DIR, "ccpp", "ccpp_temp_suite_cap.F90")] +_DEPENDENCIES = [os.path.join(_TEST_DIR, "adjust", "temp_kinds.F90"), + os.path.join(_TEST_DIR, "ddt2"), + os.path.join(_TEST_DIR, "bar.F90"), + os.path.join(_TEST_DIR, "foo.F90")] +_PROCESS_LIST = ["setter=temp_set", "adjusting=temp_calc_adjust"] +_MODULE_LIST = ["environ_conditions", "make_ddt", "setup_coeffs", "temp_adjust", + "temp_calc_adjust", "temp_set"] +_SUITE_LIST = ["ddt_suite", "temp_suite"] +_INPUT_VARS_DDT = ["model_times", "number_of_model_times", + "horizontal_loop_begin", "horizontal_loop_end", + "surface_air_pressure", "horizontal_dimension"] +_OUTPUT_VARS_DDT = ["ccpp_error_code", "ccpp_error_message", "model_times", + "surface_air_pressure", "number_of_model_times"] +_REQUIRED_VARS_DDT = _INPUT_VARS_DDT + _OUTPUT_VARS_DDT +_PROT_VARS_TEMP = ["horizontal_loop_begin", "horizontal_loop_end", + "horizontal_dimension", "vertical_layer_dimension", + "number_of_tracers", + "lower_bound_of_vertical_dimension_of_soil", + "upper_bound_of_vertical_dimension_of_soil", + "configuration_variable", + # Added for --debug + "index_of_water_vapor_specific_humidity", + "vertical_interface_dimension"] +_REQUIRED_VARS_TEMP = ["ccpp_error_code", "ccpp_error_message", + "potential_temperature", + "potential_temperature_at_interface", + "coefficients_for_interpolation", + "potential_temperature_increment", + "surface_air_pressure", "time_step_for_physics", + "water_vapor_specific_humidity", + "soil_levels", + "temperature_at_diagnostic_levels", + "array_variable_for_testing"] +_INPUT_VARS_TEMP = ["potential_temperature", + "potential_temperature_at_interface", + "coefficients_for_interpolation", + "potential_temperature_increment", + "surface_air_pressure", "time_step_for_physics", + "water_vapor_specific_humidity", + "soil_levels", + "temperature_at_diagnostic_levels", + "array_variable_for_testing"] +_OUTPUT_VARS_TEMP = ["ccpp_error_code", "ccpp_error_message", + "potential_temperature", + "potential_temperature_at_interface", + "coefficients_for_interpolation", + "surface_air_pressure", "water_vapor_specific_humidity", + "soil_levels", + "temperature_at_diagnostic_levels", + "array_variable_for_testing"] + + +class TestCapgenHostDataTables(unittest.TestCase, BaseTests.TestHostDataTables): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + + +class CommandLineCapgenHostDatafileRequiredFiles(unittest.TestCase, BaseTests.TestHostCommandLineDataFiles): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" + + +class TestCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuite): + database = _DATABASE + required_vars = _REQUIRED_VARS_DDT + input_vars = _INPUT_VARS_DDT + output_vars = _OUTPUT_VARS_DDT + suite_name = "ddt_suite" + + +class CommandLineCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuiteCommandLine): + database = _DATABASE + required_vars = _REQUIRED_VARS_DDT + input_vars = _INPUT_VARS_DDT + output_vars = _OUTPUT_VARS_DDT + suite_name = "ddt_suite" + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" + + +class TestCapgenTempSuite(unittest.TestCase, BaseTests.TestSuiteExcludeProtected): + database = _DATABASE + required_vars = _REQUIRED_VARS_TEMP + _PROT_VARS_TEMP + input_vars = _INPUT_VARS_TEMP + _PROT_VARS_TEMP + required_vars_excluding_protected = _REQUIRED_VARS_TEMP + input_vars_excluding_protected = _INPUT_VARS_TEMP + output_vars = _OUTPUT_VARS_TEMP + suite_name = "temp_suite" + + +class CommandLineCapgenTempSuite(unittest.TestCase, BaseTests.TestSuiteExcludeProtectedCommandLine): + database = _DATABASE + required_vars = _REQUIRED_VARS_TEMP + _PROT_VARS_TEMP + input_vars = _INPUT_VARS_TEMP + _PROT_VARS_TEMP + required_vars_excluding_protected = _REQUIRED_VARS_TEMP + input_vars_excluding_protected = _INPUT_VARS_TEMP + output_vars = _OUTPUT_VARS_TEMP + suite_name = "temp_suite" + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" diff --git a/end-to-end-tests/capgen_ng/ddt2.F90 b/end-to-end-tests/capgen_ng/ddt2.F90 new file mode 100644 index 00000000..ce560846 --- /dev/null +++ b/end-to-end-tests/capgen_ng/ddt2.F90 @@ -0,0 +1,12 @@ +module ddt2 + + use ccpp_kinds, only: kind_phys + + implicit none + + type ty_ddt2 + integer :: foo + real(kind=kind_phys) :: bar + end type ty_ddt2 + +end module ddt2 diff --git a/end-to-end-tests/capgen_ng/ddt_suite.xml b/end-to-end-tests/capgen_ng/ddt_suite.xml new file mode 100644 index 00000000..72c9c436 --- /dev/null +++ b/end-to-end-tests/capgen_ng/ddt_suite.xml @@ -0,0 +1,8 @@ + + + + + make_ddt + environ_conditions + + diff --git a/end-to-end-tests/capgen_ng/environ_conditions.meta b/end-to-end-tests/capgen_ng/environ_conditions.meta new file mode 100644 index 00000000..30693cf7 --- /dev/null +++ b/end-to-end-tests/capgen_ng/environ_conditions.meta @@ -0,0 +1,110 @@ +[ccpp-table-properties] + name = environ_conditions + type = scheme + source_path = source_dir1 +[ccpp-arg-table] + name = environ_conditions_run + type = scheme +[ psurf ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = environ_conditions_init + type = scheme +[ nbox ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ o3 ] + standard_name = ozone + units = ppmv + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = out +[ hno3 ] + standard_name = nitric_acid + units = ppmv + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = out +[ ntimes ] + standard_name = number_of_model_times + type = integer + units = count + dimensions = () + intent = out +[ model_times ] + standard_name = model_times + units = seconds + dimensions = (number_of_model_times) + type = integer + intent = out + allocatable = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = environ_conditions_final + type = scheme +[ ntimes ] + standard_name = number_of_model_times + type = integer + units = count + dimensions = () + intent = in +[ model_times ] + standard_name = model_times + units = seconds + dimensions = (number_of_model_times) + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/capgen_ng/make_ddt.F90 b/end-to-end-tests/capgen_ng/make_ddt.F90 new file mode 100644 index 00000000..429e8939 --- /dev/null +++ b/end-to-end-tests/capgen_ng/make_ddt.F90 @@ -0,0 +1,142 @@ +!Hello demonstration parameterization +! + +module make_ddt + + use ccpp_kinds, only: kind_phys + use ddt2, only: ty_ddt2 + + implicit none + private + + public :: make_ddt_init + public :: make_ddt_run + public :: make_ddt_timestep_final + public :: vmr_type + + type ty_ddt3 + integer :: dont_lose + integer :: your_head + integer :: to_gain_a_minute + integer :: you_need_your_head + integer :: your_brains_are_in_it + end type ty_ddt3 + + !> \section arg_table_vmr_type Argument Table + !! \htmlinclude arg_table_vmr_type.html + !! + type vmr_type + integer :: nvmr + real(kind=kind_phys), allocatable :: vmr_array(:, :) + type(ty_ddt2) :: error_maybe + type(ty_ddt3) :: burma_shave + end type vmr_type + +contains + + !> \section arg_table_make_ddt_run Argument Table + !! \htmlinclude arg_table_make_ddt_run.html + !! + subroutine make_ddt_run(cols, cole, o3, hno3, vmr, errmsg, errflg) + !---------------------------------------------------------------- + implicit none + !---------------------------------------------------------------- + + ! Dummy arguments + integer, intent(in) :: cols + integer, intent(in) :: cole + real(kind=kind_phys), intent(in) :: o3(:) + real(kind=kind_phys), intent(in) :: hno3(:) + type(vmr_type), intent(inout) :: vmr + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + ! Local variable + integer :: nbox + !---------------------------------------------------------------- + + errmsg = '' + errflg = 0 + + ! Check for correct threading behavior + nbox = cole - cols + 1 + if (size(o3) /= nbox) then + errflg = 1 + write(errmsg, '(2(a,i0))') 'SIZE(O3) = ', size(o3), ', should be ', nbox + else if (size(hno3) /= nbox) then + errflg = 1 + write(errmsg, '(2(a,i0))') 'SIZE(HNO3) = ', size(hno3), & + ', should be ', nbox + else + ! NOTE -- This is prototyping one approach to passing a large number of + ! chemical VMR values and is the predecssor for adding in methods and + ! maybe nesting DDTs (especially for aerosols) + vmr%vmr_array(cols:cole, 1) = o3(:) + vmr%vmr_array(cols:cole, 2) = hno3(:) + end if + + end subroutine make_ddt_run + + !> \section arg_table_make_ddt_init Argument Table + !! \htmlinclude arg_table_make_ddt_init.html + !! + subroutine make_ddt_init(nbox, vmr, errmsg, errflg) + + ! Dummy arguments + integer, intent(in) :: nbox + type(vmr_type), intent(out) :: vmr + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine initializes the vmr array + vmr%nvmr = 2 + allocate(vmr%vmr_array(nbox, vmr%nvmr)) + + errmsg = '' + errflg = 0 + + end subroutine make_ddt_init + + !> \section arg_table_make_ddt_timestep_final Argument Table + !! \htmlinclude arg_table_make_ddt_timestep_final.html + !! + subroutine make_ddt_timestep_final(ncols, vmr, errmsg, errflg) + + ! Dummy arguments + integer, intent(in) :: ncols + type(vmr_type), intent(in) :: vmr + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + ! Local variables + integer :: index + real(kind=kind_phys) :: rind + + errmsg = '' + errflg = 0 + + ! This routine checks the array values in vmr + if (size(vmr%vmr_array, 1) /= ncols) then + errflg = 1 + write(errmsg, '(2(a,i0))') 'VMR%VMR_ARRAY first dimension size is, ', & + size(vmr%vmr_array, 1), ', should be, ', ncols + else + do index = 1, ncols + rind = real(index, kind_phys) + if (vmr%vmr_array(index, 1) /= rind * 1.e-6_kind_phys) then + errflg = 1 + write(errmsg, '(a,i0,2(a,e12.4))') 'O3(', index, ') = ', & + vmr%vmr_array(index, 1), ', should be, ', & + rind * 1.e-6_kind_phys + exit + else if (vmr%vmr_array(index, 2) /= rind * 1.e-9_kind_phys) then + errflg = 1 + write(errmsg, '(a,i0,2(a,e12.4))') 'HNO3(', index, ') = ', & + vmr%vmr_array(index, 2), ', should be, ', & + rind * 1.e-9_kind_phys + exit + end if + end do + end if + + end subroutine make_ddt_timestep_final + +end module make_ddt diff --git a/end-to-end-tests/capgen_ng/make_ddt.meta b/end-to-end-tests/capgen_ng/make_ddt.meta new file mode 100644 index 00000000..5981c26a --- /dev/null +++ b/end-to-end-tests/capgen_ng/make_ddt.meta @@ -0,0 +1,128 @@ +[ccpp-table-properties] + name = vmr_type + type = ddt + dependencies = ddt2.F90 +[ccpp-arg-table] + name = vmr_type + type = ddt +[ nvmr ] + standard_name = number_of_chemical_species + units = count + dimensions = () + type = integer +[ vmr_array ] + standard_name = array_of_volume_mixing_ratios + units = ppmv + dimensions = (horizontal_dimension, number_of_chemical_species) + type = real + kind = kind_phys +[ccpp-table-properties] + name = make_ddt + type = scheme +[ccpp-arg-table] + name = make_ddt_run + type = scheme +[ cols ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + intent = in +[ cole ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + intent = in +[ O3 ] + standard_name = ozone + units = ppmv + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = in +[ HNO3 ] + standard_name = nitric_acid + units = ppmv + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = in +[ vmr ] + standard_name = volume_mixing_ratio_ddt + dimensions = () + type = vmr_type + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = make_ddt_init + type = scheme +[ nbox ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ vmr ] + standard_name = volume_mixing_ratio_ddt + dimensions = () + type = vmr_type + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = make_ddt_timestep_final + type = scheme +[ ncols ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ vmr ] + standard_name = volume_mixing_ratio_ddt + dimensions = () + type = vmr_type + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/capgen_ng/setup_coeffs.F90 b/end-to-end-tests/capgen_ng/setup_coeffs.F90 new file mode 100644 index 00000000..09c7fcc1 --- /dev/null +++ b/end-to-end-tests/capgen_ng/setup_coeffs.F90 @@ -0,0 +1,24 @@ +module setup_coeffs + use ccpp_kinds, only: kind_phys + implicit none + + public :: setup_coeffs_timestep_init + +contains + !> \section arg_table_setup_coeffs_timestep_init Argument Table + !! \htmlinclude arg_table_setup_coeffs_timestep_init.html + !! + subroutine setup_coeffs_timestep_init(coeffs, errmsg, errflg) + + real(kind=kind_phys), intent(inout) :: coeffs(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + coeffs(:) = 1._kind_phys + + end subroutine setup_coeffs_timestep_init + +end module setup_coeffs diff --git a/end-to-end-tests/capgen_ng/setup_coeffs.meta b/end-to-end-tests/capgen_ng/setup_coeffs.meta new file mode 100644 index 00000000..8d0fc5f4 --- /dev/null +++ b/end-to-end-tests/capgen_ng/setup_coeffs.meta @@ -0,0 +1,29 @@ +[ccpp-table-properties] + name = setup_coeffs + type = scheme +[ccpp-arg-table] + name = setup_coeffs_timestep_init + type = scheme +[ coeffs ] + standard_name = coefficients_for_interpolation + long_name = coefficients for interpolation + units = none + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/capgen_ng/source_dir1/environ_conditions.F90 b/end-to-end-tests/capgen_ng/source_dir1/environ_conditions.F90 new file mode 100644 index 00000000..9c92faea --- /dev/null +++ b/end-to-end-tests/capgen_ng/source_dir1/environ_conditions.F90 @@ -0,0 +1,96 @@ +module environ_conditions + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: environ_conditions_init + public :: environ_conditions_run + public :: environ_conditions_final + + integer, parameter :: input_model_times = 3 + integer, parameter :: input_model_values(input_model_times) = (/ 31, 37, 41 /) + +contains + + !> \section arg_table_environ_conditions_run Argument Table + !! \htmlinclude arg_table_environ_conditions_run.html + !! + subroutine environ_conditions_run(psurf, errmsg, errflg) + + ! This routine currently does nothing -- should update values + + real(kind=kind_phys), intent(in) :: psurf(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + end subroutine environ_conditions_run + + !> \section arg_table_environ_conditions_init Argument Table + !! \htmlinclude arg_table_environ_conditions_init.html + !! + subroutine environ_conditions_init(nbox, o3, hno3, ntimes, model_times, & + errmsg, errflg) + + integer, intent(in) :: nbox + real(kind=kind_phys), intent(out) :: o3(:) + real(kind=kind_phys), intent(out) :: hno3(:) + integer, intent(out) :: ntimes + integer, allocatable, intent(out) :: model_times(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: i, j + + errmsg = '' + errflg = 0 + + ! This may be replaced with MusicBox json environmental conditions reader??? + + do i = 1, nbox + o3(i) = real(i, kind_phys) * 1.e-6_kind_phys + hno3(i) = real(i, kind_phys) * 1.e-9_kind_phys + end do + + ntimes = input_model_times + allocate(model_times(ntimes)) + model_times = input_model_values + + end subroutine environ_conditions_init + + !> \section arg_table_environ_conditions_final Argument Table + !! \htmlinclude arg_table_environ_conditions_final.html + !! + subroutine environ_conditions_final(ntimes, model_times, errmsg, errflg) + + integer, intent(in) :: ntimes + integer, intent(in) :: model_times(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine checks the size and values of model_times + if (ntimes /= input_model_times) then + errflg = 1 + write(errmsg, '(2(a,i0))') 'ntimes mismatch, ', ntimes, ' should be ', & + input_model_times + else if (size(model_times) /= input_model_times) then + errflg = 1 + write(errmsg, '(2(a,i0))') 'model_times size mismatch, ', & + size(model_times), ' should be ', input_model_times + else if (any(model_times /= input_model_values)) then + errflg = 1 + write(errmsg, *) 'model_times mismatch, ', & + model_times, ' should be ', input_model_values + else + errmsg = '' + errflg = 0 + end if + + end subroutine environ_conditions_final + +end module environ_conditions diff --git a/end-to-end-tests/capgen_ng/source_dir2/temp_set.F90 b/end-to-end-tests/capgen_ng/source_dir2/temp_set.F90 new file mode 100644 index 00000000..30e0ec2a --- /dev/null +++ b/end-to-end-tests/capgen_ng/source_dir2/temp_set.F90 @@ -0,0 +1,125 @@ +!Test 3D parameterization +! + +module temp_set + + use ccpp_kinds, only: kind_phys, & + kind_temp + + implicit none + private + + public :: temp_set_init + public :: temp_set_timestep_init + public :: temp_set_run + public :: temp_set_final + +contains + + !> \section arg_table_temp_set_run Argument Table + !! \htmlinclude arg_table_temp_set_run.html + !! + subroutine temp_set_run(ncol, lev, timestep, temp_level, temp_diag, temp, ps, & + to_promote, promote_pcnst, slev_lbound, soil_levs, var_array, errmsg, errflg) + !---------------------------------------------------------------- + implicit none + !---------------------------------------------------------------- + + integer, intent(in) :: ncol, lev, slev_lbound + real(kind=kind_phys), intent(out) :: temp(:, :) + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(in) :: ps(:) + real(kind=kind_phys), intent(inout) :: temp_level(:, :) + real(kind=kind_phys), intent(inout) :: temp_diag(:, :) + real(kind=kind_phys), intent(inout) :: soil_levs(slev_lbound:) + real(kind=kind_phys), intent(inout) :: var_array(:, :, :, :) + real(kind=kind_temp), intent(out) :: to_promote(:, :) + real(kind=kind_phys), intent(out) :: promote_pcnst(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + integer :: ilev + + integer :: col_index + integer :: lev_index + real(kind=kind_phys) :: internal_scalar_var + + errmsg = '' + errflg = 0 + + ilev = size(temp_level, 2) + if (ilev /= (lev + 1)) then + errflg = 1 + errmsg = 'Invalid value for ilev, must be lev+1' + return + end if + + do col_index = 1, ncol + do lev_index = 1, lev + temp(col_index, lev_index) = (temp_level(col_index, lev_index) & + + temp_level(col_index, lev_index + 1)) / 2.0_kind_phys + end do + end do + + var_array(:, :, :, :) = 1._kind_phys + + ! + internal_scalar_var = soil_levs(slev_lbound) + internal_scalar_var = soil_levs(0) + + end subroutine temp_set_run + + !> \section arg_table_temp_set_init Argument Table + !! \htmlinclude arg_table_temp_set_init.html + !! + !subroutine temp_set_init(temp_inc_in, fudge, temp_inc_set, errmsg, errflg) + subroutine temp_set_init(temp_inc_in, temp_inc_set, errmsg, errflg) + + real(kind=kind_phys), intent(in) :: temp_inc_in + !real(kind=kind_phys), intent(in) :: fudge + real(kind=kind_phys), intent(out) :: temp_inc_set + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + temp_inc_set = temp_inc_in + + errmsg = '' + errflg = 0 + + end subroutine temp_set_init + + !> \section arg_table_temp_set_timestep_init Argument Table + !! \htmlinclude arg_table_temp_set_timestep_init.html + !! + subroutine temp_set_timestep_init(ncol, temp_inc, temp_level, & + errmsg, errflg) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(in) :: temp_inc + real(kind=kind_phys), intent(inout) :: temp_level(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + temp_level = temp_level + temp_inc + + end subroutine temp_set_timestep_init + + !> \section arg_table_temp_set_final Argument Table + !! \htmlinclude arg_table_temp_set_final.html + !! + subroutine temp_set_final(errmsg, errflg) + + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine currently does nothing + + errmsg = '' + errflg = 0 + + end subroutine temp_set_final + +end module temp_set diff --git a/end-to-end-tests/capgen_ng/temp_adjust.F90 b/end-to-end-tests/capgen_ng/temp_adjust.F90 new file mode 100644 index 00000000..515cf4a8 --- /dev/null +++ b/end-to-end-tests/capgen_ng/temp_adjust.F90 @@ -0,0 +1,127 @@ +! Test parameterization with no vertical level +! + +module temp_adjust + + use ccpp_kinds, only: kind_phys, & + kind_temp + + implicit none + private + + public :: temp_adjust_register + public :: temp_adjust_init + public :: temp_adjust_run + public :: temp_adjust_final + + logical :: module_level_config = .false. + +contains + + !> \section arg_table_temp_adjust_register Argument Table + !! \htmlinclude arg_table_temp_adjust_register.hml + !! + subroutine temp_adjust_register(config_var, errmsg, errflg) + logical, intent(in) :: config_var + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + module_level_config = config_var + errflg = 0 + errmsg = '' + + end subroutine temp_adjust_register + + !> \section arg_table_temp_adjust_run Argument Table + !! \htmlinclude arg_table_temp_adjust_run.html + !! + subroutine temp_adjust_run(foo, timestep, interstitial_var, temp_prev, temp_layer, qv, ps, & + to_promote, promote_pcnst, errmsg, errflg, innie, outie, optsie) + + integer, intent(in) :: foo + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(inout), optional :: qv(:, :) + real(kind=kind_phys), intent(inout) :: ps(:) + ! codee format off + REAL(kind_phys), intent(in) :: temp_prev(:,:) + REAL(kind_phys), intent(inout) :: temp_layer(:,:) + ! codee format on + real(kind=kind_temp), intent(in) :: to_promote(:, :) + real(kind=kind_phys), intent(in) :: promote_pcnst(:) + integer, intent(out) :: interstitial_var(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + real(kind=kind_phys), optional, intent(in) :: innie + real(kind=kind_phys), optional, intent(out) :: outie + real(kind=kind_phys), optional, intent(inout) :: optsie + !---------------------------------------------------------------- + + integer :: col_index + + errmsg = '' + errflg = 0 + + interstitial_var = 6 + if (size(interstitial_var) /= 3) then + errflg = 1 + errmsg = 'interstitial variable not allocated properly!' + return + end if + + if (.not. module_level_config) then + ! do nothing + return + end if + + do col_index = 1, foo + temp_layer(col_index, :) = temp_layer(col_index, :) + temp_prev(col_index, :) + if (present(qv)) qv(col_index, :) = qv(col_index, :) + 1.0_kind_phys + end do + if (present(innie) .and. present(outie) .and. present(optsie)) then + outie = innie * optsie + optsie = optsie + 1.0_kind_phys + end if + + end subroutine temp_adjust_run + + !> \section arg_table_temp_adjust_init Argument Table + !! \htmlinclude arg_table_temp_adjust_init.html + !! + subroutine temp_adjust_init(errmsg, errflg) + + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine currently does nothing + + errmsg = '' + errflg = 0 + + end subroutine temp_adjust_init + + !> \section arg_table_temp_adjust_final Argument Table + !! \htmlinclude arg_table_temp_adjust_final.html + !! + subroutine temp_adjust_final(interstitial_var, errmsg, errflg) + + integer, intent(in) :: interstitial_var(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine currently does nothing + + errmsg = '' + errflg = 0 + if (size(interstitial_var) /= 3) then + errflg = 1 + errmsg = 'interstitial variable not allocated properly!' + return + end if + if (interstitial_var(1) /= 6) then + errflg = 1 + errmsg = 'interstitial variable not set properly!' + end if + + end subroutine temp_adjust_final + +end module temp_adjust diff --git a/end-to-end-tests/capgen_ng/temp_adjust.meta b/end-to-end-tests/capgen_ng/temp_adjust.meta new file mode 100644 index 00000000..8f53ad76 --- /dev/null +++ b/end-to-end-tests/capgen_ng/temp_adjust.meta @@ -0,0 +1,155 @@ +[ccpp-table-properties] + name = temp_adjust + type = scheme + kind_spec = temp_kinds:kind_temp=>temp_r8 + dependencies = temp_kinds.F90 + dependencies_path = adjust +[ccpp-arg-table] + name = temp_adjust_register + type = scheme +[ config_var ] + standard_name = configuration_variable + type = logical + units = none + dimensions = () + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = temp_adjust_run + type = scheme +[ foo ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ interstitial_var ] + standard_name = output_only_interstitial_variable + units = 1 + dimensions = (dimension_for_interstitial_variable) + type = integer + intent = out +[ temp_prev ] + standard_name = potential_temperature_at_previous_timestep + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = in +[ temp_layer ] + standard_name = potential_temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + diagnostic_name = temperature +[ qv ] + standard_name = water_vapor_specific_humidity + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + diagnostic_name_fixed = Q + optional = True +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) + intent = inout +[ to_promote ] + standard_name = promote_this_variable_to_suite + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_temp + intent = in +[ promote_pcnst ] + standard_name = promote_this_variable_with_no_horizontal_dimension + units = K + dimensions = (number_of_tracers) + type = real + kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = temp_adjust_init + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = temp_adjust_final + type = scheme +[ interstitial_var ] + standard_name = output_only_interstitial_variable + units = 1 + dimensions = (dimension_for_interstitial_variable) + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/capgen_ng/temp_calc_adjust.F90 b/end-to-end-tests/capgen_ng/temp_calc_adjust.F90 new file mode 100644 index 00000000..54312133 --- /dev/null +++ b/end-to-end-tests/capgen_ng/temp_calc_adjust.F90 @@ -0,0 +1,111 @@ +!Test parameterization with no vertical level and hanging intent(out) variable +! + +module temp_calc_adjust + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: temp_calc_adjust_register + public :: temp_calc_adjust_init + public :: temp_calc_adjust_run + public :: temp_calc_adjust_final + +contains + +! codee format off +!> \section arg_table_temp_calc_adjust_register Argument Table +!! \htmlinclude arg_table_temp_calc_adjust_register.html +!! + SUBROUTINE temp_calc_adjust_register(dim_inter, errmsg, errflg) +! codee format on + integer, intent(out) :: dim_inter + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errflg = 0 + errmsg = '' + dim_inter = 3 + end subroutine temp_calc_adjust_register + + !> \section arg_table_temp_calc_adjust_run Argument Table + !! \htmlinclude arg_table_temp_calc_adjust_run.html + !! + subroutine temp_calc_adjust_run(nbox, timestep, temp_level, temp_calc, & + errmsg, errflg) + + integer, intent(in) :: nbox + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(in) :: temp_level(:, :) + real(kind=kind_phys), intent(out) :: temp_calc(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: col_index + real(kind=kind_phys) :: bar = 1.0_kind_phys + + errmsg = '' + errflg = 0 + + call temp_calc_adjust_nested_subroutine(temp_calc) + if (check_foo()) then + call foo(bar) + end if + + contains + + elemental subroutine temp_calc_adjust_nested_subroutine(temp) + + real(kind=kind_phys), intent(out) :: temp + !------------------------------------------------------------- + + temp = 1.0_kind_phys + + end subroutine temp_calc_adjust_nested_subroutine + + subroutine foo(bar) + real(kind=kind_phys), intent(inout) :: bar + bar = bar + 1.0_kind_phys + + end subroutine foo + + logical function check_foo() + check_foo = .true. + end function check_foo + + end subroutine temp_calc_adjust_run + + !> \section arg_table_temp_calc_adjust_init Argument Table + !! \htmlinclude arg_table_temp_calc_adjust_init.html + !! + subroutine temp_calc_adjust_init(errmsg, errflg) + + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine currently does nothing + + errmsg = '' + errflg = 0 + + end subroutine temp_calc_adjust_init + + !> \section arg_table_temp_calc_adjust_final Argument Table + !! \htmlinclude arg_table_temp_calc_adjust_final.html + !! + subroutine temp_calc_adjust_final(errmsg, errflg) + + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine currently does nothing + + errmsg = '' + errflg = 0 + + end subroutine temp_calc_adjust_final + +end module temp_calc_adjust diff --git a/end-to-end-tests/capgen_ng/temp_calc_adjust.meta b/end-to-end-tests/capgen_ng/temp_calc_adjust.meta new file mode 100644 index 00000000..4a959c42 --- /dev/null +++ b/end-to-end-tests/capgen_ng/temp_calc_adjust.meta @@ -0,0 +1,111 @@ +[ccpp-table-properties] + name = temp_calc_adjust + type = scheme + dependencies = +[ccpp-arg-table] + name = temp_calc_adjust_register + type = scheme +[ dim_inter ] + standard_name = dimension_for_interstitial_variable + type = integer + units = count + dimensions = () + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = temp_calc_adjust_run + type = scheme + process = adjusting +[ nbox ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp_level ] + standard_name = potential_temperature_at_interface + units = K + dimensions = (ccpp_constant_one:horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys + intent = in +[ temp_calc ] + standard_name = potential_temperature_at_previous_timestep + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = temp_calc_adjust_init + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = temp_calc_adjust_final + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/capgen_ng/temp_set.meta b/end-to-end-tests/capgen_ng/temp_set.meta new file mode 100644 index 00000000..f6cdec65 --- /dev/null +++ b/end-to-end-tests/capgen_ng/temp_set.meta @@ -0,0 +1,213 @@ +[ccpp-table-properties] + name = temp_set + type = scheme + source_path = source_dir2 + kind_spec = temp_kinds:kind_temp=>temp_r8 + dependencies = temp_kinds.F90 + dependencies_path = adjust +[ccpp-arg-table] + name = temp_set_run + type = scheme + process = setter +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ lev ] + standard_name = vertical_layer_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp_level ] + standard_name = potential_temperature_at_interface + units = K + dimensions = (ccpp_constant_one:horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys + intent = inout +[ temp_diag ] + standard_name = temperature_at_diagnostic_levels + units = K + dimensions = (horizontal_dimension, 6) + type = real + kind = kind_phys + intent = inout +[ temp ] + standard_name = potential_temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = out +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) + intent = in +[ to_promote ] + standard_name = promote_this_variable_to_suite + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_temp + intent = out +[ promote_pcnst ] + standard_name = promote_this_variable_with_no_horizontal_dimension + units = K + dimensions = (number_of_tracers) + type = real + kind = kind_phys + intent = out +[ slev_lbound ] + standard_name = lower_bound_of_vertical_dimension_of_soil + type = integer + units = count + dimensions = () + intent = in +[ soil_levs ] + standard_name = soil_levels + long_name = soil levels + units = cm + dimensions = (lower_bound_of_vertical_dimension_of_soil:upper_bound_of_vertical_dimension_of_soil) + type = real + kind = kind_phys + intent = inout +[ var_array ] + standard_name = array_variable_for_testing + long_name = array variable for testing + units = none + dimensions = (horizontal_dimension,2,4,6) + type = real + kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +# Init +[ccpp-arg-table] + name = temp_set_init + type = scheme +[ temp_inc_in ] + standard_name = potential_temperature_increment + long_name = Per time step potential temperature increment + units = K + dimensions = () + type = real + kind = kind_phys + intent = in +#[ fudge ] +# standard_name = random_fudge_factor +# long_name = Ignore this +# units = 1 +# dimensions = () +# type = real +# kind = kind_phys +# intent = in +# default_value = 1.0_kind_phys +[ temp_inc_set ] + standard_name = test_potential_temperature_increment + long_name = Per time step potential temperature increment + units = K + dimensions = () + type = real + kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +# Timestep Initialization +[ccpp-arg-table] + name = temp_set_timestep_init + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ temp_inc ] + standard_name = test_potential_temperature_increment + long_name = Per time step potential temperature increment + units = K + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp_level ] + standard_name = potential_temperature_at_interface + units = K + dimensions = (horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +# Finalize +[ccpp-arg-table] + name = temp_set_final + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/capgen_ng/temp_suite.xml b/end-to-end-tests/capgen_ng/temp_suite.xml new file mode 100644 index 00000000..7a4795c4 --- /dev/null +++ b/end-to-end-tests/capgen_ng/temp_suite.xml @@ -0,0 +1,12 @@ + + + + + setup_coeffs + temp_set + + + temp_calc_adjust + temp_adjust + + diff --git a/end-to-end-tests/capgen_ng/test_capgen_host_integration.F90 b/end-to-end-tests/capgen_ng/test_capgen_host_integration.F90 new file mode 100644 index 00000000..9dc43c89 --- /dev/null +++ b/end-to-end-tests/capgen_ng/test_capgen_host_integration.F90 @@ -0,0 +1,91 @@ +program test + use test_prog, only: test_host, & + suite_info, & + cm, & + cs + + implicit none + + character(len=cs), target :: test_parts1(2) = (/ 'physics1 ', & + 'physics2 ' /) + character(len=cs), target :: test_parts2(1) = (/ 'data_prep ' /) + character(len=cm), target :: test_invars1(11) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'index_of_water_vapor_specific_humidity', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'potential_temperature_increment ', & + 'soil_levels ', & + 'temperature_at_diagnostic_levels ', & + 'time_step_for_physics ', & + 'array_variable_for_testing ' /) + character(len=cm), target :: test_outvars1(10) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'soil_levels ', & + 'temperature_at_diagnostic_levels ', & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'array_variable_for_testing ' /) + character(len=cm), target :: test_reqvars1(13) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'index_of_water_vapor_specific_humidity', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'potential_temperature_increment ', & + 'time_step_for_physics ', & + 'soil_levels ', & + 'temperature_at_diagnostic_levels ', & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'array_variable_for_testing ' /) + + character(len=cm), target :: test_invars2(3) = (/ & + 'model_times ', & + 'number_of_model_times ', & + 'surface_air_pressure ' /) + + character(len=cm), target :: test_outvars2(4) = (/ & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'model_times ', & + 'number_of_model_times ' /) + + character(len=cm), target :: test_reqvars2(5) = (/ & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'model_times ', & + 'number_of_model_times ', & + 'surface_air_pressure ' /) + + type(suite_info) :: test_suites(2) + logical :: run_okay + + ! Setup expected test suite info + test_suites(1)%suite_name = 'temp_suite' + test_suites(1)%suite_parts => test_parts1 + test_suites(1)%suite_input_vars => test_invars1 + test_suites(1)%suite_output_vars => test_outvars1 + test_suites(1)%suite_required_vars => test_reqvars1 + test_suites(2)%suite_name = 'ddt_suite' + test_suites(2)%suite_parts => test_parts2 + test_suites(2)%suite_input_vars => test_invars2 + test_suites(2)%suite_output_vars => test_outvars2 + test_suites(2)%suite_required_vars => test_reqvars2 + + call test_host(run_okay, test_suites) + + if (run_okay) then + stop 0 + else + stop -1 + end if + +end program test diff --git a/end-to-end-tests/capgen_ng/test_host.F90 b/end-to-end-tests/capgen_ng/test_host.F90 new file mode 100644 index 00000000..409338a3 --- /dev/null +++ b/end-to-end-tests/capgen_ng/test_host.F90 @@ -0,0 +1,349 @@ +module test_prog + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public test_host + + ! Public data and interfaces + integer, public, parameter :: cs = 16 + integer, public, parameter :: cm = 64 + + !> \section arg_table_suite_info Argument Table + !! \htmlinclude arg_table_suite_info.html + !! + type, public :: suite_info + character(len=cs) :: suite_name = '' + character(len=cs), pointer :: suite_parts(:) => null() + character(len=cm), pointer :: suite_input_vars(:) => null() + character(len=cm), pointer :: suite_output_vars(:) => null() + character(len=cm), pointer :: suite_required_vars(:) => null() + end type suite_info + +contains + + logical function check_suite(test_suite) + use ccpp_static_api, only: ccpp_physics_suite_part_list + use ccpp_static_api, only: ccpp_physics_suite_variables + use test_utils, only: check_list + + ! Dummy argument + type(suite_info), intent(in) :: test_suite + ! Local variables + integer :: sind + logical :: check + integer :: errflg + character(len=512) :: errmsg + character(len=128), allocatable :: test_list(:) + + check_suite = .true. + write(6, *) "Checking suite ", trim(test_suite%suite_name) + ! First, check the suite parts + call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_parts, 'part names', & + suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the input variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.true., output_vars=.false.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_input_vars, & + 'input variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the output variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.false., output_vars=.true.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_output_vars, & + 'output variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check all required variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_required_vars, & + 'required variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + end function check_suite + + !> \section arg_table_test_host Argument Table + !! \htmlinclude arg_table_test_host.html + !! + subroutine test_host(retval, test_suites) + +#ifdef _OPENMP + use omp_lib +#endif + use test_host_mod, only: ncols, & + num_time_steps + use ccpp_static_api, only: ccpp_register + use ccpp_static_api, only: ccpp_init + use ccpp_static_api, only: ccpp_physics_init + use ccpp_static_api, only: ccpp_physics_timestep_init + use ccpp_static_api, only: ccpp_physics_run + use ccpp_static_api, only: ccpp_physics_timestep_final + use ccpp_static_api, only: ccpp_physics_final + use ccpp_static_api, only: ccpp_final + use ccpp_static_api, only: ccpp_physics_suite_list + use test_host_mod, only: init_data, & + compare_data, & + check_model_times + use test_utils, only: check_list + + type(suite_info), intent(in) :: test_suites(:) + logical, intent(out) :: retval + + logical :: check + integer :: col_start, col_end + integer :: thread_num, num_threads + integer :: index, sind + integer :: time_step + integer :: num_suites + character(len=128), allocatable :: suite_names(:) + character(len=512) :: errmsg + integer :: errflg + + ! Initialize our 'data' + call init_data() + + ! Gather and test the inspection routines + num_suites = size(test_suites) + call ccpp_physics_suite_list(suite_names) + retval = check_list(suite_names, test_suites(:)%suite_name, & + 'suite names') + write(6, *) 'Available suites are:' + do index = 1, size(suite_names) + do sind = 1, num_suites + if (trim(test_suites(sind)%suite_name) == & + trim(suite_names(index))) then + exit + end if + end do + write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & + ' = test_suites(', sind, ')' + end do + if (retval) then + do sind = 1, num_suites + check = check_suite(test_suites(sind)) + retval = retval .and. check + end do + end if + !!! Return here if any check failed + if (.not. retval) then + return + end if + + ! Use the suite information to call the register phase + do sind = 1, num_suites + call ccpp_register(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in ccpp_register for ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + ! Call the CCPP init phase for each suite + do sind = 1, num_suites + call ccpp_init(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in ccpp_init for ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + ! Use the suite information to setup the run + do sind = 1, num_suites + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in ccpp_physics_init for ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + ! Loop over time steps + do time_step = 1, num_time_steps + ! Initialize the timestep + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) + end if + if (errflg /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + exit + end if + if (errflg /= 0) then + exit + end if + end do + + run_phase_if_no_error: if (errflg == 0) then +#ifdef _OPENMP + num_threads = omp_get_max_threads() +#else + num_threads = 1 +#endif + !$OMP parallel num_threads (num_threads) & + !$OMP default (none) & + !$OMP shared (num_threads, num_suites, test_suites) & + !$OMP private (thread_num, col_start, col_end, errmsg) & + !$OMP reduction (+:errflg) +#ifdef _OPENMP + thread_num = omp_get_thread_num() +#else + thread_num = 0 +#endif + !$OMP do + do col_start = 1, ncols, 5 + if (errflg /= 0) then + continue + end if + col_end = min(col_start + 4, ncols) + do sind = 1, num_suites + if (errflg /= 0) then + continue + end if + do index = 1, size(test_suites(sind)%suite_parts) + if (errflg /= 0) then + continue + end if + write(0, '(a,i0,a,i0,5a,i0,a,i0)') 'Thread ', thread_num, '/', num_threads, & + ': calling run phase for suite ', trim(test_suites(sind)%suite_name), & + ' part ', trim(test_suites(sind)%suite_parts(index)), & + ' columns ', col_start, ':', col_end + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + col_start=col_start, col_end=col_end, & + errmsg=errmsg, errflg=errflg, & + thread_num=thread_num, nthreads=num_threads, & + nphys_threads=1) + if (errflg /= 0) then + write(6, '(5a)') trim(test_suites(sind)%suite_name), & + '/', trim(test_suites(sind)%suite_parts(index)), & + ': ', trim(errmsg) + end if + end do + end do + end do + !$OMP end do + !$OMP end parallel + end if run_phase_if_no_error + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) + end if + if (errflg /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + exit + end if + end do + end do ! End time step loop + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) + end if + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_name, ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & + 'Exiting...' + exit + end if + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_final(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_name, ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_final, ', & + 'Exiting...' + exit + end if + end do + + if (errflg == 0) then + ! Run finished without error, check answers + if (.not. check_model_times()) then + write(6, *) 'Model times error!' + errflg = -1 + else if (compare_data()) then + write(6, *) 'Answers are correct!' + errflg = 0 + else + write(6, *) 'Answers are not correct!' + errflg = -1 + end if + end if + + retval = errflg == 0 + + end subroutine test_host + +end module test_prog diff --git a/end-to-end-tests/capgen_ng/test_host.meta b/end-to-end-tests/capgen_ng/test_host.meta new file mode 100644 index 00000000..ab33172f --- /dev/null +++ b/end-to-end-tests/capgen_ng/test_host.meta @@ -0,0 +1,70 @@ +[ccpp-table-properties] + name = suite_info + type = ddt +[ccpp-arg-table] + name = suite_info + type = ddt + +[ccpp-table-properties] + name = test_host + type = control +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/end-to-end-tests/capgen_ng/test_host_data.F90 b/end-to-end-tests/capgen_ng/test_host_data.F90 new file mode 100644 index 00000000..3d332af3 --- /dev/null +++ b/end-to-end-tests/capgen_ng/test_host_data.F90 @@ -0,0 +1,60 @@ +module test_host_data + + use ccpp_kinds, only: kind_phys + + implicit none + private + + !> \section arg_table_physics_state Argument Table + !! \htmlinclude arg_table_physics_state.html + type physics_state + real(kind=kind_phys), dimension(:), allocatable :: & + ps, & ! surface pressure + soil_levs ! soil temperature (cm) + real(kind=kind_phys), dimension(:, :), allocatable :: & + u, & ! zonal wind (m/s) + v, & ! meridional wind (m/s) + pmid ! midpoint pressure (Pa) + real(kind=kind_phys), dimension(:, :, :), pointer :: & + q ! constituent mixing ratio (kg/kg moist or dry air depending on type) + end type physics_state + + public :: physics_state + public :: allocate_physics_state + +contains + + subroutine allocate_physics_state(cols, levels, constituents, lbnd_slev, ubnd_slev, state) + integer, intent(in) :: cols + integer, intent(in) :: levels + integer, intent(in) :: constituents + integer, intent(in) :: lbnd_slev, ubnd_slev + type(physics_state), intent(out) :: state + + if (allocated(state%ps)) then + deallocate(state%ps) + end if + allocate(state%ps(cols)) + if (allocated(state%u)) then + deallocate(state%u) + end if + allocate(state%u(cols, levels)) + if (allocated(state%v)) then + deallocate(state%v) + end if + allocate(state%v(cols, levels)) + if (allocated(state%pmid)) then + deallocate(state%pmid) + end if + allocate(state%pmid(cols, levels)) + if (associated(state%q)) then + nullify(state%q) + end if + allocate(state%q(cols, levels, constituents)) + if (allocated(state%soil_levs)) then + deallocate(state%soil_levs) + end if + allocate(state%soil_levs(lbnd_slev:ubnd_slev)) + + end subroutine allocate_physics_state +end module test_host_data diff --git a/end-to-end-tests/capgen_ng/test_host_data.meta b/end-to-end-tests/capgen_ng/test_host_data.meta new file mode 100644 index 00000000..aab2ce4a --- /dev/null +++ b/end-to-end-tests/capgen_ng/test_host_data.meta @@ -0,0 +1,53 @@ +[ccpp-table-properties] + name = physics_state + type = ddt +[ccpp-arg-table] + name = physics_state + type = ddt +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) +[ u ] + standard_name = eastward_wind + long_name = Zonal wind + type = real + kind = kind_phys + units = m s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) +[ v ] + standard_name = northward_wind + long_name = Meridional wind + type = real + kind = kind_phys + units = m s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) +[ pmid ] + standard_name = air_pressure + long_name = Midpoint air pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension, vertical_layer_dimension) +[ soil_levs ] + standard_name = soil_levels + long_name = soil levels + units = cm + dimensions = (lower_bound_of_vertical_dimension_of_soil:upper_bound_of_vertical_dimension_of_soil) + type = real + kind = kind_phys +[ q ] + standard_name = constituent_mixing_ratio + type = real + kind = kind_phys + units = kg kg-1 moist or dry air depending on type + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) +[ q(:,:,index_of_water_vapor_specific_HUMidity) ] + standard_name = water_vapor_specific_humidity + type = real + kind = kind_phys + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + active = (index_of_water_vapor_specific_humidity > 0) diff --git a/end-to-end-tests/capgen_ng/test_host_mod.F90 b/end-to-end-tests/capgen_ng/test_host_mod.F90 new file mode 100644 index 00000000..48ee959b --- /dev/null +++ b/end-to-end-tests/capgen_ng/test_host_mod.F90 @@ -0,0 +1,164 @@ +module test_host_mod + + use ccpp_kinds, only: kind_phys + use test_host_data, only: physics_state, & + allocate_physics_state + + implicit none + public + + !> \section arg_table_test_host_mod Argument Table + !! \htmlinclude arg_table_test_host_host.html + !! + integer, parameter :: ncols = 10 + integer, parameter :: pver = 5 + integer, parameter :: pverp = 6 + integer, parameter :: pcnst = 2 + integer, parameter :: slevs = 4 + integer, parameter :: slev_lbound = -3 + integer, parameter :: slev_ubound = 0 + integer, parameter :: diagdimstart = 2 + integer, parameter :: index_qv = 1 + logical, parameter :: config_var = .true. + real(kind=kind_phys), allocatable :: temp_midpoints(:, :) + real(kind=kind_phys) :: temp_interfaces(ncols, pverp) + real(kind=kind_phys) :: temp_diag(ncols, 6) + real(kind=kind_phys) :: coeffs(ncols) + real(kind=kind_phys) :: var_array(ncols, 2, 4, 6) + real(kind=kind_phys), dimension(diagdimstart:ncols, diagdimstart:pver) :: & + diag1, & + diag2 + real(kind=kind_phys) :: dt + real(kind=kind_phys), parameter :: temp_inc = 0.05_kind_phys + type(physics_state) :: phys_state + integer :: num_model_times = -1 + integer, allocatable :: model_times(:) + + integer, parameter :: num_time_steps = 2 + real(kind=kind_phys), parameter :: tolerance = 1.0e-13_kind_phys + real(kind=kind_phys) :: tint_save(ncols, pverp) + + public :: init_data + public :: compare_data + public :: check_model_times + +contains + + subroutine init_data() + + integer :: col + integer :: lev + integer :: cind + integer :: offsize + + ! Allocate and initialize temperature + allocate(temp_midpoints(ncols, pver)) + temp_midpoints = 0.0_kind_phys + cind = 1 + do lev = 1, pverp + offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) + do col = 1, ncols + temp_interfaces(col, lev) = real(offsize + col, kind=kind_phys) + tint_save(col, lev) = temp_interfaces(col, lev) + end do + end do + ! Allocate and initialize state + call allocate_physics_state(ncols, pver, pcnst, slev_lbound, slev_ubound, phys_state) + do cind = 1, pcnst + do lev = 1, pver + offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) + do col = 1, ncols + phys_state%q(col, lev, cind) = real(offsize + col, kind=kind_phys) + end do + end do + end do + + end subroutine init_data + + logical function check_model_times() + + check_model_times = (num_model_times > 0) + if (check_model_times) then + check_model_times = (size(model_times) == num_model_times) + if (.not. check_model_times) then + write(6, '(2(a,i0))') 'model_times size mismatch, ', & + size(model_times), ' should be ', num_model_times + end if + else + write(6, '(a,i0,a)') 'num_model_times mismatch, ', num_model_times, & + ' should be greater than zero' + end if + + end function check_model_times + + logical function compare_data() + + integer :: col + integer :: lev + integer :: cind + integer :: offsize + logical :: need_header + real(kind=kind_phys) :: avg + integer, parameter :: cincrements(pcnst) = (/ 1, 0 /) + real(kind=kind_phys) :: total_test + real(kind=kind_phys), parameter :: total_ref = 6730.0_kind_phys + + compare_data = .true. + + total_test = 0.0_kind_phys + need_header = .true. + do lev = 1, pver + do col = 1, ncols + avg = (tint_save(col, lev) + tint_save(col, lev + 1)) + avg = 1.0_kind_phys + (avg / 2.0_kind_phys) + avg = avg + (temp_inc * num_time_steps) + total_test = total_test + avg + if (abs((temp_midpoints(col, lev) - avg) / avg) > tolerance) then + if (need_header) then + write(6, '(" COL LEV T MIDPOINTS EXPECTED")') + need_header = .false. + end if + write(6, '(2i5,2(3x,es15.7))') col, lev, & + temp_midpoints(col, lev), avg + compare_data = .false. + end if + end do + end do + ! Check constituents + need_header = .true. + do cind = 1, pcnst + do lev = 1, pver + offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) + do col = 1, ncols + avg = real(offsize + col + (cincrements(cind) * num_time_steps), & + kind=kind_phys) + total_test = total_test + avg + if (abs((phys_state%q(col, lev, cind) - avg) / avg) > & + tolerance) then + if (need_header) then + write(6, '(2(2x,a),3x,a,10x,a,14x,a)') & + 'COL', 'LEV', 'C#', 'Q', 'EXPECTED' + need_header = .false. + end if + write(6, '(3i5,2(3x,es15.7))') col, lev, cind, & + phys_state%q(col, lev, cind), avg + compare_data = .false. + end if + end do + end do + end do + if (abs((total_test - total_ref) / total_ref) > tolerance) then + write(6, '(a,e12.4)') 'TOTAL REFERENCE: ', total_ref + write(6, '(a,e12.4)') 'TOTAL TEST: ', total_test + write(6, '(2(a,e12.4))') 'REL.DIFF > TOLERANCE:', & + abs((total_test - total_ref) / total_ref), ' >', tolerance + compare_data = .false. + else + write(0, '(a,e12.4)') 'TOTAL REFERENCE: ', total_ref + write(0, '(a,e12.4)') 'TOTAL TEST: ', total_test + write(0, '(2(a,e12.4))') 'REL.DIFF < TOLERANCE:', & + abs((total_test - total_ref) / total_ref), ' <', tolerance + end if + end function compare_data + +end module test_host_mod diff --git a/end-to-end-tests/capgen_ng/test_host_mod.meta b/end-to-end-tests/capgen_ng/test_host_mod.meta new file mode 100644 index 00000000..b92ce89a --- /dev/null +++ b/end-to-end-tests/capgen_ng/test_host_mod.meta @@ -0,0 +1,133 @@ +[ccpp-table-properties] + name = test_host_mod + type = host +[ccpp-arg-table] + name = test_host_mod + type = host +[ index_qv ] + standard_name = index_of_water_vapor_specific_HUMidity + units = index + type = integer + protected = True + dimensions = () +[ config_var ] + standard_name = configuration_variable + units = none + type = logical + protected = True + dimensions = () +[ ncols] + standard_name = horizontal_dimension + units = count + type = integer + protected = True + dimensions = () +[ pver ] + standard_name = vertical_layer_dimension + units = count + type = integer + protected = True + dimensions = () +[ pverP ] + standard_name = vertical_interface_dimension + type = integer + units = count + protected = True + dimensions = () +[ pcnst ] + standard_name = number_of_tracers + type = integer + units = count + protected = True + dimensions = () +[ slevs ] + standard_name = vertical_dimension_of_soil + type = integer + units = count + protected = True + dimensions = () +[ slev_lbound] + standard_name = lower_bound_of_vertical_dimension_of_soil + type = integer + units = count + protected = True + dimensions = () +[ slev_ubound] + standard_name = upper_bound_of_vertical_dimension_of_soil + type = integer + units = count + protected = True + dimensions = () +[ DiagDimStart ] + standard_name = first_index_of_diag_fields + type = integer + units = count + protected = True + dimensions = () +[ temp_midpoints ] + standard_name = potential_temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys +[ temp_interfaces ] + standard_name = potential_temperature_at_interface + units = K + dimensions = (horizontal_dimension, vertical_interface_dimension) + type = real | kind = kind_phys +[ temp_diag ] + standard_name = temperature_at_diagnostic_levels + units = K + dimensions = (horizontal_dimension, 6) + type = real | kind = kind_phys +[ diag1 ] + standard_name = diagnostic_stuff_type_1 + long_name = This is just a test field + units = K + dimensions = (first_index_of_diag_fields:horizontal_dimension, first_index_of_diag_fields:vertical_layer_dimension) + type = real | kind = kind_phys +[ diag2 ] + standard_name = diagnostic_stuff_type_2 + long_name = This is just a test field + units = K + dimensions = (first_index_of_diag_fields: horizontal_dimension, first_index_of_diag_fields :vertical_layer_dimension) + type = real | kind = kind_phys +[ dt ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real | kind = kind_phys +[ temp_inc ] + standard_name = potential_temperature_increment + long_name = Per time step potential temperature increment + units = K + dimensions = () + type = real | kind = kind_phys +[ phys_state ] + standard_name = physics_state_derived_type + long_name = Physics State DDT + type = physics_state + dimensions = () +[ num_model_times ] + standard_name = number_of_model_times + type = integer + units = count + dimensions = () +[ model_times ] + standard_name = model_times + units = seconds + dimensions = (number_of_model_times) + type = integer + allocatable = True +[ coeffs ] + standard_name = coefficients_for_interpolation + long_name = coefficients for interpolation + units = none + dimensions = (horizontal_dimension) + type = real | kind = kind_phys +[ var_array ] + standard_name = array_variable_for_testing + long_name = array variable for testing + units = none + dimensions = (horizontal_dimension,2,4,6) + type = real | kind = kind_phys diff --git a/end-to-end-tests/chunked_data/CMakeLists.txt b/end-to-end-tests/chunked_data/CMakeLists.txt new file mode 100644 index 00000000..131b5239 --- /dev/null +++ b/end-to-end-tests/chunked_data/CMakeLists.txt @@ -0,0 +1,64 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "chunked_data_scheme") +set(HOST_FILES "data" "main") +set(SUITE_FILES "suite_chunked_data_suite.xml") +set(HOST "test_host") + +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES}) + +# Run ccpp_capgen_ng +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_chunked_data.x + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} +) +target_link_libraries(test_chunked_data.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_chunked_data.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_chunked_data.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_chunked_data + COMMAND test_chunked_data.x) diff --git a/end-to-end-tests/chunked_data/chunked_data_scheme.F90 b/end-to-end-tests/chunked_data/chunked_data_scheme.F90 new file mode 100644 index 00000000..577ee01c --- /dev/null +++ b/end-to-end-tests/chunked_data/chunked_data_scheme.F90 @@ -0,0 +1,126 @@ +!>\file chunked_data_scheme.F90 +!! This file contains a chunked_data_scheme CCPP scheme that does nothing +!! except requesting the minimum, mandatory variables. + +module chunked_data_scheme + + use, intrinsic :: iso_fortran_env, only: error_unit + implicit none + + private + public :: chunked_data_scheme_init, & + chunked_data_scheme_timestep_init, & + chunked_data_scheme_run, & + chunked_data_scheme_timestep_final, & + chunked_data_scheme_final + + ! This is for unit testing only + integer, parameter, dimension(4) :: data_array_sizes = (/6, 6, 6, 3/) + +contains + + !! \section arg_table_chunked_data_scheme_init Argument Table + !! \htmlinclude chunked_data_scheme_init.html + !! + subroutine chunked_data_scheme_init(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(error_unit, '(a,i3)') 'In chunked_data_scheme_init: checking size of data array to be', sum(data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& + data_array) + errflg = 1 + return + end if + end subroutine chunked_data_scheme_init + + !! \section arg_table_chunked_data_scheme_timestep_init Argument Table + !! \htmlinclude chunked_data_scheme_timestep_init.html + !! + subroutine chunked_data_scheme_timestep_init(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(error_unit, '(a,i3)') 'In chunked_data_scheme_timestep_init: checking size of data array to be', sum(& + data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), " but got ", size(& + data_array) + errflg = 1 + return + end if + end subroutine chunked_data_scheme_timestep_init + + !! \section arg_table_chunked_data_scheme_run Argument Table + !! \htmlinclude chunked_data_scheme_run.html + !! + subroutine chunked_data_scheme_run(nchunk, nchunks, data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: nchunk, nchunks + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(error_unit, '(2(a,i3))') 'In chunked_data_scheme_run: checking size of data array for chunk', & + nchunk, '/', nchunks, ' to be', data_array_sizes(nchunk) + if (size(data_array)/=data_array_sizes(nchunk)) then + write(errmsg, '(a,i4)') "Error in chunked_data_scheme_run, expected size(data_array)==6, got ", size(data_array) + errflg = 1 + return + end if + end subroutine chunked_data_scheme_run + + !! \section arg_table_chunked_data_scheme_timestep_final Argument Table + !! \htmlinclude chunked_data_scheme_timestep_final.html + !! + subroutine chunked_data_scheme_timestep_final(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(error_unit, '(a,i3)') 'In chunked_data_scheme_timestep_final: checking size of data array to be', sum(& + data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& + data_array) + errflg = 1 + return + end if + end subroutine chunked_data_scheme_timestep_final + + !! \section arg_table_chunked_data_scheme_final Argument Table + !! \htmlinclude chunked_data_scheme_final.html + !! + subroutine chunked_data_scheme_final(data_array, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: data_array(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Check size of data array + write(error_unit, '(a,i3)') 'In chunked_data_scheme_final: checking size of data array to be', sum(& + data_array_sizes) + if (size(data_array)/=sum(data_array_sizes)) then + write(errmsg, '(2(a,i3))') "Error, expected size(data_array)==", sum(data_array_sizes), "but got ", size(& + data_array) + errflg = 1 + return + end if + end subroutine chunked_data_scheme_final + +end module chunked_data_scheme diff --git a/end-to-end-tests/chunked_data/chunked_data_scheme.meta b/end-to-end-tests/chunked_data/chunked_data_scheme.meta new file mode 100644 index 00000000..a74bed43 --- /dev/null +++ b/end-to-end-tests/chunked_data/chunked_data_scheme.meta @@ -0,0 +1,154 @@ +[ccpp-table-properties] + name = chunked_data_scheme + type = scheme + dependencies = + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_init + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_timestep_init + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_run + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[nchunk] + standard_name = ccpp_chunk_number + long_name = number of chunk for chunked arrays in CCPP + units = index + dimensions = () + type = integer + intent = in +[nchunks] + standard_name = ccpp_chunk_extent + long_name = number of chunks of array data used in run phase + units = count + dimensions = () + type = integer + intent = in +[data_array] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_timestep_final + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_final + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[data_array] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + diff --git a/end-to-end-tests/chunked_data/data.F90 b/end-to-end-tests/chunked_data/data.F90 new file mode 100644 index 00000000..737f46c7 --- /dev/null +++ b/end-to-end-tests/chunked_data/data.F90 @@ -0,0 +1,47 @@ +module data + + !! \section arg_table_dATa Argument Table + !! \htmlinclude datA.Html + !! + + implicit none + + private + + public nchunks, nchunk, chunksize, chunk_begin, chunk_end, ncols + public chunked_data_type, chunked_data_instance + + integer, parameter :: nchunks = 4 + integer :: nchunk + + integer, parameter, dimension(nchunks) :: chunksize = (/6, 6, 6, 3/) + integer, parameter, dimension(nchunks) :: chunk_begin = (/1, 7, 13, 19/) + integer, parameter, dimension(nchunks) :: chunk_end = (/6, 12, 18, 21/) + integer, parameter :: ncols = sum(chunksize) + + !! \section arg_table_cHuNkEd_dATa_TYPe + !! \htmlinclude CHuNKed_Data_tYpe.hTMl + !! + type chunked_data_type + integer, dimension(:), allocatable :: array_data + contains + procedure :: create => chunked_data_create + procedure :: destroy => chunked_data_destroy + end type chunked_data_type + + type(chunked_data_type) :: chunked_data_instance + +contains + + subroutine chunked_data_create(chunked_data_instance, ncol) + class(chunked_data_type), intent(inout) :: chunked_data_instance + integer, intent(in) :: ncol + allocate(chunked_data_instance%array_data(ncol)) + end subroutine chunked_data_create + + subroutine chunked_data_destroy(chunked_data_instance) + class(chunked_data_type), intent(inout) :: chunked_data_instance + deallocate(chunked_data_instance%array_data) + end subroutine chunked_data_destroy + +end module data diff --git a/end-to-end-tests/chunked_data/data.meta b/end-to-end-tests/chunked_data/data.meta new file mode 100644 index 00000000..38f8bfc2 --- /dev/null +++ b/end-to-end-tests/chunked_data/data.meta @@ -0,0 +1,45 @@ +[ccpp-table-properties] + name = chunked_data_type + type = ddt + dependencies = +[ccpp-arg-table] + name = chunked_data_type + type = ddt +[array_data] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (ccpp_constant_one:horizontal_dimension) + type = integer + +[ccpp-table-properties] + name = data + type = host + dependencies = +[ccpp-arg-table] + name = data + type = host +[ncols] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer +[nchunk] + standard_name = ccpp_chunk_number + long_name = number of current chunk + units = index + dimensions = () + type = integer +[nchunks] + standard_name = ccpp_chunk_extent + long_name = number of chunks of array data used in run phase + units = count + dimensions = () + type = integer +[chunked_data_instance] + standard_name = chunked_data_type_instance + long_name = instance of derived data type chunked_data_type + units = DDT + dimensions = () + type = chunked_data_type diff --git a/end-to-end-tests/chunked_data/main.F90 b/end-to-end-tests/chunked_data/main.F90 new file mode 100644 index 00000000..5bea870d --- /dev/null +++ b/end-to-end-tests/chunked_data/main.F90 @@ -0,0 +1,124 @@ +program test_chunked_data + + use, intrinsic :: iso_fortran_env, only: error_unit + + use data, only: nchunks, & + chunksize, & + chunk_begin, & + chunk_end, & + ncols, & + nchunk + use data, only: chunked_data_type, & + chunked_data_instance + + use ccpp_static_api, only: ccpp_register, & + ccpp_init, & + ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_final, & + ccpp_physics_final, & + ccpp_final + + implicit none + + character(len=*), parameter :: ccpp_suite = 'chunked_data_suite' + integer :: ic, ierr + integer :: lb, ub + integer :: errflg + character(len=512) :: errmsg + + call chunked_data_instance%create(ncols) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP register step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_register(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_register:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_init(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_init(lb=1, ub=ncols, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !cdata => ccpp_data_domain + call ccpp_physics_timestep_init(lb=1, ub=ncols, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics run step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do nchunk = 1, nchunks + lb=chunk_begin(nchunk) + ub=chunk_end(nchunk) + call ccpp_physics_run(lb=lb, ub=ub, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a,i3,a)') "An error occurred in ccpp_physics_run for chunk", nchunk, ":" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !cdata => ccpp_data_domain + call ccpp_physics_timestep_final(lb=1, ub=ncols, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_finalize:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + !cdata => ccpp_data_domain + call ccpp_physics_final(lb=1, ub=ncols, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_finalize:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP final step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_final(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + call chunked_data_instance%destroy() + +end program test_chunked_data diff --git a/end-to-end-tests/chunked_data/main.meta b/end-to-end-tests/chunked_data/main.meta new file mode 100644 index 00000000..b90de81e --- /dev/null +++ b/end-to-end-tests/chunked_data/main.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer diff --git a/end-to-end-tests/chunked_data/suite_chunked_data_suite.xml b/end-to-end-tests/chunked_data/suite_chunked_data_suite.xml new file mode 100644 index 00000000..32159fae --- /dev/null +++ b/end-to-end-tests/chunked_data/suite_chunked_data_suite.xml @@ -0,0 +1,9 @@ + + + + + + chunked_data_scheme + + + diff --git a/end-to-end-tests/cmake/ccpp_capgen.cmake b/end-to-end-tests/cmake/ccpp_capgen.cmake new file mode 100644 index 00000000..a7a1e719 --- /dev/null +++ b/end-to-end-tests/cmake/ccpp_capgen.cmake @@ -0,0 +1,205 @@ +# CMake wrapper for ccpp_validator.py +# +# SOURCE_FILES - CMake list of Fortran source files +# METADATA_FILES - CMake list of corresponding metadata files +function(ccpp_validator) + set(optionalArgs) + set(oneValueArgs VERBOSITY) + set(multi_value_keywords SOURCE_FILES METADATA_FILES) + cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN}) + + # Error if script file not found. + set(CCPP_VALIDATOR_CMD_LIST "${CMAKE_SOURCE_DIR}/../capgen-ng/ccpp_validator.py") + if(NOT EXISTS ${CCPP_VALIDATOR_CMD_LIST}) + message(FATAL_ERROR "function(ccpp_validator): Could not find ccpp_validator.py. Looked for ${CCPP_VALIDATOR_CMD_LIST}.") + endif() + + # Interpret parsed arguments + if(NOT DEFINED arg_SOURCE_FILES) + message(FATAL_ERROR "function(ccpp_capgen): SOURCE_FILES not set.") + endif() + list(JOIN arg_SOURCE_FILES "," SOURCE_FILES_SEPARATED) + list(APPEND CCPP_VALIDATOR_CMD_LIST "--source-files" "${SOURCE_FILES_SEPARATED}") + + if(NOT DEFINED arg_METADATA_FILES) + message(FATAL_ERROR "function(ccpp_capgen): METADATA_FILES not set.") + endif() + list(JOIN arg_METADATA_FILES "," METADATA_FILES_SEPARATED) + list(APPEND CCPP_VALIDATOR_CMD_LIST "--scheme-files" "${METADATA_FILES_SEPARATED}") + + if(DEFINED arg_VERBOSITY) + string(REPEAT "--verbose " ${arg_VERBOSITY} VERBOSE_PARAMS_SEPARATED) + separate_arguments(VERBOSE_PARAMS UNIX_COMMAND "${VERBOSE_PARAMS_SEPARATED}") + list(APPEND CCPP_VALIDATOR_CMD_LIST ${VERBOSE_PARAMS}) + endif() + + message(STATUS "Running ccpp_validator.py from ${CMAKE_CURRENT_SOURCE_DIR}") + + unset(VALIDATOR_OUT) + execute_process(COMMAND ${CCPP_VALIDATOR_CMD_LIST} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE VALIDATOR_OUT + ERROR_VARIABLE VALIDATOR_OUT + RESULT_VARIABLE RES + COMMAND_ECHO STDOUT) + if(RES EQUAL 0) + message(STATUS "ccpp-validator stdout: ${VALIDATOR_OUT}") + else() + message(STATUS "${CCPP_VALIDATOR_CMD_LIST} FAILED: result = ${RES}") + message(STATUS "ccpp-validator stdout: ${VALIDATOR_OUT}") + message(FATAL_ERROR "Validation of source files failed, abort.") + endif() + +endfunction() + + +# CMake wrapper for ccpp_capgen_ng.py +# +# CAPGEN_EXPECT_THROW_ERROR - ON/OFF (Default: OFF) - Scans ccpp_capgen.py log for error string and errors if not found. +# HOST_NAME - String name of host +# OUTPUT_ROOT - String path to put generated caps +# VERBOSITY - Number of --verbose flags to pass to capgen +# HOSTFILES - CMake list of host metadata filenames +# SCHEMEFILES - CMake list of scheme metadata files +# SUITES - CMake list of suite xml files +function(ccpp_capgen) + set(optionalArgs CAPGEN_EXPECT_THROW_ERROR) + set(oneValueArgs HOST_NAME OUTPUT_ROOT VERBOSITY KIND_SPECS) + set(multi_value_keywords HOSTFILES SCHEMEFILES SUITES) + + cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN}) + + # Error if script file not found. + set(CCPP_CAPGEN_CMD_LIST "${CMAKE_SOURCE_DIR}/../capgen-ng/ccpp_capgen_ng.py") + if(NOT EXISTS ${CCPP_CAPGEN_CMD_LIST}) + message(FATAL_ERROR "function(ccpp_capgen): Could not find ccpp_capgen_ng.py. Looked for ${CCPP_CAPGEN_CMD_LIST}.") + endif() + + # Interpret parsed arguments + if(NOT DEFINED arg_HOSTFILES) + message(FATAL_ERROR "function(ccpp_capgen): HOSTFILES not set.") + endif() + list(JOIN arg_HOSTFILES "," HOSTFILES_SEPARATED) + list(APPEND CCPP_CAPGEN_CMD_LIST "--host-files" "${HOSTFILES_SEPARATED}") + + if(NOT DEFINED arg_SCHEMEFILES) + message(FATAL_ERROR "function(ccpp_capgen): SCHEMEFILES not set.") + endif() + list(JOIN arg_SCHEMEFILES "," SCHEMEFILES_SEPARATED) + list(APPEND CCPP_CAPGEN_CMD_LIST "--scheme-files" "${SCHEMEFILES_SEPARATED}") + + if(NOT DEFINED arg_SUITES) + message(FATAL_ERROR "function(ccpp_capgen): SUITES not set.") + endif() + list(JOIN arg_SUITES "," SUITES_SEPARATED) + list(APPEND CCPP_CAPGEN_CMD_LIST "--suites" "${SUITES_SEPARATED}") + + if(NOT DEFINED arg_HOST_NAME) + message(FATAL_ERROR "function(ccpp_capgen): HOSTNAME not set.") + endif() + list(APPEND CCPP_CAPGEN_CMD_LIST "--host-name" "${arg_HOST_NAME}") + + if(NOT DEFINED arg_OUTPUT_ROOT) + message(FATAL_ERROR "function(ccpp_capgen): OUTPUT_ROOT not set.") + endif() + #file(MAKE_DIRECTORY "${arg_OUTPUT_ROOT}") + list(APPEND CCPP_CAPGEN_CMD_LIST "--output-root" "${arg_OUTPUT_ROOT}") + + if(DEFINED arg_VERBOSITY) + string(REPEAT "--verbose " ${arg_VERBOSITY} VERBOSE_PARAMS_SEPARATED) + separate_arguments(VERBOSE_PARAMS UNIX_COMMAND "${VERBOSE_PARAMS_SEPARATED}") + list(APPEND CCPP_CAPGEN_CMD_LIST ${VERBOSE_PARAMS}) + endif() + + if(DEFINED arg_KIND_SPECS) + string(REPLACE "," ";" KIND_SPEC_LIST "${arg_KIND_SPECS}") + set(KIND_ARGS "") # start empty + foreach(pair IN LISTS KIND_SPEC_LIST) + # Append each pair prefixed with --kind-type and quoted. + # The surrounding double‑quotes are added explicitly so the + # resulting string contains them. + set(KIND_ARGS "${KIND_ARGS}--kind-type \"${pair}\"") + string(STRIP "${KIND_ARGS}" KIND_ARGS) + endforeach() + list(APPEND CCPP_CAPGEN_CMD_LIST ${KIND_SPEC_PARAMS}) + endif() + + message(STATUS "Running ccpp_capgen.py from ${CMAKE_CURRENT_SOURCE_DIR}") + + # Unset CAPGEN_OUT to prevent incorrect output on subsequent ccpp_capgen(...) calls + unset(CAPGEN_OUT) + execute_process(COMMAND ${CCPP_CAPGEN_CMD_LIST} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE CAPGEN_OUT + ERROR_VARIABLE CAPGEN_OUT + RESULT_VARIABLE RES + COMMAND_ECHO STDOUT) + + message(STATUS "ccpp-capgen stdout: ${CAPGEN_OUT}") + + if(arg_CAPGEN_EXPECT_THROW_ERROR) + # Determine if the process succeeded but had an expected string in the process log. + string(FIND "${CAPGEN_OUT}" "Variables of type ccpp_constituent_properties_t only allowed in register phase" ERROR_INDEX) + + if (ERROR_INDEX GREATER -1) + message(STATUS "Capgen build produces expected error message.") + else() + message(FATAL_ERROR "CCPP cap generation did not generate expected error. Expected 'Variables of type constituent_properties_t only allowed in register phase.") + endif() + else() + if(RES EQUAL 0) + message(STATUS "ccpp-capgen completed successfully") + else() + message(FATAL_ERROR "CCPP cap generation FAILED: result = ${RES}") + endif() + endif() +endfunction() + + +# CMake wrapper for ccpp_datafile.py +# +# DATATABLE - Path to generated datatable.xml file +# REPORT_NAME - String report name to get list of generated files form capgen (typically --ccpp-files) +function(ccpp_datafile) + set(mandatoryArgs DATATABLE REPORT_NAME) + cmake_parse_arguments(arg "" "${mandatoryArgs}" "" ${ARGN}) + + set(CCPP_DATAFILE_CMD "${CMAKE_SOURCE_DIR}/../capgen-ng/ccpp_datafile.py") + + if(NOT EXISTS ${CCPP_DATAFILE_CMD}) + message(FATAL_ERROR "function(ccpp_datafile): Could not find ccpp_datafile.py. Looked for ${CCPP_DATAFILE_CMD}.") + endif() + + if(NOT DEFINED arg_REPORT_NAME) + message(FATAL_ERROR "function(ccpp_datafile): REPORT_NAME not set.") + endif() + list(APPEND CCPP_DATAFILE_CMD "${arg_REPORT_NAME}") + + if(NOT DEFINED arg_DATATABLE) + message(FATAL_ERROR "function(ccpp_datafile): DATATABLE not set.") + endif() + list(APPEND CCPP_DATAFILE_CMD "${arg_DATATABLE}") + + message(STATUS "Running ccpp_datafile from ${CMAKE_CURRENT_SOURCE_DIR}") + + # Unset CCPP_FILES to prevent incorrect output on subsequent ccpp_datafile(...) calls + unset(CCPP_FILES) + execute_process(COMMAND ${CCPP_DATAFILE_CMD} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE CCPP_FILES + RESULT_VARIABLE RES + OUTPUT_STRIP_TRAILING_WHITESPACE + ERROR_STRIP_TRAILING_WHITESPACE + COMMAND_ECHO STDOUT) + #message(STATUS "CCPP_FILES = ${CCPP_FILES}") + if(RES EQUAL 0) + message(STATUS "CCPP files retrieved") + else() + message(FATAL_ERROR "CCPP file retrieval FAILED: result = ${RES}") + endif() + if(CCPP_FILES) + # Convert "," separated list from python back to ";" separated list for CMake + string(REPLACE "," ";" CCPP_FILES ${CCPP_FILES}) + endif() + set(CCPP_FILES "${CCPP_FILES}" PARENT_SCOPE) +endfunction() diff --git a/end-to-end-tests/ddthost/CMakeLists.txt b/end-to-end-tests/ddthost/CMakeLists.txt new file mode 100644 index 00000000..a99274bc --- /dev/null +++ b/end-to-end-tests/ddthost/CMakeLists.txt @@ -0,0 +1,71 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "environ_conditions" "setup_coeffs" "temp_adjust" "temp_calc_adjust" "temp_set" "make_ddt") +set(HOST_FILES "host_ccpp_ddt" "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "ddt_suite.xml" "temp_suite.xml") +set(HOST "test_host") + +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES}) + +# Run ccpp_capgen_ng +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + KIND_TYPES ${KIND_TYPES} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_ddthost.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_ddt_host_integration.F90 +) +target_link_libraries(test_ddthost.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_ddthost.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_ddthost.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_ddthost + COMMAND test_ddthost.x) diff --git a/end-to-end-tests/ddthost/README.md b/end-to-end-tests/ddthost/README.md new file mode 100644 index 00000000..9c90d754 --- /dev/null +++ b/end-to-end-tests/ddthost/README.md @@ -0,0 +1,16 @@ +# DDT Host Test + +Contains tests to exercise more DDT functionality: +- Passing around and modifying a DDT +- Making DDT in host model & using it in CCPP-ized physics code + +## Building/Running + +To explicitly build/run the ddt test host, run: + +```bash +$ cmake --fresh -S -B -DCCPP_RUN_DDT_HOST_TEST=ON +$ cd +$ make +$ ctest +``` diff --git a/end-to-end-tests/ddthost/ddt_suite.xml b/end-to-end-tests/ddthost/ddt_suite.xml new file mode 100644 index 00000000..749bb3bc --- /dev/null +++ b/end-to-end-tests/ddthost/ddt_suite.xml @@ -0,0 +1,8 @@ + + + + + make_ddt + environ_conditions + + diff --git a/end-to-end-tests/ddthost/ddthost_test_reports.py b/end-to-end-tests/ddthost/ddthost_test_reports.py new file mode 100644 index 00000000..612cbbbf --- /dev/null +++ b/end-to-end-tests/ddthost/ddthost_test_reports.py @@ -0,0 +1,139 @@ +#! /usr/bin/env python3 +""" +----------------------------------------------------------------------- + Description: Test DDT host database report python interface + + Assumptions: + + Command line arguments: build_dir database_filepath + + Usage: python test_reports +----------------------------------------------------------------------- +""" +import os +import unittest + +from test_stub import BaseTests + +_BUILD_DIR = os.path.join(os.path.abspath(os.environ['BUILD_DIR']), "test", "ddthost_test") +_DATABASE = os.path.abspath(os.path.join(_BUILD_DIR, "ccpp", "datatable.xml")) + +_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +_FRAMEWORK_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, os.pardir)) +_SCRIPTS_DIR = os.path.join(_FRAMEWORK_DIR, "scripts") +_SRC_DIR = os.path.join(_FRAMEWORK_DIR, "src") + + +# Check data +_HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] +_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_ddt_suite_cap.F90"), + os.path.join(_BUILD_DIR, "ccpp", "ccpp_temp_suite_cap.F90")] +_UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), + os.path.join(_SRC_DIR, "ccpp_constituent_prop_mod.F90"), + os.path.join(_SRC_DIR, "ccpp_scheme_utils.F90"), + os.path.join(_SRC_DIR, "ccpp_hashable.F90"), + os.path.join(_SRC_DIR, "ccpp_hash_table.F90")] +_CCPP_FILES = _UTILITY_FILES + \ + [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90"), + os.path.join(_BUILD_DIR, "ccpp", "ccpp_ddt_suite_cap.F90"), + os.path.join(_BUILD_DIR, "ccpp", "ccpp_temp_suite_cap.F90")] +_DEPENDENCIES = [os.path.join(_TEST_DIR, "adjust", "qux.F90"), + os.path.join(_TEST_DIR, "bar.F90"), + os.path.join(_TEST_DIR, "foo.F90")] +_PROCESS_LIST = ["setter=temp_set", "adjusting=temp_calc_adjust"] +_MODULE_LIST = ["environ_conditions", "make_ddt", "setup_coeffs", "temp_adjust", + "temp_calc_adjust", "temp_set"] +_SUITE_LIST = ["ddt_suite", "temp_suite"] +_INPUT_VARS_DDT = ["model_times", "number_of_model_times", + "horizontal_loop_begin", "horizontal_loop_end", + "surface_air_pressure", "horizontal_dimension", + "host_standard_ccpp_type"] +_OUTPUT_VARS_DDT = ["ccpp_error_code", "ccpp_error_message", "model_times", + "number_of_model_times", "surface_air_pressure"] +_REQUIRED_VARS_DDT = _INPUT_VARS_DDT + _OUTPUT_VARS_DDT +_PROT_VARS_TEMP = ["horizontal_loop_begin", "horizontal_loop_end", + "horizontal_dimension", "vertical_layer_dimension", + "number_of_tracers", + # Added for --debug + "index_of_water_vapor_specific_humidity", + "vertical_interface_dimension"] +_REQUIRED_VARS_TEMP = ["ccpp_error_code", "ccpp_error_message", + "potential_temperature", + "potential_temperature_at_interface", + "coefficients_for_interpolation", + "potential_temperature_increment", + "surface_air_pressure", "time_step_for_physics", + "water_vapor_specific_humidity"] +_INPUT_VARS_TEMP = ["potential_temperature", + "potential_temperature_at_interface", + "coefficients_for_interpolation", + "potential_temperature_increment", + "surface_air_pressure", "time_step_for_physics", + "water_vapor_specific_humidity"] +_OUTPUT_VARS_TEMP = ["ccpp_error_code", "ccpp_error_message", + "potential_temperature", + "potential_temperature_at_interface", + "coefficients_for_interpolation", + "surface_air_pressure", "water_vapor_specific_humidity"] + +class TestDdtHostDataTables(unittest.TestCase, BaseTests.TestHostDataTables): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + + +class CommandLineDdtHostDatafileRequiredFiles(unittest.TestCase, BaseTests.TestHostCommandLineDataFiles): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" + + +class TestDdtSuite(unittest.TestCase, BaseTests.TestSuite): + database = _DATABASE + required_vars = _REQUIRED_VARS_DDT + input_vars = _INPUT_VARS_DDT + output_vars = _OUTPUT_VARS_DDT + suite_name = "ddt_suite" + + +class CommandLineDdtSuite(unittest.TestCase, BaseTests.TestSuiteCommandLine): + database = _DATABASE + required_vars = _REQUIRED_VARS_DDT + input_vars = _INPUT_VARS_DDT + output_vars = _OUTPUT_VARS_DDT + suite_name = "ddt_suite" + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" + + +class TestDdtTempSuite(unittest.TestCase, BaseTests.TestSuiteExcludeProtected): + database = _DATABASE + required_vars = _REQUIRED_VARS_TEMP + _PROT_VARS_TEMP + input_vars = _INPUT_VARS_TEMP + _PROT_VARS_TEMP + required_vars_excluding_protected = _REQUIRED_VARS_TEMP + input_vars_excluding_protected = _INPUT_VARS_TEMP + output_vars = _OUTPUT_VARS_TEMP + suite_name = "temp_suite" + + +class CommandLineDdtTempSuite(unittest.TestCase, BaseTests.TestSuiteExcludeProtectedCommandLine): + database = _DATABASE + required_vars = _REQUIRED_VARS_TEMP + _PROT_VARS_TEMP + input_vars = _INPUT_VARS_TEMP + _PROT_VARS_TEMP + required_vars_excluding_protected = _REQUIRED_VARS_TEMP + input_vars_excluding_protected = _INPUT_VARS_TEMP + output_vars = _OUTPUT_VARS_TEMP + suite_name = "temp_suite" + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" diff --git a/end-to-end-tests/ddthost/environ_conditions.F90 b/end-to-end-tests/ddthost/environ_conditions.F90 new file mode 100644 index 00000000..fd2d15d7 --- /dev/null +++ b/end-to-end-tests/ddthost/environ_conditions.F90 @@ -0,0 +1,96 @@ +module environ_conditions + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: environ_conditions_init + public :: environ_conditions_run + public :: environ_conditions_final + + integer, parameter :: input_model_times = 3 + integer, parameter :: input_model_values(input_model_times) = (/ 31, 37, 41 /) + +contains + + !> \section arg_table_environ_conditions_run Argument Table + !! \htmlinclude arg_table_environ_conditions_run.html + !! + subroutine environ_conditions_run(psurf, errmsg, errflg) + + ! This routine currently does nothing -- should update values + + real(kind=kind_phys), intent(in) :: psurf(:) + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + end subroutine environ_conditions_run + + !> \section arg_table_environ_conditions_init Argument Table + !! \htmlinclude arg_table_environ_conditions_init.html + !! + subroutine environ_conditions_init(nbox, o3, hno3, ntimes, model_times, & + errmsg, errflg) + + integer, intent(in) :: nbox + real(kind=kind_phys), intent(out) :: o3(:) + real(kind=kind_phys), intent(out) :: hno3(:) + integer, intent(out) :: ntimes + integer, allocatable, intent(out) :: model_times(:) + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: i, j + + errmsg = '' + errflg = 0 + + ! This may be replaced with MusicBox json environmental conditions reader??? + + do i = 1, nbox + o3(i) = real(i, kind_phys) * 1.e-6_kind_phys + hno3(i) = real(i, kind_phys) * 1.e-9_kind_phys + end do + + ntimes = input_model_times + allocate(model_times(ntimes)) + model_times = input_model_values + + end subroutine environ_conditions_init + + !> \section arg_table_environ_conditions_final Argument Table + !! \htmlinclude arg_table_environ_conditions_final.html + !! + subroutine environ_conditions_final(ntimes, model_times, errmsg, errflg) + + integer, intent(in) :: ntimes + integer, intent(in) :: model_times(:) + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine checks the size and values of model_times + if (ntimes /= input_model_times) then + errflg = 1 + write(errmsg, '(2(a,i0))') 'ntimes mismatch, ', ntimes, ' should be ', & + input_model_times + else if (size(model_times) /= input_model_times) then + errflg = 1 + write(errmsg, '(2(a,i0))') 'model_times size mismatch, ', & + size(model_times), ' should be ', input_model_times + else if (any(model_times /= input_model_values)) then + errflg = 1 + write(errmsg, *) 'model_times mismatch, ', & + model_times, ' should be ', input_model_values + else + errmsg = '' + errflg = 0 + end if + + end subroutine environ_conditions_final + +end module environ_conditions diff --git a/end-to-end-tests/ddthost/environ_conditions.meta b/end-to-end-tests/ddthost/environ_conditions.meta new file mode 100644 index 00000000..0d425579 --- /dev/null +++ b/end-to-end-tests/ddthost/environ_conditions.meta @@ -0,0 +1,109 @@ +[ccpp-table-properties] + name = environ_conditions + type = scheme +[ccpp-arg-table] + name = environ_conditions_run + type = scheme +[ psurf ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = environ_conditions_init + type = scheme +[ nbox ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ o3 ] + standard_name = ozone + units = ppmv + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = out +[ hno3 ] + standard_name = nitric_acid + units = ppmv + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = out +[ ntimes ] + standard_name = number_of_model_times + type = integer + units = count + dimensions = () + intent = out +[ model_times ] + standard_name = model_times + units = seconds + dimensions = (number_of_model_times) + type = integer + intent = out + allocatable = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = environ_conditions_final + type = scheme +[ ntimes ] + standard_name = number_of_model_times + type = integer + units = count + dimensions = () + intent = in +[ model_times ] + standard_name = model_times + units = seconds + dimensions = (number_of_model_times) + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/ddthost/host_ccpp_ddt.F90 b/end-to-end-tests/ddthost/host_ccpp_ddt.F90 new file mode 100644 index 00000000..b60c81af --- /dev/null +++ b/end-to-end-tests/ddthost/host_ccpp_ddt.F90 @@ -0,0 +1,16 @@ +module host_ccpp_ddt + + implicit none + private + + !> \section arg_table_ccpp_info_t Argument Table + !! \htmlinclude arg_table_ccpp_info_t.html + !! + type, public :: ccpp_info_t + integer :: col_start ! horizontal_loop_begin + integer :: col_end ! horizontal_loop_end + character(len=512) :: errmsg ! ccpp_error_message + integer :: errflg ! ccpp_error_code + end type ccpp_info_t + +end module host_ccpp_ddt diff --git a/end-to-end-tests/ddthost/host_ccpp_ddt.meta b/end-to-end-tests/ddthost/host_ccpp_ddt.meta new file mode 100644 index 00000000..4129f461 --- /dev/null +++ b/end-to-end-tests/ddthost/host_ccpp_ddt.meta @@ -0,0 +1,31 @@ +[ccpp-table-properties] + name = ccpp_info_t + type = ddt +[ccpp-arg-table] + name = ccpp_info_t + type = ddt +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer diff --git a/end-to-end-tests/ddthost/make_ddt.F90 b/end-to-end-tests/ddthost/make_ddt.F90 new file mode 100644 index 00000000..514827cb --- /dev/null +++ b/end-to-end-tests/ddthost/make_ddt.F90 @@ -0,0 +1,131 @@ +!Hello demonstration parameterization +! + +module make_ddt + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: make_ddt_init + public :: make_ddt_run + public :: make_ddt_timestep_final + public :: vmr_type + + !> \section arg_table_vmr_type Argument Table + !! \htmlinclude arg_table_vmr_type.html + !! + type vmr_type + integer :: nvmr + real(kind=kind_phys), allocatable :: vmr_array(:, :) + end type vmr_type + +contains + + !> \section arg_table_make_ddt_run Argument Table + !! \htmlinclude arg_table_make_ddt_run.html + !! + subroutine make_ddt_run(cols, cole, o3, hno3, vmr, errmsg, errflg) + !---------------------------------------------------------------- + implicit none + !---------------------------------------------------------------- + + ! Dummy arguments + integer, intent(in) :: cols + integer, intent(in) :: cole + real(kind=kind_phys), intent(in) :: o3(:) + real(kind=kind_phys), intent(in) :: hno3(:) + type(vmr_type), intent(inout) :: vmr + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + ! Local variable + integer :: nbox + !---------------------------------------------------------------- + + errmsg = '' + errflg = 0 + + ! Check for correct threading behavior + nbox = cole - cols + 1 + if (size(o3) /= nbox) then + errflg = 1 + write(errmsg, '(2(a,i0))') 'SIZE(O3) = ', size(o3), ', should be ', nbox + else if (size(hno3) /= nbox) then + errflg = 1 + write(errmsg, '(2(a,i0))') 'SIZE(HNO3) = ', size(hno3), & + ', should be ', nbox + else + ! NOTE -- This is prototyping one approach to passing a large number of + ! chemical VMR values and is the predecessor for adding in methods and + ! maybe nesting DDTs (especially for aerosols) + vmr%vmr_array(cols:cole, 1) = o3(:) + vmr%vmr_array(cols:cole, 2) = hno3(:) + end if + + end subroutine make_ddt_run + + !> \section arg_table_make_ddt_init Argument Table + !! \htmlinclude arg_table_make_ddt_init.html + !! + subroutine make_ddt_init(nbox, vmr, errmsg, errflg) + + ! Dummy arguments + integer, intent(in) :: nbox + type(vmr_type), intent(out) :: vmr + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine initializes the vmr array + vmr%nvmr = 2 + allocate(vmr%vmr_array(nbox, vmr%nvmr)) + + errmsg = '' + errflg = 0 + + end subroutine make_ddt_init + + !> \section arg_table_make_ddt_timestep_final Argument Table + !! \htmlinclude arg_table_make_ddt_timestep_final.html + !! + subroutine make_ddt_timestep_final(ncols, vmr, errmsg, errflg) + + ! Dummy arguments + integer, intent(in) :: ncols + type(vmr_type), intent(in) :: vmr + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + ! Local variables + integer :: index + real(kind=kind_phys) :: rind + + errmsg = '' + errflg = 0 + + ! This routine checks the array values in vmr + if (size(vmr%vmr_array, 1) /= ncols) then + errflg = 1 + write(errmsg, '(2(a,i0))') 'VMR%VMR_ARRAY first dimension size is, ', & + size(vmr%vmr_array, 1), ', should be, ', ncols + else + do index = 1, ncols + rind = real(index, kind_phys) + if (vmr%vmr_array(index, 1) /= rind * 1.e-6_kind_phys) then + errflg = 1 + write(errmsg, '(a,i0,2(a,e12.4))') 'O3(', index, ') = ', & + vmr%vmr_array(index, 1), ', should be, ', & + rind * 1.e-6_kind_phys + exit + else if (vmr%vmr_array(index, 2) /= rind * 1.e-9_kind_phys) then + errflg = 1 + write(errmsg, '(a,i0,2(a,e12.4))') 'HNO3(', index, ') = ', & + vmr%vmr_array(index, 2), ', should be, ', & + rind * 1.e-9_kind_phys + exit + end if + end do + end if + + end subroutine make_ddt_timestep_final + +end module make_ddt diff --git a/end-to-end-tests/ddthost/make_ddt.meta b/end-to-end-tests/ddthost/make_ddt.meta new file mode 100644 index 00000000..5b667a53 --- /dev/null +++ b/end-to-end-tests/ddthost/make_ddt.meta @@ -0,0 +1,129 @@ +[ccpp-table-properties] + name = vmr_type + type = ddt +[ccpp-arg-table] + name = vmr_type + type = ddt +[ nvmr ] + standard_name = number_of_chemical_species + units = count + dimensions = () + type = integer +[ vmr_array ] + standard_name = array_of_volume_mixing_ratios + units = ppmv + dimensions = (horizontal_dimension, number_of_chemical_species) + type = real + kind = kind_phys + +[ccpp-table-properties] + name = make_ddt + type = scheme +[ccpp-arg-table] + name = make_ddt_run + type = scheme +[ cols ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + intent = in +[ cole ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + intent = in +[ O3 ] + standard_name = ozone + units = ppmv + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = in +[ HNO3 ] + standard_name = nitric_acid + units = ppmv + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = in +[ vmr ] + standard_name = volume_mixing_ratio_ddt + dimensions = () + type = vmr_type + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = make_ddt_init + type = scheme +[ nbox ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ vmr ] + standard_name = volume_mixing_ratio_ddt + dimensions = () + type = vmr_type + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = make_ddt_timestep_final + type = scheme +[ ncols ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ vmr ] + standard_name = volume_mixing_ratio_ddt + dimensions = () + type = vmr_type + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/ddthost/setup_coeffs.F90 b/end-to-end-tests/ddthost/setup_coeffs.F90 new file mode 100644 index 00000000..60963f6c --- /dev/null +++ b/end-to-end-tests/ddthost/setup_coeffs.F90 @@ -0,0 +1,24 @@ +module setup_coeffs + use ccpp_kinds, only: kind_phys + implicit none + + public :: setup_coeffs_timestep_init + +contains + !> \section arg_table_setup_coeffs_timestep_init Argument Table + !! \htmlinclude arg_table_setup_coeffs_timestep_init.html + !! + subroutine setup_coeffs_timestep_init(coeffs, errmsg, errflg) + + real(kind=kind_phys), intent(inout) :: coeffs(:) + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + coeffs(:) = 1._kind_phys + + end subroutine setup_coeffs_timestep_init + +end module setup_coeffs diff --git a/end-to-end-tests/ddthost/setup_coeffs.meta b/end-to-end-tests/ddthost/setup_coeffs.meta new file mode 100644 index 00000000..f911df13 --- /dev/null +++ b/end-to-end-tests/ddthost/setup_coeffs.meta @@ -0,0 +1,29 @@ +[ccpp-table-properties] + name = setup_coeffs + type = scheme +[ccpp-arg-table] + name = setup_coeffs_timestep_init + type = scheme +[ coeffs ] + standard_name = coefficients_for_interpolation + long_name = coefficients for interpolation + units = none + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/ddthost/temp_adjust.F90 b/end-to-end-tests/ddthost/temp_adjust.F90 new file mode 100644 index 00000000..c0c4746c --- /dev/null +++ b/end-to-end-tests/ddthost/temp_adjust.F90 @@ -0,0 +1,84 @@ +! Test parameterization with no vertical level +! + +module temp_adjust + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: temp_adjust_init + public :: temp_adjust_run + public :: temp_adjust_final + +contains + + !> \section arg_table_temp_adjust_run Argument Table + !! \htmlinclude arg_table_temp_adjust_run.html + !! + subroutine temp_adjust_run(foo, timestep, temp_prev, temp_layer, qv, ps, & + to_promote, promote_pcnst, errmsg, errflg, innie, outie, optsie) + + integer, intent(in) :: foo + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(inout), optional :: qv(:, :) + real(kind=kind_phys), intent(inout) :: ps(:) + real(kind=kind_phys), intent(in) :: temp_prev(:, :) + real(kind=kind_phys), intent(inout) :: temp_layer(:, :) + real(kind=kind_phys), intent(in) :: to_promote(:, :) + real(kind=kind_phys), intent(in) :: promote_pcnst(:) + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + real(kind=kind_phys), optional, intent(in) :: innie + real(kind=kind_phys), optional, intent(out) :: outie + real(kind=kind_phys), optional, intent(inout) :: optsie + !---------------------------------------------------------------- + + integer :: col_index + + errmsg = '' + errflg = 0 + + do col_index = 1, foo + temp_layer(col_index, :) = temp_layer(col_index, :) + temp_prev(col_index, :) + if (present(qv)) qv(col_index, :) = qv(col_index, :) + 1.0_kind_phys + end do + if (present(innie) .and. present(outie) .and. present(optsie)) then + outie = innie * optsie + optsie = optsie + 1.0_kind_phys + end if + + end subroutine temp_adjust_run + + !> \section arg_table_temp_adjust_init Argument Table + !! \htmlinclude arg_table_temp_adjust_init.html + !! + subroutine temp_adjust_init(errmsg, errflg) + + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine currently does nothing + + errmsg = '' + errflg = 0 + + end subroutine temp_adjust_init + + !> \section arg_table_temp_adjust_final Argument Table + !! \htmlinclude arg_table_temp_adjust_final.html + !! + subroutine temp_adjust_final(errmsg, errflg) + + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine currently does nothing + + errmsg = '' + errflg = 0 + + end subroutine temp_adjust_final + +end module temp_adjust diff --git a/end-to-end-tests/ddthost/temp_adjust.meta b/end-to-end-tests/ddthost/temp_adjust.meta new file mode 100644 index 00000000..12880d13 --- /dev/null +++ b/end-to-end-tests/ddthost/temp_adjust.meta @@ -0,0 +1,118 @@ +[ccpp-table-properties] + name = temp_adjust + type = scheme + dependencies_path = + +[ccpp-arg-table] + name = temp_adjust_run + type = scheme +[ foo ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp_prev ] + standard_name = potential_temperature_at_previous_timestep + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = in +[ temp_layer ] + standard_name = potential_temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + diagnostic_name = temperature +[ qv ] + standard_name = water_vapor_specific_humidity + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + diagnostic_name_fixed = Q + optional = True +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) + intent = inout +[ to_promote ] + standard_name = promote_this_variable_to_suite + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = in +[ promote_pcnst ] + standard_name = promote_this_variable_with_no_horizontal_dimension + units = K + dimensions = (number_of_tracers) + type = real + kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = temp_adjust_init + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = temp_adjust_final + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/ddthost/temp_calc_adjust.F90 b/end-to-end-tests/ddthost/temp_calc_adjust.F90 new file mode 100644 index 00000000..40b5866e --- /dev/null +++ b/end-to-end-tests/ddthost/temp_calc_adjust.F90 @@ -0,0 +1,95 @@ +!Test parameterization with no vertical level and hanging intent(out) variable +! + +module temp_calc_adjust + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: temp_calc_adjust_init + public :: temp_calc_adjust_run + public :: temp_calc_adjust_final + +contains + + !> \section arg_table_temp_calc_adjust_run Argument Table + !! \htmlinclude arg_table_temp_calc_adjust_run.html + !! + subroutine temp_calc_adjust_run(nbox, timestep, temp_level, temp_calc, & + errmsg, errflg) + + integer, intent(in) :: nbox + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(in) :: temp_level(:, :) + real(kind=kind_phys), intent(out) :: temp_calc(:, :) + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: col_index + real(kind=kind_phys) :: bar = 1.0_kind_phys + + errmsg = '' + errflg = 0 + + call temp_calc_adjust_nested_subroutine(temp_calc) + if (check_foo()) then + call foo(bar) + end if + + contains + + elemental subroutine temp_calc_adjust_nested_subroutine(temp) + + real(kind=kind_phys), intent(out) :: temp + !------------------------------------------------------------- + + temp = 1.0_kind_phys + + end subroutine temp_calc_adjust_nested_subroutine + + subroutine foo(bar) + real(kind=kind_phys), intent(inout) :: bar + bar = bar + 1.0_kind_phys + + end subroutine foo + + logical function check_foo() + check_foo = .true. + end function check_foo + + end subroutine temp_calc_adjust_run + + !> \section arg_table_temp_calc_adjust_init Argument Table + !! \htmlinclude arg_table_temp_calc_adjust_init.html + !! + subroutine temp_calc_adjust_init(errmsg, errflg) + + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine currently does nothing + + errmsg = '' + errflg = 0 + + end subroutine temp_calc_adjust_init + + !> \section arg_table_temp_calc_adjust_final Argument Table + !! \htmlinclude arg_table_temp_calc_adjust_final.html + !! + subroutine temp_calc_adjust_final(errmsg, errflg) + + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine currently does nothing + + errmsg = '' + errflg = 0 + + end subroutine temp_calc_adjust_final + +end module temp_calc_adjust diff --git a/end-to-end-tests/ddthost/temp_calc_adjust.meta b/end-to-end-tests/ddthost/temp_calc_adjust.meta new file mode 100644 index 00000000..94fb9921 --- /dev/null +++ b/end-to-end-tests/ddthost/temp_calc_adjust.meta @@ -0,0 +1,88 @@ +[ccpp-table-properties] + name = temp_calc_adjust + type = scheme + dependencies = + +[ccpp-arg-table] + name = temp_calc_adjust_run + type = scheme + process = adjusting +[ nbox ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp_level ] + standard_name = potential_temperature_at_interface + units = K + dimensions = (ccpp_constant_one:horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys + intent = in +[ temp_calc ] + standard_name = potential_temperature_at_previous_timestep + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = temp_calc_adjust_init + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = temp_calc_adjust_final + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/ddthost/temp_set.F90 b/end-to-end-tests/ddthost/temp_set.F90 new file mode 100644 index 00000000..f817e34c --- /dev/null +++ b/end-to-end-tests/ddthost/temp_set.F90 @@ -0,0 +1,112 @@ +!Test 3D parameterization +! + +module temp_set + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: temp_set_init + public :: temp_set_timestep_init + public :: temp_set_run + public :: temp_set_final + +contains + + !> \section arg_table_temp_set_run Argument Table + !! \htmlinclude arg_table_temp_set_run.html + !! + subroutine temp_set_run(ncol, lev, timestep, temp_level, temp, ps, & + to_promote, promote_pcnst, errmsg, errflg) + !---------------------------------------------------------------- + implicit none + !---------------------------------------------------------------- + + integer, intent(in) :: ncol, lev + real(kind=kind_phys), intent(out) :: temp(:, :) + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(in) :: ps(:) + real(kind=kind_phys), intent(inout) :: temp_level(:, :) + real(kind=kind_phys), intent(out) :: to_promote(:, :) + real(kind=kind_phys), intent(out) :: promote_pcnst(:) + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + integer :: ilev + + integer :: col_index + integer :: lev_index + + errmsg = '' + errflg = 0 + + ilev = size(temp_level, 2) + if (ilev /= (lev + 1)) then + errflg = 1 + errmsg = 'Invalid value for ilev, must be lev+1' + return + end if + + do col_index = 1, ncol + do lev_index = 1, lev + temp(col_index, lev_index) = (temp_level(col_index, lev_index) & + + temp_level(col_index, lev_index + 1)) / 2.0_kind_phys + end do + end do + + end subroutine temp_set_run + + !> \section arg_table_temp_set_init Argument Table + !! \htmlinclude arg_table_temp_set_init.html + !! + subroutine temp_set_init(temp_inc_in, temp_inc_set, errmsg, errflg) + + real(kind=kind_phys), intent(in) :: temp_inc_in + real(kind=kind_phys), intent(out) :: temp_inc_set + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + + temp_inc_set = temp_inc_in + + errmsg = '' + errflg = 0 + + end subroutine temp_set_init + + !> \section arg_table_temp_set_timestep_init Argument Table + !! \htmlinclude arg_table_temp_set_timestep_init.html + !! + subroutine temp_set_timestep_init(ncol, temp_inc, temp_level, & + errmsg, errflg) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(in) :: temp_inc + real(kind=kind_phys), intent(inout) :: temp_level(:, :) + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + temp_level = temp_level + temp_inc + + end subroutine temp_set_timestep_init + + !> \section arg_table_temp_set_final Argument Table + !! \htmlinclude arg_table_temp_set_final.html + !! + subroutine temp_set_final(errmsg, errflg) + + character(len=256), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine currently does nothing + + errmsg = '' + errflg = 0 + + end subroutine temp_set_final + +end module temp_set diff --git a/end-to-end-tests/ddthost/temp_set.meta b/end-to-end-tests/ddthost/temp_set.meta new file mode 100644 index 00000000..e3375860 --- /dev/null +++ b/end-to-end-tests/ddthost/temp_set.meta @@ -0,0 +1,171 @@ +[ccpp-table-properties] + name = temp_set + type = scheme +[ccpp-arg-table] + name = temp_set_run + type = scheme + process = setter +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ lev ] + standard_name = vertical_layer_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp_level ] + standard_name = potential_temperature_at_interface + units = K + dimensions = (ccpp_constant_one:horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys + intent = inout +[ temp ] + standard_name = potential_temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = out +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) + intent = in +[ to_promote ] + standard_name = promote_this_variable_to_suite + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = out +[ promote_pcnst ] + standard_name = promote_this_variable_with_no_horizontal_dimension + units = K + dimensions = (number_of_tracers) + type = real + kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +# Init +[ccpp-arg-table] + name = temp_set_init + type = scheme +[ temp_inc_in ] + standard_name = potential_temperature_increment + long_name = Per time step potential temperature increment + units = K + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp_inc_set ] + standard_name = test_potential_temperature_increment + long_name = Per time step potential temperature increment + units = K + dimensions = () + type = real + kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +# Timestep Initialization +[ccpp-arg-table] + name = temp_set_timestep_init + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ temp_inc ] + standard_name = test_potential_temperature_increment + long_name = Per time step potential temperature increment + units = K + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp_level ] + standard_name = potential_temperature_at_interface + units = K + dimensions = (horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +# Finalize +[ccpp-arg-table] + name = temp_set_final + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/ddthost/temp_suite.xml b/end-to-end-tests/ddthost/temp_suite.xml new file mode 100644 index 00000000..7a4795c4 --- /dev/null +++ b/end-to-end-tests/ddthost/temp_suite.xml @@ -0,0 +1,12 @@ + + + + + setup_coeffs + temp_set + + + temp_calc_adjust + temp_adjust + + diff --git a/end-to-end-tests/ddthost/test_ddt_host_integration.F90 b/end-to-end-tests/ddthost/test_ddt_host_integration.F90 new file mode 100644 index 00000000..8e7358d2 --- /dev/null +++ b/end-to-end-tests/ddthost/test_ddt_host_integration.F90 @@ -0,0 +1,82 @@ +program test + use test_prog, only: test_host, & + suite_info, & + cm, & + cs + + implicit none + + character(len=cs), target :: test_parts1(2) = (/ 'physics1 ', & + 'physics2 ' /) + character(len=cs), target :: test_parts2(1) = (/ 'data_prep ' /) + character(len=cm), target :: test_invars1(8) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'index_of_water_vapor_specific_humidity', & + 'potential_temperature_increment ', & + 'time_step_for_physics ' /) + character(len=cm), target :: test_outvars1(7) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'ccpp_error_code ', & + 'ccpp_error_message ' /) + character(len=cm), target :: test_reqvars1(10) = (/ & + 'potential_temperature ', & + 'potential_temperature_at_interface ', & + 'coefficients_for_interpolation ', & + 'surface_air_pressure ', & + 'water_vapor_specific_humidity ', & + 'index_of_water_vapor_specific_humidity', & + 'potential_temperature_increment ', & + 'time_step_for_physics ', & + 'ccpp_error_code ', & + 'ccpp_error_message ' /) + + character(len=cm), target :: test_invars2(3) = (/ & + 'model_times ', & + 'number_of_model_times ', & + 'surface_air_pressure ' /) + + character(len=cm), target :: test_outvars2(4) = (/ & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'model_times ', & + 'number_of_model_times ' /) + + character(len=cm), target :: test_reqvars2(5) = (/ & + 'model_times ', & + 'number_of_model_times ', & + 'surface_air_pressure ', & + 'ccpp_error_code ', & + 'ccpp_error_message ' /) + + type(suite_info) :: test_suites(2) + logical :: run_okay + + ! Setup expected test suite info + test_suites(1)%suite_name = 'temp_suite' + test_suites(1)%suite_parts => test_parts1 + test_suites(1)%suite_input_vars => test_invars1 + test_suites(1)%suite_output_vars => test_outvars1 + test_suites(1)%suite_required_vars => test_reqvars1 + test_suites(2)%suite_name = 'ddt_suite' + test_suites(2)%suite_parts => test_parts2 + test_suites(2)%suite_input_vars => test_invars2 + test_suites(2)%suite_output_vars => test_outvars2 + test_suites(2)%suite_required_vars => test_reqvars2 + + call test_host(run_okay, test_suites) + + if (run_okay) then + stop 0 + else + stop -1 + end if + +end program test diff --git a/end-to-end-tests/ddthost/test_host.F90 b/end-to-end-tests/ddthost/test_host.F90 new file mode 100644 index 00000000..ef25b4cb --- /dev/null +++ b/end-to-end-tests/ddthost/test_host.F90 @@ -0,0 +1,333 @@ +module test_prog + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public test_host + + ! Public data and interfaces + integer, public, parameter :: cs = 16 + integer, public, parameter :: cm = 64 + + !> \section arg_table_suite_info Argument Table + !! \htmlinclude arg_table_suite_info.html + !! + type, public :: suite_info + character(len=cs) :: suite_name = '' + character(len=cs), pointer :: suite_parts(:) => null() + character(len=cm), pointer :: suite_input_vars(:) => null() + character(len=cm), pointer :: suite_output_vars(:) => null() + character(len=cm), pointer :: suite_required_vars(:) => null() + end type suite_info + +contains + + logical function check_suite(test_suite) + use ccpp_static_api, only: ccpp_physics_suite_part_list + use ccpp_static_api, only: ccpp_physics_suite_variables + use test_utils, only: check_list + + ! Dummy argument + type(suite_info), intent(in) :: test_suite + ! Local variables + integer :: sind + logical :: check + integer :: errflg + character(len=256) :: errmsg + character(len=128), allocatable :: test_list(:) + + check_suite = .true. + write(6, *) "Checking suite ", trim(test_suite%suite_name) + ! First, check the suite parts + call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_parts, 'part names', & + suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the input variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.true., output_vars=.false.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_input_vars, & + 'input variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the output variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.false., output_vars=.true.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_output_vars, & + 'output variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check all required variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_required_vars, & + 'required variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + end function check_suite + + !> \section arg_table_test_host Argument Table + !! \htmlinclude arg_table_test_host.html + !! + subroutine test_host(retval, test_suites) + + use test_host_mod, only: ncols, & + num_time_steps + use ccpp_static_api, only: ccpp_register + use ccpp_static_api, only: ccpp_init + use ccpp_static_api, only: ccpp_physics_init + use ccpp_static_api, only: ccpp_physics_timestep_init + use ccpp_static_api, only: ccpp_physics_run + use ccpp_static_api, only: ccpp_physics_timestep_final + use ccpp_static_api, only: ccpp_physics_final + use ccpp_static_api, only: ccpp_final + use ccpp_static_api, only: ccpp_physics_suite_list + use test_host_mod, only: init_data, & + compare_data, & + check_model_times + use test_utils, only: check_list + + type(suite_info), intent(in) :: test_suites(:) + logical, intent(out) :: retval + + logical :: check + integer :: col_start, col_end + integer :: index, sind + integer :: time_step + integer :: num_suites + character(len=128), allocatable :: suite_names(:) + integer :: errflg + character(len=256) :: errmsg + + + ! Initialize our 'data' + call init_data() + + ! Gather and test the inspection routines + num_suites = size(test_suites) + call ccpp_physics_suite_list(suite_names) + retval = check_list(suite_names, test_suites(:)%suite_name, & + 'suite names') + write(6, *) 'Available suites are:' + do index = 1, size(suite_names) + do sind = 1, num_suites + if (trim(test_suites(sind)%suite_name) == & + trim(suite_names(index))) then + exit + end if + end do + write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & + ' = test_suites(', sind, ')' + end do + if (retval) then + do sind = 1, num_suites + check = check_suite(test_suites(sind)) + retval = retval .and. check + end do + end if + !!! Return here if any check failed + if (.not. retval) then + return + end if + + ! Register CCPP + do sind = 1, num_suites + call ccpp_register( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in register of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Initialize CCPP + do sind = 1, num_suites + call ccpp_init( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in init of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Use the suite information to setup the run + do sind = 1, num_suites + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='', errmsg=errmsg, errflg=errflg, & + col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in physics_init of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Loop over time steps + do time_step = 1, num_time_steps + ! Initialize the timestep + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='', errmsg=errmsg, errflg=errflg, & + col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) + end if + if (errflg /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + exit + end if + if (errflg /= 0) then + exit + end if + end do + + do col_start = 1, ncols, 5 + if (errflg /= 0) then + exit + end if + col_end = min(col_start + 4, ncols) + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + do index = 1, size(test_suites(sind)%suite_parts) + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + errmsg=errmsg, errflg=errflg, & + col_start=col_start, col_end=col_end, & + thread_num=1, nthreads=1, nphys_threads=1) + end if + if (errflg /= 0) then + write(6, '(5a)') trim(test_suites(sind)%suite_name), & + '/', trim(test_suites(sind)%suite_parts(index)), & + ': ', trim(errmsg) + exit + end if + end do + end do + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='', errmsg=errmsg, errflg=errflg, & + col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) + end if + if (errflg /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_timestep_final, ', & + 'Exiting...' + exit + end if + end do + end do ! End time step loop + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='', errmsg=errmsg, errflg=errflg, & + col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1) + end if + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & + 'Exiting...' + exit + end if + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_final( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_final, ', & + 'Exiting...' + exit + end if + end do + + if (errflg == 0) then + ! Run finished without error, check answers + if (.not. check_model_times()) then + write(6, *) 'Model times error!' + errflg = -1 + else if (compare_data()) then + write(6, *) 'Answers are correct!' + errflg = 0 + else + write(6, *) 'Answers are not correct!' + errflg = -1 + end if + end if + + retval = errflg == 0 + + end subroutine test_host + +end module test_prog diff --git a/end-to-end-tests/ddthost/test_host.meta b/end-to-end-tests/ddthost/test_host.meta new file mode 100644 index 00000000..13da7afc --- /dev/null +++ b/end-to-end-tests/ddthost/test_host.meta @@ -0,0 +1,64 @@ +[ccpp-table-properties] + name = test_host + type = control + +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/end-to-end-tests/ddthost/test_host_data.F90 b/end-to-end-tests/ddthost/test_host_data.F90 new file mode 100644 index 00000000..88812719 --- /dev/null +++ b/end-to-end-tests/ddthost/test_host_data.F90 @@ -0,0 +1,51 @@ +module test_host_data + + use ccpp_kinds, only: kind_phys + + !> \section arg_table_physics_state Argument Table + !! \htmlinclude arg_table_physics_state.html + type physics_state + real(kind=kind_phys), dimension(:), allocatable :: & + ps ! surface pressure + real(kind=kind_phys), dimension(:, :), allocatable :: & + u, & ! zonal wind (m/s) + v, & ! meridional wind (m/s) + pmid ! midpoint pressure (Pa) + + real(kind=kind_phys), dimension(:, :, :), allocatable :: & + q ! constituent mixing ratio (kg/kg moist or dry air depending on type) + end type physics_state + + public allocate_physics_state + +contains + + subroutine allocate_physics_state(cols, levels, constituents, state) + integer, intent(in) :: cols + integer, intent(in) :: levels + integer, intent(in) :: constituents + type(physics_state), intent(out) :: state + + if (allocated(state%ps)) then + deallocate(state%ps) + end if + allocate(state%ps(cols)) + if (allocated(state%u)) then + deallocate(state%u) + end if + allocate(state%u(cols, levels)) + if (allocated(state%v)) then + deallocate(state%v) + end if + allocate(state%v(cols, levels)) + if (allocated(state%pmid)) then + deallocate(state%pmid) + end if + allocate(state%pmid(cols, levels)) + if (allocated(state%q)) then + deallocate(state%q) + end if + allocate(state%q(cols, levels, constituents)) + + end subroutine allocate_physics_state +end module test_host_data diff --git a/end-to-end-tests/ddthost/test_host_data.meta b/end-to-end-tests/ddthost/test_host_data.meta new file mode 100644 index 00000000..f195605e --- /dev/null +++ b/end-to-end-tests/ddthost/test_host_data.meta @@ -0,0 +1,46 @@ +[ccpp-table-properties] + name = physics_state + type = ddt +[ccpp-arg-table] + name = physics_state + type = ddt +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) +[ u ] + standard_name = eastward_wind + long_name = Zonal wind + type = real + kind = kind_phys + units = m s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) +[ v ] + standard_name = northward_wind + long_name = Meridional wind + type = real + kind = kind_phys + units = m s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) +[ pmid ] + standard_name = air_pressure + long_name = Midpoint air pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension, vertical_layer_dimension) +[ q ] + standard_name = constituent_mixing_ratio + type = real + kind = kind_phys + units = kg kg-1 moist or dry air depending on type + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) +[ q(:,:,index_of_water_vapor_specific_humidity) ] + standard_name = water_vapor_specific_humidity + type = real + kind = kind_phys + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + active = (index_of_water_vapor_specific_humidity > 0) diff --git a/end-to-end-tests/ddthost/test_host_mod.F90 b/end-to-end-tests/ddthost/test_host_mod.F90 new file mode 100644 index 00000000..066d8a7d --- /dev/null +++ b/end-to-end-tests/ddthost/test_host_mod.F90 @@ -0,0 +1,141 @@ +module test_host_mod + + use ccpp_kinds, only: kind_phys + use test_host_data, only: physics_state, & + allocate_physics_state + + implicit none + public + + !> \section arg_table_test_host_mod Argument Table + !! \htmlinclude arg_table_test_host_host.html + !! + integer, parameter :: ncols = 10 + integer, parameter :: pver = 5 + integer, parameter :: pverp = 6 + integer, parameter :: pcnst = 2 + integer, parameter :: diagdimstart = 2 + integer, parameter :: index_qv = 1 + real(kind=kind_phys), allocatable :: temp_midpoints(:, :) + real(kind=kind_phys) :: temp_interfaces(ncols, pverp) + real(kind=kind_phys) :: coeffs(ncols) + real(kind=kind_phys), dimension(diagdimstart:ncols, diagdimstart:pver) :: & + diag1, & + diag2 + real(kind=kind_phys) :: dt + real(kind=kind_phys), parameter :: temp_inc = 0.05_kind_phys + type(physics_state), target :: phys_state + integer :: num_model_times = -1 + integer, allocatable :: model_times(:) + + integer, parameter :: num_time_steps = 2 + real(kind=kind_phys), parameter :: tolerance = 1.0e-13_kind_phys + real(kind=kind_phys) :: tint_save(ncols, pverp) + + public :: init_data + public :: compare_data + public :: check_model_times + +contains + + subroutine init_data() + + integer :: col + integer :: lev + integer :: cind + integer :: offsize + + ! Allocate and initialize temperature + allocate(temp_midpoints(ncols, pver)) + temp_midpoints = 0.0_kind_phys + do lev = 1, pverp + offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) + do col = 1, ncols + temp_interfaces(col, lev) = real(offsize + col, kind=kind_phys) + tint_save(col, lev) = temp_interfaces(col, lev) + end do + end do + ! Allocate and initialize state + call allocate_physics_state(ncols, pver, pcnst, phys_state) + do cind = 1, pcnst + do lev = 1, pver + offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) + do col = 1, ncols + phys_state%q(col, lev, cind) = real(offsize + col, kind=kind_phys) + end do + end do + end do + + end subroutine init_data + + logical function check_model_times() + + check_model_times = (num_model_times > 0) + if (check_model_times) then + check_model_times = (size(model_times) == num_model_times) + if (.not. check_model_times) then + write(6, '(2(a,i0))') 'model_times size mismatch, ', & + size(model_times), ' should be ', num_model_times + end if + else + write(6, '(a,i0,a)') 'num_model_times mismatch, ', num_model_times, & + ' should be greater than zero' + end if + + end function check_model_times + + logical function compare_data() + + integer :: col + integer :: lev + integer :: cind + integer :: offsize + logical :: need_header + real(kind=kind_phys) :: avg + integer, parameter :: cincrements(pcnst) = (/ 1, 0 /) + + compare_data = .true. + + need_header = .true. + do lev = 1, pver + do col = 1, ncols + avg = (tint_save(col, lev) + tint_save(col, lev + 1)) + avg = 1.0_kind_phys + (avg / 2.0_kind_phys) + avg = avg + (temp_inc * num_time_steps) + if (abs((temp_midpoints(col, lev) - avg) / avg) > tolerance) then + if (need_header) then + write(6, '(" COL LEV T MIDPOINTS EXPECTED")') + need_header = .false. + end if + write(6, '(2i5,2(3x,es15.7))') col, lev, & + temp_midpoints(col, lev), avg + compare_data = .false. + end if + end do + end do + ! Check constituents + need_header = .true. + do cind = 1, pcnst + do lev = 1, pver + offsize = ((cind - 1) * (ncols * pver)) + ((lev - 1) * ncols) + do col = 1, ncols + avg = real(offsize + col + (cincrements(cind) * num_time_steps), & + kind=kind_phys) + if (abs((phys_state%q(col, lev, cind) - avg) / avg) > & + tolerance) then + if (need_header) then + write(6, '(2(2x,a),3x,a,10x,a,14x,a)') & + 'COL', 'LEV', 'C#', 'Q', 'EXPECTED' + need_header = .false. + end if + write(6, '(3i5,2(3x,es15.7))') col, lev, cind, & + phys_state%q(col, lev, cind), avg + compare_data = .false. + end if + end do + end do + end do + + end function compare_data + +end module test_host_mod diff --git a/end-to-end-tests/ddthost/test_host_mod.meta b/end-to-end-tests/ddthost/test_host_mod.meta new file mode 100644 index 00000000..e278742a --- /dev/null +++ b/end-to-end-tests/ddthost/test_host_mod.meta @@ -0,0 +1,98 @@ +[ccpp-table-properties] + name = test_host_mod + type = host +[ccpp-arg-table] + name = test_host_mod + type = host +[ index_qv ] + standard_name = index_of_water_vapor_specific_humidity + units = index + type = integer + protected = True + dimensions = () +[ ncols] + standard_name = horizontal_dimension + units = count + type = integer + protected = True + dimensions = () +[ pver ] + standard_name = vertical_layer_dimension + units = count + type = integer + protected = True + dimensions = () +[ pverP ] + standard_name = vertical_interface_dimension + type = integer + units = count + protected = True + dimensions = () +[ pcnst ] + standard_name = number_of_tracers + type = integer + units = count + protected = True + dimensions = () +[ DiagDimStart ] + standard_name = first_index_of_diag_fields + type = integer + units = count + protected = True + dimensions = () +[ temp_midpoints ] + standard_name = potential_temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys +[ temp_interfaces ] + standard_name = potential_temperature_at_interface + units = K + dimensions = (horizontal_dimension, vertical_interface_dimension) + type = real | kind = kind_phys +[ diag1 ] + standard_name = diagnostic_stuff_type_1 + long_name = This is just a test field + units = K + dimensions = (first_index_of_diag_fields:horizontal_dimension, first_index_of_diag_fields:vertical_layer_dimension) + type = real | kind = kind_phys +[ diag2 ] + standard_name = diagnostic_stuff_type_2 + long_name = This is just a test field + units = K + dimensions = (first_index_of_diag_fields: horizontal_dimension, first_index_of_diag_fields :vertical_layer_dimension) + type = real | kind = kind_phys +[ dt ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real | kind = kind_phys +[ temp_inc ] + standard_name = potential_temperature_increment + long_name = Per time step potential temperature increment + units = K + dimensions = () + type = real | kind = kind_phys +[ phys_state ] + standard_name = physics_state_derived_type + long_name = Physics State DDT + type = physics_state + dimensions = () +[ num_model_times ] + standard_name = number_of_model_times + type = integer + units = count + dimensions = () +[ model_times ] + standard_name = model_times + units = seconds + dimensions = (number_of_model_times) + type = integer + allocatable = True +[ coeffs ] + standard_name = coefficients_for_interpolation + long_name = coefficients for interpolation + units = none + dimensions = (horizontal_dimension) + type = real | kind = kind_phys \ No newline at end of file diff --git a/end-to-end-tests/instances/CMakeLists.txt b/end-to-end-tests/instances/CMakeLists.txt new file mode 100644 index 00000000..2b56aeab --- /dev/null +++ b/end-to-end-tests/instances/CMakeLists.txt @@ -0,0 +1,59 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "unit_conv_scheme_1" "unit_conv_scheme_2") +set(HOST_FILES "data" "main") +set(SUITE_FILES "suite_unit_conv_suite.xml") +set(HOST "test_host") + +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES}) + +# Run ccpp_capgen_ng +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_instances.x ${SCHEME_FORTRAN_FILES} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES}) +target_link_libraries(test_instances.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_instances.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_instances.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_instances + COMMAND test_instances.x) diff --git a/end-to-end-tests/instances/README.md b/end-to-end-tests/instances/README.md new file mode 100644 index 00000000..787d0926 --- /dev/null +++ b/end-to-end-tests/instances/README.md @@ -0,0 +1,16 @@ +# How to build the unit conv test + +1. Set compiler environment as appropriate for your system +2. Run the following commands: +``` +cd test_prebuild/test_unit_conv/ +#rm -fr build +mkdir build +../../scripts/ccpp_prebuild.py --config=ccpp_prebuild_config.py --builddir=build +cd build +cmake .. 2>&1 | tee log.cmake +make 2>&1 | tee log.make +./test_unit_conv.x +# On systems where linking against the MPI library requires a parallel launcher, +# use 'mpirun -np 1 ./test_unit_conv.x' or 'srun -n 1 ./test_unit_conv.x' etc. +``` diff --git a/end-to-end-tests/instances/data.F90 b/end-to-end-tests/instances/data.F90 new file mode 100644 index 00000000..924a209b --- /dev/null +++ b/end-to-end-tests/instances/data.F90 @@ -0,0 +1,30 @@ +module data + + !! \section arg_table_data Argument Table + !! \htmlinclude data.html + !! + use ccpp_kinds, only : kind_phys + + implicit none + + private + + public ncols, nspecies, ninstances + public instance_type, instance_data + + integer, parameter :: ncols = 4 + integer, parameter :: nspecies = 2 + integer, parameter :: ninstances = 2 + + !! \section arg_table_instance_type Argument Table + !! \htmlinclude instance_type.html + !! + type instance_type + real(kind=kind_phys), dimension(1:ncols, 1:nspecies) :: data_array + real(kind=kind_phys), dimension(1:ncols) :: data_array2 + logical :: opt_array_flag + end type instance_type + + type(instance_type), dimension(1:ninstances), target :: instance_data + +end module data diff --git a/end-to-end-tests/instances/data.meta b/end-to-end-tests/instances/data.meta new file mode 100644 index 00000000..7abae0e2 --- /dev/null +++ b/end-to-end-tests/instances/data.meta @@ -0,0 +1,77 @@ +[ccpp-table-properties] + name = instance_type + type = ddt + dependencies = + +[ccpp-arg-table] + name = instance_type + type = ddt +[data_array] + standard_name = data_array_all_species + long_name = data array in module + units = m + dimensions = (horizontal_dimension, number_of_species) + type = real + kind = kind_phys +[data_array(:,2)] + standard_name = data_array + long_name = data array in module + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys +[data_array2] + standard_name = data_array2 + long_name = data array 2 in module + units = m2 s-2 + dimensions = (horizontal_dimension) + type = real + kind = kind_phys +[opt_array_flag] + standard_name = flag_for_opt_array + long_name = flag for passing optional data array + units = 1 + dimensions = () + type = logical +[data_array(:,1)] + standard_name = data_array_opt + long_name = optional data array in km + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + active = (flag_for_opt_array) + + +[ccpp-table-properties] + name = data + type = host + dependencies = + +[ccpp-arg-table] + name = data + type = host +[ncols] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer +[nspecies] + standard_name = number_of_species + long_name = number of species in data array + units = count + dimensions = () + type = integer +[ninstances] + standard_name = number_of_instances + long_name = number of instances for multi-instance test + units = count + dimensions = () + type = integer +[instance_data] + standard_name = instance_data + long_name = instance data for multi-instance test + units = ddt + dimensions = (number_of_instances) + type = instance_type diff --git a/end-to-end-tests/instances/main.F90 b/end-to-end-tests/instances/main.F90 new file mode 100644 index 00000000..20a49128 --- /dev/null +++ b/end-to-end-tests/instances/main.F90 @@ -0,0 +1,177 @@ +program test_unit_conv + + use, intrinsic :: iso_fortran_env, only: error_unit +#ifdef _OPENMP + use omp_lib +#endif + + use data, only: ncols, & + nspecies, ninstances + !use data, only: cdata, & + ! data_array, & + ! data_array2, & + ! opt_array_flag + use data, only: instance_data + + use ccpp_static_api, only: ccpp_register, & + ccpp_init, & + ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_final, & + ccpp_physics_final, & + ccpp_final + + implicit none + + character(len=*), parameter :: ccpp_suite = 'unit_conv_suite' + ! An updated ccpp_validator.py should detect this - metadata has len=512 + character(len=256) :: errmsg + integer :: errflg + integer :: nphys_threads + integer :: ins + + !data_array = 1.0_8 + !data_array2 = 42.0_8 + !opt_array_flag = .true. + + instance_data(1)%data_array = -1.0_8 + instance_data(1)%data_array2 = -42.0_8 + instance_data(1)%opt_array_flag = .true. + + instance_data(2)%data_array = +1.0_8 + instance_data(2)%data_array2 = +42.0_8 + instance_data(2)%opt_array_flag = .false. + + ! Use OpenMP threading in physics (internally) +#ifdef _OPENMP + nphys_threads = omp_get_max_threads() +#else + nphys_threads = 1 +#endif + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP register step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_register(suite_name=ccpp_suite, instance=ins, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_register:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_init(suite_name=ccpp_suite, instance=ins, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_init:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_physics_init( & + suite_name=ccpp_suite, group_name='all', instance=ins, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_physics_timestep_init( & + suite_name=ccpp_suite, group_name='all', instance=ins, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics run step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_physics_run( & + suite_name=ccpp_suite, group_name='all', instance=ins, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_run:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep final step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_physics_timestep_final( & + suite_name=ccpp_suite, group_name='all', instance=ins, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_final:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics final step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_physics_final( & + suite_name=ccpp_suite, group_name='all', instance=ins, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_final:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP final step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + do ins=1,ninstances + call ccpp_final(suite_name=ccpp_suite, instance=ins, errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_final:" + write(error_unit, '(a)') trim(errmsg) + write(error_unit, '(a,i0)') "instance: ", ins + stop 1 + end if + end do + +end program test_unit_conv diff --git a/end-to-end-tests/instances/main.meta b/end-to-end-tests/instances/main.meta new file mode 100644 index 00000000..8d774264 --- /dev/null +++ b/end-to-end-tests/instances/main.meta @@ -0,0 +1,71 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ instance ] + standard_name = instance_number + long_name = current model instance number + units = index + dimensions = () + type = integer diff --git a/end-to-end-tests/instances/suite_unit_conv_suite.xml b/end-to-end-tests/instances/suite_unit_conv_suite.xml new file mode 100644 index 00000000..0f499a42 --- /dev/null +++ b/end-to-end-tests/instances/suite_unit_conv_suite.xml @@ -0,0 +1,11 @@ + + + + + + unit_conv_scheme_1 + unit_conv_scheme_2 + unit_conv_scheme_1 + + + diff --git a/end-to-end-tests/instances/unit_conv_scheme_1.F90 b/end-to-end-tests/instances/unit_conv_scheme_1.F90 new file mode 100644 index 00000000..3cf05bf8 --- /dev/null +++ b/end-to-end-tests/instances/unit_conv_scheme_1.F90 @@ -0,0 +1,86 @@ +!>\file unit_conv_scheme_1.F90 +!! This file contains a unit_conv_scheme_1 CCPP scheme that does nothing +!! except requesting the minimum, mandatory variables. + +module unit_conv_scheme_1 + + use, intrinsic :: iso_fortran_env, only: error_unit + use ccpp_kinds, only : kind_phys + implicit none + + private + public :: unit_conv_scheme_1_run + + ! This is for unit testing only + real(kind=kind_phys), parameter, dimension(1:2) :: target_values = (/-1.0_kind_phys, 1.0_kind_phys/) + real(kind=kind_phys), parameter, dimension(1:2) :: target_values2 = (/-42.0_kind_phys, 42.0_kind_phys/) + +contains + + !! \section arg_table_unit_conv_scheme_1_run Argument Table + !! \htmlinclude unit_conv_scheme_1_run.html + !! + subroutine unit_conv_scheme_1_run(instance, data_array, data_array2, data_array_opt, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: instance + real(kind=kind_phys), intent(inout) :: data_array(:) + real(kind=kind_phys), intent(inout) :: data_array2(:) + real(kind=kind_phys), intent(inout), optional :: data_array_opt(:) + + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + + ! Check values in data array + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_1_run: checking min/max values of data array to be approximately ', & + target_values(instance) + if (abs(minval(data_array) - target_values(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array) - target_values(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') & + "Error in unit_conv_scheme_1_run, expected values for data_array of approximately ", & + target_values(instance), " but got [ ", minval(data_array), " : ", maxval(data_array), " ]" + errflg = 1 + return + end if + ! Check values in data array2 + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_1_run: checking min/max values of data array 2 to be approximately ', & + target_values2(instance) + if (abs(minval(data_array2) - target_values2(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array2) - target_values2(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') & + "Error in unit_conv_scheme_1_run, expected values for data array 2 of approximately ", & + target_values2(instance), " but got [ ", minval(data_array2), " : ", maxval(data_array2), " ]" + errflg = 1 + return + end if + ! Check for presence of optional data array, then check its values + write(error_unit, '(a)') 'In unit_conv_scheme_1_run: checking for presence of optional data array' + if (instance==1) then + if (.not. present(data_array_opt)) then + write(errmsg, '(a)') 'Error in unit_conv_scheme_1_run, optional data array expected but not present' + write(errmsg, '(a,i0)') 'for instance ', instance + errflg = 1 + return + end if + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_1_run: checking min/max values of optional data array to be approximately ', target_values(instance) + if (abs(minval(data_array_opt) - target_values(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array_opt) - target_values(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') 'Error in unit_conv_scheme_1_run, expected values of approximately ', & + target_values(instance), ' but got [ ', minval(data_array_opt), ' : ', maxval(data_array_opt), ' ]' + errflg = 1 + return + end if + else if (instance==2 .and. present(data_array_opt)) then + write(errmsg, '(a)') 'Error in unit_conv_scheme_1_run, optional data array not expected but present' + write(errmsg, '(a,i0)') 'for instance ', instance + errflg = 1 + return + end if + + end subroutine unit_conv_scheme_1_run + +end module unit_conv_scheme_1 diff --git a/end-to-end-tests/instances/unit_conv_scheme_1.meta b/end-to-end-tests/instances/unit_conv_scheme_1.meta new file mode 100644 index 00000000..ef096774 --- /dev/null +++ b/end-to-end-tests/instances/unit_conv_scheme_1.meta @@ -0,0 +1,56 @@ +[ccpp-table-properties] + name = unit_conv_scheme_1 + type = scheme + dependencies = + +######################################################################## +[ccpp-arg-table] + name = unit_conv_scheme_1_run + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[instance] + standard_name = instance_number + long_name = instance number for testing multi-instance support + units = index + dimensions = () + type = integer + intent = in +[data_array] + standard_name = data_array + long_name = data array in m + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[data_array2] + standard_name = data_array2 + long_name = data array in J kg-1 + units = J kg-1 + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[data_array_opt] + standard_name = data_array_opt + long_name = optional data array in m + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout + optional = True diff --git a/end-to-end-tests/instances/unit_conv_scheme_2.F90 b/end-to-end-tests/instances/unit_conv_scheme_2.F90 new file mode 100644 index 00000000..f1d8fbed --- /dev/null +++ b/end-to-end-tests/instances/unit_conv_scheme_2.F90 @@ -0,0 +1,86 @@ +!>\file unit_conv_scheme_2.F90 +!! This file contains a unit_conv_scheme_2 CCPP scheme that does nothing +!! except requesting the minimum, mandatory variables. + +module unit_conv_scheme_2 + + use, intrinsic :: iso_fortran_env, only: error_unit + use ccpp_kinds, only : kind_phys + implicit none + + private + public :: unit_conv_scheme_2_run + + ! This is for unit testing only + real(kind=kind_phys), parameter, dimension(1:2) :: target_values = (/-1.0E-3_kind_phys, 1.0E-3_kind_phys/) + real(kind=kind_phys), parameter, dimension(1:2) :: target_values2 = (/-42.0_kind_phys, 42.0_kind_phys/) + +contains + + !! \section arg_table_unit_conv_scheme_2_run Argument Table + !! \htmlinclude unit_conv_scheme_2_run.html + !! + subroutine unit_conv_scheme_2_run(instance, data_array, data_array2, data_array_opt, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: instance + real(kind=kind_phys), intent(inout) :: data_array(:) + real(kind=kind_phys), intent(inout) :: data_array2(:) + real(kind=kind_phys), intent(inout), optional :: data_array_opt(:) + + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + + ! Check values in data array + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_2_run: checking min/max values of data array to be approximately ', & + target_values(instance) + if (abs(minval(data_array) - target_values(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array) - target_values(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') & + "Error in unit_conv_scheme_2_run, expected values for data_array of approximately ", & + target_values(instance), " but got [ ", minval(data_array), " : ", maxval(data_array), " ]" + errflg = 1 + return + end if + ! Check values in data array2 + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_2_run: checking min/max values of data array 2 to be approximately ', & + target_values2(instance) + if (abs(minval(data_array2) - target_values2(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array2) - target_values2(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') & + "Error in unit_conv_scheme_2_run, expected values for data array 2 of approximately ", & + target_values2(instance), " but got [ ", minval(data_array2), " : ", maxval(data_array2), " ]" + errflg = 1 + return + end if + ! Check for presence of optional data array, then check its values + write(error_unit, '(a)') 'In unit_conv_scheme_2_run: checking for presence of optional data array' + if (instance==1) then + if (.not. present(data_array_opt)) then + write(errmsg, '(a)') 'Error in unit_conv_scheme_2_run, optional data array expected but not present' + write(errmsg, '(a,i0)') 'for instance ', instance + errflg = 1 + return + end if + write(error_unit, '(a,e12.4)') & + 'In unit_conv_scheme_2_run: checking min/max values of optional data array to be approximately ', target_values(instance) + if (abs(minval(data_array_opt) - target_values(instance)) > 0.01_kind_phys .or. & + abs(maxval(data_array_opt) - target_values(instance)) > 0.01_kind_phys) then + write(errmsg, '(3(a,e12.4),a)') 'Error in unit_conv_scheme_2_run, expected values of approximately ', & + target_values(instance), ' but got [ ', minval(data_array_opt), ' : ', maxval(data_array_opt), ' ]' + errflg = 1 + return + end if + else if (instance==2 .and. present(data_array_opt)) then + write(errmsg, '(a)') 'Error in unit_conv_scheme_2_run, optional data array not expected but present' + write(errmsg, '(a,i0)') 'for instance ', instance + errflg = 1 + return + end if + + end subroutine unit_conv_scheme_2_run + +end module unit_conv_scheme_2 diff --git a/end-to-end-tests/instances/unit_conv_scheme_2.meta b/end-to-end-tests/instances/unit_conv_scheme_2.meta new file mode 100644 index 00000000..e1b916c2 --- /dev/null +++ b/end-to-end-tests/instances/unit_conv_scheme_2.meta @@ -0,0 +1,56 @@ +[ccpp-table-properties] + name = unit_conv_scheme_2 + type = scheme + dependencies = + +######################################################################## +[ccpp-arg-table] + name = unit_conv_scheme_2_run + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[instance] + standard_name = instance_number + long_name = instance number for testing multi-instance support + units = index + dimensions = () + type = integer + intent = in +[data_array] + standard_name = data_array + long_name = data array in km + units = km + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[data_array2] + standard_name = data_array2 + long_name = data array in m+2 s-2 + units = m+2 s-2 + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[data_array_opt] + standard_name = data_array_opt + long_name = optional data array in km + units = km + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout + optional = True diff --git a/end-to-end-tests/nested_suite/CMakeLists.txt b/end-to-end-tests/nested_suite/CMakeLists.txt new file mode 100644 index 00000000..43a2e1f4 --- /dev/null +++ b/end-to-end-tests/nested_suite/CMakeLists.txt @@ -0,0 +1,71 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "effr_calc" "effrs_calc" "effr_diag" "effr_pre" "effr_post" "rad_lw" "rad_sw" "suite_lifecycle") +set(HOST_FILES "module_rad_ddt" "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "main_suite.xml") +set(HOST "test_host") + +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES}) + +# Run ccpp_capgen_ng +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + KIND_TYPES ${KIND_TYPES} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_nested_suite.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_nested_suite_integration.F90 +) +target_link_libraries(test_nested_suite.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_nested_suite.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_nested_suite.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_nested_suite + COMMAND test_nested_suite.x) diff --git a/end-to-end-tests/nested_suite/README.md b/end-to-end-tests/nested_suite/README.md new file mode 100644 index 00000000..5723560b --- /dev/null +++ b/end-to-end-tests/nested_suite/README.md @@ -0,0 +1,19 @@ +# Nested Suite Test + +Tests the capability to process nested suites: +- Inherited from the variable compatibility test as of 2025/10/01 + - Perform same tests as variable compatibility test at that date +- Parse new XML schema 2.0 +- Expand nested suites at the group level and inside groups +- Test single init and final schemes in suite + +## Building/Running + +To explicitly build/run the nested suite test host, run: + +```bash +$ cmake --fresh -S -B -DCCPP_RUN_NESTED_SUITE_TEST=ON +$ cd +$ make +$ ctest +``` diff --git a/end-to-end-tests/nested_suite/effr_calc.F90 b/end-to-end-tests/nested_suite/effr_calc.F90 new file mode 100644 index 00000000..b8fc43ed --- /dev/null +++ b/end-to-end-tests/nested_suite/effr_calc.F90 @@ -0,0 +1,84 @@ +!Test unit conversions for intent in, inout, out variables +! + +module effr_calc + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: effr_calc_run, effr_calc_init + +contains + !> \section arg_table_effr_calc_init Argument Table + !! \htmlinclude arg_table_effr_calc_init.html + !! + subroutine effr_calc_init(scheme_order, errmsg, errflg) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: scheme_order + + errmsg = '' + errflg = 0 + + if (scheme_order /= 2) then + errflg = 1 + errmsg = 'ERROR: effr_calc_init() needs to be called second' + return + else + scheme_order = scheme_order + 1 + end if + + end subroutine effr_calc_init + + !> \section arg_table_effr_calc_run Argument Table + !! \htmlinclude arg_table_effr_calc_run.html + !! + subroutine effr_calc_run(ncol, nlev, effrr_in, effrg_in, ncg_in, nci_out, & + effrl_inout, effri_out, effrs_inout, ncl_out, & + has_graupel, scalar_var, tke_inout, tke2_inout, & + errmsg, errflg) + + integer, intent(in) :: ncol + integer, intent(in) :: nlev + real(kind=kind_phys), intent(in) :: effrr_in(:, :) + real(kind=kind_phys), intent(in), optional :: effrg_in(:, :) + real(kind=kind_phys), intent(in), optional :: ncg_in(:, :) + real(kind=kind_phys), intent(out), optional :: nci_out(:, :) + real(kind=kind_phys), intent(inout) :: effrl_inout(:, :) + real(kind=kind_phys), intent(out), optional :: effri_out(:, :) + real(kind=8), intent(inout) :: effrs_inout(:, :) + logical, intent(in) :: has_graupel + real(kind=kind_phys), intent(inout) :: scalar_var + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + real(kind=kind_phys), intent(out), optional :: ncl_out(:, :) + real(kind=kind_phys), intent(inout) :: tke_inout + real(kind=kind_phys), intent(inout) :: tke2_inout + + !---------------------------------------------------------------- + + real(kind=kind_phys), parameter :: re_qc_min = 2.5 ! microns + real(kind=kind_phys), parameter :: re_qc_max = 50. ! microns + real(kind=kind_phys), parameter :: re_qi_avg = 75. ! microns + real(kind=kind_phys) :: effrr_local(ncol, nlev) + real(kind=kind_phys) :: effrg_local(ncol, nlev) + real(kind=kind_phys) :: ncg_in_local(ncol, nlev) + real(kind=kind_phys) :: nci_out_local(ncol, nlev) + + errmsg = '' + errflg = 0 + + effrr_local = effrr_in + if (present(effrg_in)) effrg_local = effrg_in + if (present(ncg_in)) ncg_in_local = ncg_in + if (present(nci_out)) nci_out_local = nci_out + effrl_inout = min(max(effrl_inout, re_qc_min), re_qc_max) + if (present(effri_out)) effri_out = re_qi_avg + effrs_inout = effrs_inout + (10.0 / 6.0) ! in micrometer + scalar_var = 2.0 ! in km + + end subroutine effr_calc_run + +end module effr_calc diff --git a/end-to-end-tests/nested_suite/effr_calc.meta b/end-to-end-tests/nested_suite/effr_calc.meta new file mode 100644 index 00000000..6361eac6 --- /dev/null +++ b/end-to-end-tests/nested_suite/effr_calc.meta @@ -0,0 +1,163 @@ +[ccpp-table-properties] + name = effr_calc + type = scheme + dependencies = +######################################################################## +[ccpp-arg-table] + name = effr_calc_init + type = scheme +[ scheme_order ] + standard_name = scheme_order_in_suite + long_name = scheme order in suite definition file + units = None + dimensions = () + type = integer + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +######################################################################## +[ccpp-arg-table] + name = effr_calc_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ nlev ] + standard_name = vertical_layer_dimension + type = integer + units = count + dimensions = () + intent = in +[effrr_in] + standard_name = effective_radius_of_stratiform_cloud_rain_particle + long_name = effective radius of cloud rain particle in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + top_at_one = True +[effrg_in] + standard_name = effective_radius_of_stratiform_cloud_graupel + long_name = effective radius of cloud graupel in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + optional = True +[ncg_in] + standard_name = cloud_graupel_number_concentration + long_name = number concentration of cloud graupel + units = kg-1 + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + optional = True +[nci_out] + standard_name = cloud_ice_number_concentration + long_name = number concentration of cloud ice + units = kg-1 + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = out + optional = True +[effrl_inout] + standard_name = effective_radius_of_stratiform_cloud_liquid_water_particle + long_name = effective radius of cloud liquid water particle in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[effri_out] + standard_name = effective_radius_of_stratiform_cloud_ice_particle + long_name = effective radius of cloud ice water particle in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = out + optional = True +[effrs_inout] + standard_name = effective_radius_of_stratiform_cloud_snow_particle + long_name = effective radius of cloud snow particle in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = 8 + intent = inout + top_at_one = True +[ncl_out] + standard_name = cloud_liquid_number_concentration + long_name = number concentration of cloud liquid + units = kg-1 + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = out + optional = True +[has_graupel] + standard_name = flag_indicating_cloud_microphysics_has_graupel + long_name = flag indicating that the cloud microphysics produces graupel + units = flag + dimensions = () + type = logical + intent = in +[ scalar_var ] + standard_name = scalar_variable_for_testing + long_name = scalar variable for testing + units = km + dimensions = () + type = real + kind = kind_phys + intent = inout +[ tke_inout ] + standard_name = turbulent_kinetic_energy + long_name = turbulent_kinetic_energy + units = m2 s-2 + dimensions = () + type = real + kind = kind_phys + intent = inout +[ tke2_inout ] + standard_name = turbulent_kinetic_energy2 + long_name = turbulent_kinetic_energy2 + units = m+2 s-2 + dimensions = () + type = real + kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/nested_suite/effr_diag.F90 b/end-to-end-tests/nested_suite/effr_diag.F90 new file mode 100644 index 00000000..75da29c7 --- /dev/null +++ b/end-to-end-tests/nested_suite/effr_diag.F90 @@ -0,0 +1,68 @@ +!Test unit conversions for intent in, inout, out variables +! + +module effr_diag + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: effr_diag_run, effr_diag_init + +contains + + !> \section arg_table_effr_diag_init Argument Table + !! \htmlinclude arg_table_effr_diag_init.html + !! + subroutine effr_diag_init(scheme_order, errmsg, errflg) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: scheme_order + + errmsg = '' + errflg = 0 + + if (scheme_order /= 4) then + errflg = 1 + errmsg = 'ERROR: effr_diag_init() needs to be called fourth' + return + else + scheme_order = scheme_order + 1 + end if + + end subroutine effr_diag_init + + !> \section arg_table_effr_diag_run Argument Table + !! \htmlinclude arg_table_effr_diag_run.html + !! + subroutine effr_diag_run(effrr_in, scalar_var, errmsg, errflg) + + real(kind=kind_phys), intent(in) :: effrr_in(:, :) + integer, intent(in) :: scalar_var + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + real(kind=kind_phys) :: effrr_min, effrr_max + + errmsg = '' + errflg = 0 + + call cmp_effr_diag(effrr_in, effrr_min, effrr_max) + + if (scalar_var /= 380) then + errmsg = 'ERROR: effr_diag_run(): scalar_var should be 380' + errflg = 1 + end if + end subroutine effr_diag_run + + subroutine cmp_effr_diag(effr, effr_min, effr_max) + real(kind=kind_phys), intent(in) :: effr(:, :) + real(kind=kind_phys), intent(out) :: effr_min, effr_max + + ! Do some diagnostic calcualtions... + effr_min = minval(effr) + effr_max = maxval(effr) + + end subroutine cmp_effr_diag +end module effr_diag diff --git a/end-to-end-tests/nested_suite/effr_diag.meta b/end-to-end-tests/nested_suite/effr_diag.meta new file mode 100644 index 00000000..5a5c9e67 --- /dev/null +++ b/end-to-end-tests/nested_suite/effr_diag.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = effr_diag + type = scheme + dependencies = +######################################################################## +[ccpp-arg-table] + name = effr_diag_init + type = scheme +[ scheme_order ] + standard_name = scheme_order_in_suite + long_name = scheme order in suite definition file + units = None + dimensions = () + type = integer + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +######################################################################## +[ccpp-arg-table] + name = effr_diag_run + type = scheme +[effrr_in] + standard_name = effective_radius_of_stratiform_cloud_rain_particle + long_name = effective radius of cloud rain particle in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + top_at_one = True +[ scalar_var ] + standard_name = scalar_variable_for_testing_c + long_name = unused scalar variable C + units = m + dimensions = () + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/nested_suite/effr_post.F90 b/end-to-end-tests/nested_suite/effr_post.F90 new file mode 100644 index 00000000..01357350 --- /dev/null +++ b/end-to-end-tests/nested_suite/effr_post.F90 @@ -0,0 +1,61 @@ +!Test unit conversions for intent in, inout, out variables +! + +module effr_post + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: effr_post_run, effr_post_init + +contains + + !> \section arg_table_effr_post_init Argument Table + !! \htmlinclude arg_table_effr_post_init.html + !! + subroutine effr_post_init(scheme_order, errmsg, errflg) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: scheme_order + + errmsg = '' + errflg = 0 + + if (scheme_order /= 3) then + errflg = 1 + errmsg = 'ERROR: effr_post_init() needs to be called third' + return + else + scheme_order = scheme_order + 1 + end if + + end subroutine effr_post_init + + !> \section arg_table_effr_post_run Argument Table + !! \htmlinclude arg_table_effr_post_run.html + !! + subroutine effr_post_run(effrr_inout, scalar_var, errmsg, errflg) + + real(kind=kind_phys), intent(inout) :: effrr_inout(:, :) + real(kind=kind_phys), intent(in) :: scalar_var + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + real(kind=kind_phys) :: effrr_min, effrr_max + + errmsg = '' + errflg = 0 + + ! Do some post-processing on effrr... + effrr_inout(:, :) = effrr_inout(:, :) * 1._kind_phys + + if (scalar_var /= 1013.0) then + errmsg = 'ERROR: effr_post_run(): scalar_var should be 1013.0' + errflg = 1 + end if + + end subroutine effr_post_run + +end module effr_post diff --git a/end-to-end-tests/nested_suite/effr_post.meta b/end-to-end-tests/nested_suite/effr_post.meta new file mode 100644 index 00000000..703b5ebc --- /dev/null +++ b/end-to-end-tests/nested_suite/effr_post.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = effr_post + type = scheme + dependencies = +######################################################################## +[ccpp-arg-table] + name = effr_post_init + type = scheme +[ scheme_order ] + standard_name = scheme_order_in_suite + long_name = scheme order in suite definition file + units = None + dimensions = () + type = integer + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +######################################################################## +[ccpp-arg-table] + name = effr_post_run + type = scheme +[effrr_inout] + standard_name = effective_radius_of_stratiform_cloud_rain_particle + long_name = effective radius of cloud rain particle in micrometer + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ scalar_var ] + standard_name = scalar_variable_for_testing_b + long_name = unused scalar variable B + units = m + dimensions = () + type = real + kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/nested_suite/effr_pre.F90 b/end-to-end-tests/nested_suite/effr_pre.F90 new file mode 100644 index 00000000..a2fe2f5c --- /dev/null +++ b/end-to-end-tests/nested_suite/effr_pre.F90 @@ -0,0 +1,60 @@ +!Test unit conversions for intent in, inout, out variables +! + +module mod_effr_pre + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: effr_pre_run, effr_pre_init + +contains + !> \section arg_table_effr_pre_init Argument Table + !! \htmlinclude arg_table_effr_pre_init.html + !! + subroutine effr_pre_init(scheme_order, errmsg, errflg) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: scheme_order + + errmsg = '' + errflg = 0 + + if (scheme_order /= 1) then + errflg = 1 + errmsg = 'ERROR: effr_pre_init() needs to be called first' + return + else + scheme_order = scheme_order + 1 + end if + + end subroutine effr_pre_init + + !> \section arg_table_effr_pre_run Argument Table + !! \htmlinclude arg_table_effr_pre_run.html + !! + subroutine effr_pre_run(effrr_inout, scalar_var, errmsg, errflg) + + real(kind=kind_phys), intent(inout) :: effrr_inout(:, :) + real(kind=kind_phys), intent(in) :: scalar_var + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + real(kind=kind_phys) :: effrr_min, effrr_max + + errmsg = '' + errflg = 0 + + ! Do some pre-processing on effrr... + effrr_inout(:, :) = effrr_inout(:, :) * 1._kind_phys + + if (scalar_var /= 273.15) then + errmsg = 'ERROR: effr_pre_run(): scalar_var should be 273.15' + errflg = 1 + end if + + end subroutine effr_pre_run + +end module mod_effr_pre diff --git a/end-to-end-tests/nested_suite/effr_pre.meta b/end-to-end-tests/nested_suite/effr_pre.meta new file mode 100644 index 00000000..c47d1abf --- /dev/null +++ b/end-to-end-tests/nested_suite/effr_pre.meta @@ -0,0 +1,66 @@ +[ccpp-table-properties] + name = effr_pre + type = scheme + module_name = mod_effr_pre + dependencies = +######################################################################## +[ccpp-arg-table] + name = effr_pre_init + type = scheme +[ scheme_order ] + standard_name = scheme_order_in_suite + long_name = scheme order in suite definition file + units = None + dimensions = () + type = integer + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +######################################################################## +[ccpp-arg-table] + name = effr_pre_run + type = scheme +[effrr_inout] + standard_name = effective_radius_of_stratiform_cloud_rain_particle + long_name = effective radius of cloud rain particle in micrometer + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ scalar_var ] + standard_name = scalar_variable_for_testing_a + long_name = unused scalar variable A + units = m + dimensions = () + type = real + kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/nested_suite/effrs_calc.F90 b/end-to-end-tests/nested_suite/effrs_calc.F90 new file mode 100644 index 00000000..3aa8d196 --- /dev/null +++ b/end-to-end-tests/nested_suite/effrs_calc.F90 @@ -0,0 +1,32 @@ +!Test unit conversions for intent in, inout, out variables +! + +module effrs_calc + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: effrs_calc_run + +contains + !> \section arg_table_effrs_calc_run Argument Table + !! \htmlinclude arg_table_effrs_calc_run.html + !! + subroutine effrs_calc_run(effrs_inout, errmsg, errflg) + + real(kind=kind_phys), intent(inout) :: effrs_inout(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + !---------------------------------------------------------------- + + errmsg = '' + errflg = 0 + + effrs_inout = effrs_inout + (10.E-6_kind_phys / 3._kind_phys) ! in meters + + end subroutine effrs_calc_run + +end module effrs_calc diff --git a/end-to-end-tests/nested_suite/effrs_calc.meta b/end-to-end-tests/nested_suite/effrs_calc.meta new file mode 100644 index 00000000..e2fd1de9 --- /dev/null +++ b/end-to-end-tests/nested_suite/effrs_calc.meta @@ -0,0 +1,25 @@ +[ccpp-table-properties] + name = effrs_calc + type = scheme + +[ccpp-arg-table] + name = effrs_calc_run + type = scheme +[ effrs_inout ] + standard_name = effective_radius_of_stratiform_cloud_snow_particle + units = m + type = real | kind = kind_phys + dimensions = (horizontal_dimension,vertical_layer_dimension) + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + units = none + type = character | kind = len=512 + dimensions = () + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + type = integer + dimensions = () + intent = out diff --git a/end-to-end-tests/nested_suite/main_suite.xml b/end-to-end-tests/nested_suite/main_suite.xml new file mode 100644 index 00000000..be2d0d07 --- /dev/null +++ b/end-to-end-tests/nested_suite/main_suite.xml @@ -0,0 +1,20 @@ + + + + suite_lifecycle + + + effr_pre + + + effr_calc + + + effr_post + + + + + + suite_lifecycle + diff --git a/end-to-end-tests/nested_suite/module_rad_ddt.F90 b/end-to-end-tests/nested_suite/module_rad_ddt.F90 new file mode 100644 index 00000000..6e992250 --- /dev/null +++ b/end-to-end-tests/nested_suite/module_rad_ddt.F90 @@ -0,0 +1,23 @@ +module mod_rad_ddt + use ccpp_kinds, only: kind_phys + implicit none + + public ty_rad_lw, ty_rad_sw + + !> \section arg_table_ty_rad_lw Argument Table + !! \htmlinclude arg_table_ty_rad_lw.html + !! + type ty_rad_lw + real(kind=kind_phys) :: sfc_up_lw + real(kind=kind_phys) :: sfc_down_lw + end type ty_rad_lw + + !> \section arg_table_ty_rad_sw Argument Table + !! \htmlinclude arg_table_ty_rad_sw.html + !! + type ty_rad_sw + real(kind=kind_phys), pointer :: sfc_up_sw(:) => null() + real(kind=kind_phys), pointer :: sfc_down_sw(:) => null() + end type ty_rad_sw + +end module mod_rad_ddt diff --git a/end-to-end-tests/nested_suite/module_rad_ddt.meta b/end-to-end-tests/nested_suite/module_rad_ddt.meta new file mode 100644 index 00000000..c4792547 --- /dev/null +++ b/end-to-end-tests/nested_suite/module_rad_ddt.meta @@ -0,0 +1,40 @@ +[ccpp-table-properties] + name = ty_rad_lw + type = ddt + dependencies = + module_name = mod_rad_ddt +[ccpp-arg-table] + name = ty_rad_lw + type = ddt +[ sfc_up_lw ] + standard_name = surface_upwelling_longwave_radiation_flux + units = W m2 + dimensions = () + type = real + kind = kind_phys +[ sfc_down_lw ] + standard_name = surface_downwelling_longwave_radiation_flux + units = W m2 + dimensions = () + type = real + kind = kind_phys + +[ccpp-table-properties] + name = ty_rad_sw + type = ddt + module_name = mod_rad_ddt +[ccpp-arg-table] + name = ty_rad_sw + type = ddt +[ sfc_up_sw ] + standard_name = surface_upwelling_shortwave_radiation_flux + units = W m2 + dimensions = (horizontal_dimension) + type = real + kind = kind_phys +[ sfc_down_sw ] + standard_name = surface_downwelling_shortwave_radiation_flux + units = W m2 + dimensions = (horizontal_dimension) + type = real + kind = kind_phys diff --git a/end-to-end-tests/nested_suite/rad_lw.F90 b/end-to-end-tests/nested_suite/rad_lw.F90 new file mode 100644 index 00000000..ded4861f --- /dev/null +++ b/end-to-end-tests/nested_suite/rad_lw.F90 @@ -0,0 +1,35 @@ +module rad_lw + use ccpp_kinds, only: kind_phys + use mod_rad_ddt, only: ty_rad_lw + + implicit none + private + + public :: rad_lw_run + +contains + + !> \section arg_table_rad_lw_run Argument Table + !! \htmlinclude arg_table_rad_lw_run.html + !! + subroutine rad_lw_run(ncol, fluxlw, errmsg, errflg) + + integer, intent(in) :: ncol + type(ty_rad_lw), intent(inout) :: fluxlw(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! Locals + integer :: icol + + errmsg = '' + errflg = 0 + + do icol = 1, ncol + fluxlw(icol)%sfc_up_lw = 300._kind_phys + fluxlw(icol)%sfc_down_lw = 50._kind_phys + end do + + end subroutine rad_lw_run + +end module rad_lw diff --git a/end-to-end-tests/nested_suite/rad_lw.meta b/end-to-end-tests/nested_suite/rad_lw.meta new file mode 100644 index 00000000..bfab7426 --- /dev/null +++ b/end-to-end-tests/nested_suite/rad_lw.meta @@ -0,0 +1,35 @@ +[ccpp-table-properties] + name = rad_lw + type = scheme + dependencies = module_rad_ddt.F90 +[ccpp-arg-table] + name = rad_lw_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[fluxLW] + standard_name = longwave_radiation_fluxes + long_name = longwave radiation fluxes + units = W m-2 + dimensions = (horizontal_dimension) + type = ty_rad_lw + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/nested_suite/rad_sw.F90 b/end-to-end-tests/nested_suite/rad_sw.F90 new file mode 100644 index 00000000..64756217 --- /dev/null +++ b/end-to-end-tests/nested_suite/rad_sw.F90 @@ -0,0 +1,35 @@ +module rad_sw + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: rad_sw_run + +contains + + !> \section arg_table_rad_sw_run Argument Table + !! \htmlinclude arg_table_rad_sw_run.html + !! + subroutine rad_sw_run(ncol, sfc_up_sw, sfc_down_sw, errmsg, errflg) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(inout) :: sfc_up_sw(:) + real(kind=kind_phys), intent(inout) :: sfc_down_sw(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! Locals + integer :: icol + + errmsg = '' + errflg = 0 + + do icol = 1, ncol + sfc_up_sw(icol) = 100._kind_phys + sfc_down_sw(icol) = 400._kind_phys + end do + + end subroutine rad_sw_run + +end module rad_sw diff --git a/end-to-end-tests/nested_suite/rad_sw.meta b/end-to-end-tests/nested_suite/rad_sw.meta new file mode 100644 index 00000000..af88530f --- /dev/null +++ b/end-to-end-tests/nested_suite/rad_sw.meta @@ -0,0 +1,41 @@ +[ccpp-table-properties] + name = rad_sw + type = scheme +[ccpp-arg-table] + name = rad_sw_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ sfc_up_sw ] + standard_name = surface_upwelling_shortwave_radiation_flux + units = W m2 + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[ sfc_down_sw ] + standard_name = surface_downwelling_shortwave_radiation_flux + units = W m2 + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/nested_suite/radiation2_suite.xml b/end-to-end-tests/nested_suite/radiation2_suite.xml new file mode 100644 index 00000000..e20b81e8 --- /dev/null +++ b/end-to-end-tests/nested_suite/radiation2_suite.xml @@ -0,0 +1,10 @@ + + + + + + effrs_calc + + effr_diag + + diff --git a/end-to-end-tests/nested_suite/radiation3_subsuite.xml b/end-to-end-tests/nested_suite/radiation3_subsuite.xml new file mode 100644 index 00000000..346db62d --- /dev/null +++ b/end-to-end-tests/nested_suite/radiation3_subsuite.xml @@ -0,0 +1,7 @@ + + + + + rad_sw + + diff --git a/end-to-end-tests/nested_suite/radiation3_suite.xml b/end-to-end-tests/nested_suite/radiation3_suite.xml new file mode 100644 index 00000000..89e5bc13 --- /dev/null +++ b/end-to-end-tests/nested_suite/radiation3_suite.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/end-to-end-tests/nested_suite/radiation4_suite.xml b/end-to-end-tests/nested_suite/radiation4_suite.xml new file mode 100644 index 00000000..d3df4fb9 --- /dev/null +++ b/end-to-end-tests/nested_suite/radiation4_suite.xml @@ -0,0 +1,7 @@ + + + + + rad_lw + + diff --git a/end-to-end-tests/nested_suite/suite_lifecycle.F90 b/end-to-end-tests/nested_suite/suite_lifecycle.F90 new file mode 100644 index 00000000..09b85670 --- /dev/null +++ b/end-to-end-tests/nested_suite/suite_lifecycle.F90 @@ -0,0 +1,34 @@ +module suite_lifecycle + + implicit none + private + + public :: suite_lifecycle_init, suite_lifecycle_final + +contains + + !> \section arg_table_suite_lifecycle_init Argument Table + !! \htmlinclude arg_table_suite_lifecycle_init.html + !! + subroutine suite_lifecycle_init(counter, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: counter + errmsg = '' + errflg = 0 + counter = counter + 1 + end subroutine suite_lifecycle_init + + !> \section arg_table_suite_lifecycle_final Argument Table + !! \htmlinclude arg_table_suite_lifecycle_final.html + !! + subroutine suite_lifecycle_final(counter, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: counter + errmsg = '' + errflg = 0 + counter = counter + 1 + end subroutine suite_lifecycle_final + +end module suite_lifecycle diff --git a/end-to-end-tests/nested_suite/suite_lifecycle.meta b/end-to-end-tests/nested_suite/suite_lifecycle.meta new file mode 100644 index 00000000..673e348d --- /dev/null +++ b/end-to-end-tests/nested_suite/suite_lifecycle.meta @@ -0,0 +1,49 @@ +[ccpp-table-properties] + name = suite_lifecycle + type = scheme + +[ccpp-arg-table] + name = suite_lifecycle_init + type = scheme +[counter] + standard_name = lifecycle_counter + units = 1 + dimensions = () + type = integer + intent = inout +[errmsg] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[errflg] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = suite_lifecycle_final + type = scheme +[counter] + standard_name = lifecycle_counter + units = 1 + dimensions = () + type = integer + intent = inout +[errmsg] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[errflg] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/nested_suite/test_host.F90 b/end-to-end-tests/nested_suite/test_host.F90 new file mode 100644 index 00000000..9871a827 --- /dev/null +++ b/end-to-end-tests/nested_suite/test_host.F90 @@ -0,0 +1,314 @@ +module test_prog + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public test_host + + ! Public data and interfaces + integer, public, parameter :: cs = 32 + integer, public, parameter :: cm = 60 + + !> \section arg_table_suite_info Argument Table + !! \htmlinclude arg_table_suite_info.html + !! + type, public :: suite_info + character(len=cs) :: suite_name = '' + character(len=cs), pointer :: suite_parts(:) => null() + character(len=cm), pointer :: suite_input_vars(:) => null() + character(len=cm), pointer :: suite_output_vars(:) => null() + character(len=cm), pointer :: suite_required_vars(:) => null() + end type suite_info + +contains + + logical function check_suite(test_suite) + use ccpp_static_api, only: ccpp_physics_suite_part_list + use ccpp_static_api, only: ccpp_physics_suite_variables + use test_utils, only: check_list + + ! Dummy argument + type(suite_info), intent(in) :: test_suite + ! Local variables + integer :: sind + logical :: check + integer :: errflg + character(len=512) :: errmsg + character(len=128), allocatable :: test_list(:) + + check_suite = .true. + write(6, *) "Checking suite ", trim(test_suite%suite_name) + ! First, check the suite parts + call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_parts, 'part names', & + suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the input variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.true., output_vars=.false.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_input_vars, & + 'input variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the output variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.false., output_vars=.true.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_output_vars, & + 'output variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check all required variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_required_vars, & + 'required variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + end function check_suite + + !> \section arg_table_test_host Argument Table + !! \htmlinclude arg_table_test_host.html + !! + subroutine test_host(retval, test_suites) + + use test_host_mod, only: ncols + use ccpp_static_api, only: ccpp_register + use ccpp_static_api, only: ccpp_init + use ccpp_static_api, only: ccpp_physics_init + use ccpp_static_api, only: ccpp_physics_timestep_init + use ccpp_static_api, only: ccpp_physics_run + use ccpp_static_api, only: ccpp_physics_timestep_final + use ccpp_static_api, only: ccpp_physics_final + use ccpp_static_api, only: ccpp_final + use ccpp_static_api, only: ccpp_physics_suite_list + use test_host_mod, only: init_data, & + compare_data + use test_utils, only: check_list + + type(suite_info), intent(in) :: test_suites(:) + logical, intent(out) :: retval + + logical :: check + integer :: col_start, col_end + integer :: index, sind + integer :: num_suites + character(len=128), allocatable :: suite_names(:) + character(len=512) :: errmsg + integer :: errflg + + ! Initialize our 'data' + call init_data() + + ! Gather and test the inspection routines + num_suites = size(test_suites) + call ccpp_physics_suite_list(suite_names) + retval = check_list(suite_names, test_suites(:)%suite_name, & + 'suite names') + write(6, *) 'Available suites are:' + do index = 1, size(suite_names) + do sind = 1, num_suites + if (trim(test_suites(sind)%suite_name) == & + trim(suite_names(index))) then + exit + end if + end do + write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & + ' = test_suites(', sind, ')' + end do + if (retval) then + do sind = 1, num_suites + check = check_suite(test_suites(sind)) + retval = retval .and. check + end do + end if + !!! Return here if any check failed + if (.not. retval) then + return + end if + + ! Use the suite information to register + do sind = 1, num_suites + call ccpp_register( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Use the suite information to initialize + do sind = 1, num_suites + call ccpp_init( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Use the suite information to setup the run + do sind = 1, num_suites + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Initialize the timestep + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(2a)') 'An error occurred in ccpp_physics_timestep_init, ', & + 'Exiting...' + exit + end if + end do + + do col_start = 1, ncols, 5 + if (errflg /= 0) then + exit + end if + col_end = min(col_start + 4, ncols) + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + do index = 1, size(test_suites(sind)%suite_parts) + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + col_start=col_start, col_end=col_end, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(5a)') trim(test_suites(sind)%suite_name), & + '/', trim(test_suites(sind)%suite_parts(index)), & + ': ', trim(errmsg) + exit + end if + end do + end do + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(2a)') 'An error occurred in ccpp_physics_timestep_final, ', & + 'Exiting...' + exit + end if + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & + 'Exiting...' + exit + end if + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_final( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(2a)') 'An error occurred in ccpp_final, ', & + 'Exiting...' + exit + end if + end do + + if (errflg == 0) then + ! Run finished without error, check answers + if (compare_data()) then + write(6, *) 'Answers are correct!' + errflg = 0 + else + write(6, *) 'Answers are not correct!' + errflg = -1 + end if + end if + + retval = errflg == 0 + + end subroutine test_host + +end module test_prog diff --git a/end-to-end-tests/nested_suite/test_host.meta b/end-to-end-tests/nested_suite/test_host.meta new file mode 100644 index 00000000..076cd7d6 --- /dev/null +++ b/end-to-end-tests/nested_suite/test_host.meta @@ -0,0 +1,63 @@ +[ccpp-table-properties] + name = test_host + type = control +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/end-to-end-tests/nested_suite/test_host_data.F90 b/end-to-end-tests/nested_suite/test_host_data.F90 new file mode 100644 index 00000000..ece60034 --- /dev/null +++ b/end-to-end-tests/nested_suite/test_host_data.F90 @@ -0,0 +1,103 @@ +module test_host_data + + use ccpp_kinds, only: kind_phys + use mod_rad_ddt, only: ty_rad_lw, & + ty_rad_sw + + implicit none + private + + !> \section arg_table_physics_state Argument Table + !! \htmlinclude arg_table_physics_state.html + type physics_state + real(kind=kind_phys), dimension(:, :), allocatable :: & + effrr, & ! effective radius of cloud rain + effrl, & ! effective radius of cloud liquid water + effri, & ! effective radius of cloud ice + effrg, & ! effective radius of cloud graupel + ncg, & ! number concentration of cloud graupel + nci ! number concentration of cloud ice + real(kind=kind_phys) :: scalar_var + type(ty_rad_lw), dimension(:), allocatable :: & + fluxlw ! Longwave radiation fluxes + type(ty_rad_sw) :: & + fluxsw ! Shortwave radiation fluxes + real(kind=kind_phys) :: scalar_vara + real(kind=kind_phys) :: scalar_varb + real(kind=kind_phys) :: tke, tke2 + integer :: scalar_varc + integer :: scheme_order + integer :: num_subcycles + end type physics_state + + public :: physics_state + public :: allocate_physics_state + +contains + + subroutine allocate_physics_state(cols, levels, state, has_graupel, has_ice) + integer, intent(in) :: cols + integer, intent(in) :: levels + type(physics_state), intent(out) :: state + logical, intent(in) :: has_graupel + logical, intent(in) :: has_ice + + if (allocated(state%effrr)) then + deallocate(state%effrr) + end if + allocate(state%effrr(cols, levels)) + + if (allocated(state%effrl)) then + deallocate(state%effrl) + end if + allocate(state%effrl(cols, levels)) + + if (has_ice) then + if (allocated(state%effri)) then + deallocate(state%effri) + end if + allocate(state%effri(cols, levels)) + end if + + if (has_graupel) then + if (allocated(state%effrg)) then + deallocate(state%effrg) + end if + allocate(state%effrg(cols, levels)) + + if (allocated(state%ncg)) then + deallocate(state%ncg) + end if + allocate(state%ncg(cols, levels)) + end if + + if (has_ice) then + if (allocated(state%nci)) then + deallocate(state%nci) + end if + allocate(state%nci(cols, levels)) + end if + + if (allocated(state%fluxlw)) then + deallocate(state%fluxlw) + end if + allocate(state%fluxlw(cols)) + + if (associated(state%fluxsw%sfc_up_sw)) then + nullify(state%fluxsw%sfc_up_sw) + end if + allocate(state%fluxsw%sfc_up_sw(cols)) + + if (associated(state%fluxsw%sfc_down_sw)) then + nullify(state%fluxsw%sfc_down_sw) + end if + allocate(state%fluxsw%sfc_down_sw(cols)) + + ! Initialize scheme counter. + state%scheme_order = 1 + ! Initialize subcycle counter. + state%num_subcycles = 3 + + end subroutine allocate_physics_state + +end module test_host_data diff --git a/end-to-end-tests/nested_suite/test_host_data.meta b/end-to-end-tests/nested_suite/test_host_data.meta new file mode 100644 index 00000000..fd5c009c --- /dev/null +++ b/end-to-end-tests/nested_suite/test_host_data.meta @@ -0,0 +1,129 @@ +[ccpp-table-properties] + name = physics_state + type = ddt + dependencies = module_rad_ddt.F90 +[ccpp-arg-table] + name = physics_state + type = ddt +[effrr] + standard_name = effective_radius_of_stratiform_cloud_rain_particle + long_name = effective radius of cloud rain particle in meter + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys +[effrl] + standard_name = effective_radius_of_stratiform_cloud_liquid_water_particle + long_name = effective radius of cloud liquid water particle in meter + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys +[effri] + standard_name = effective_radius_of_stratiform_cloud_ice_particle + long_name = effective radius of cloud ice water particle in meter + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + active = (flag_indicating_cloud_microphysics_has_ice) +[effrg] + standard_name = effective_radius_of_stratiform_cloud_graupel + long_name = effective radius of cloud graupel in meter + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + active = (flag_indicating_cloud_microphysics_has_graupel) +[ncg] + standard_name = cloud_graupel_number_concentration + long_name = number concentration of cloud graupel + units = kg-1 + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + active = (flag_indicating_cloud_microphysics_has_graupel) +[nci] + standard_name = cloud_ice_number_concentration + long_name = number concentration of cloud ice + units = kg-1 + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + active = (flag_indicating_cloud_microphysics_has_ice) +[scalar_var] + standard_name = scalar_variable_for_testing + long_name = unused scalar variable + units = m + dimensions = () + type = real + kind = kind_phys +[ tke ] + standard_name = turbulent_kinetic_energy + long_name = turbulent_kinetic_energy + units = J kg-1 + dimensions = () + type = real + kind = kind_phys +[ tke2 ] + standard_name = turbulent_kinetic_energy2 + long_name = turbulent_kinetic_energy2 + units = m2 s-2 + dimensions = () + type = real + kind = kind_phys +[fluxSW] + standard_name = shortwave_radiation_fluxes + long_name = shortwave radiation fluxes + units = W m-2 + dimensions = () + type = ty_rad_sw +[fluxLW] + standard_name = longwave_radiation_fluxes + long_name = longwave radiation fluxes + units = W m-2 + dimensions = (horizontal_dimension) + type = ty_rad_lw +[scalar_varA] + standard_name = scalar_variable_for_testing_a + long_name = unused scalar variable A + units = m + dimensions = () + type = real + kind = kind_phys +[scalar_varB] + standard_name = scalar_variable_for_testing_b + long_name = unused scalar variable B + units = m + dimensions = () + type = real + kind = kind_phys +[scalar_varC] + standard_name = scalar_variable_for_testing_c + long_name = unused scalar variable C + units = m + dimensions = () + type = integer +[scheme_order] + standard_name = scheme_order_in_suite + long_name = scheme order in suite definition file + units = None + dimensions = () + type = integer +[num_subcycles] + standard_name = num_subcycles_for_effr + long_name = Number of times to subcycle the effr calculation + units = None + dimensions = () + type = integer + +#[ccpp-table-properties] +# name = test_host_data +# type = host +# dependencies = module_rad_ddt.F90 +#[ccpp-arg-table] +# name = test_host_data +# type = host +# \ No newline at end of file diff --git a/end-to-end-tests/nested_suite/test_host_mod.F90 b/end-to-end-tests/nested_suite/test_host_mod.F90 new file mode 100644 index 00000000..dfe95d31 --- /dev/null +++ b/end-to-end-tests/nested_suite/test_host_mod.F90 @@ -0,0 +1,141 @@ +module test_host_mod + + use ccpp_kinds, only: kind_phys + use test_host_data, only: physics_state, & + allocate_physics_state + + implicit none + public + + !> \section arg_table_test_host_mod Argument Table + !! \htmlinclude arg_table_test_host_host.html + !! + integer, parameter :: ncols = 12 + integer, parameter :: pver = 4 + type(physics_state), target :: phys_state + real(kind=kind_phys) :: effrs(ncols, pver) + logical, parameter :: has_ice = .true. + logical, parameter :: has_graupel = .true. + integer :: lifecycle_counter + + public :: init_data + public :: compare_data + +contains + + subroutine init_data() + + ! Allocate and initialize state + call allocate_physics_state(ncols, pver, phys_state, has_graupel, has_ice) + phys_state%effrr = 1.0E-3 ! 1000 microns, in meter + phys_state%effrl = 1.0E-4 ! 100 microns, in meter + phys_state%scalar_var = 1.0 ! in m + phys_state%scalar_vara = 273.15 ! in K + phys_state%scalar_varb = 1013.0 ! in mb + phys_state%scalar_varc = 380 ! in ppmv + effrs = 5.0E-4 ! 500 microns, in meter + if (has_graupel) then + phys_state%effrg = 2.5E-4 ! 250 microns, in meter + phys_state%ncg = 40 + end if + if (has_ice) then + phys_state%effri = 5.0E-5 ! 50 microns, in meter + phys_state%nci = 80 + end if + phys_state%tke = 10.0 !J kg-1 + phys_state%tke2 = 42.0 !J kg-1 + lifecycle_counter = 0 + + end subroutine init_data + + logical function compare_data() + + real(kind=kind_phys), parameter :: effrr_expected = 1.0E-3 ! 1000 microns, in meter + real(kind=kind_phys), parameter :: effrl_expected = 5.0E-5 ! 50 microns, in meter + real(kind=kind_phys), parameter :: effri_expected = 7.5E-5 ! 75 microns, in meter + real(kind=kind_phys), parameter :: effrs_expected = 5.3E-4 ! 530 microns, in meter + real(kind=kind_phys), parameter :: scalar_expected = 2.0E3 ! 2 km, in meter + real(kind=kind_phys), parameter :: tke_expected = 10.0 ! 10 J kg-1 + real(kind=kind_phys), parameter :: tolerance = 1.0E-6 ! used as scaling factor for expected value + real(kind=kind_phys), parameter :: sfc_up_sw_expected = 100. ! W/m2 + real(kind=kind_phys), parameter :: sfc_down_sw_expected = 400. ! W/m2 + real(kind=kind_phys), parameter :: sfc_up_lw_expected = 300. ! W/m2 + real(kind=kind_phys), parameter :: sfc_down_lw_expected = 50. ! W/m2 + integer, parameter :: lifecycle_counter_expected = 2 + + compare_data = .true. + + if (maxval(abs(phys_state%effrr - effrr_expected)) > tolerance * effrr_expected) then + write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effrr from expected value exceeds tolerance: ', & + maxval(abs(phys_state%effrr - effrr_expected)), ' > ', tolerance * effrr_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%effrl - effrl_expected)) > tolerance * effrl_expected) then + write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effrl from expected value exceeds tolerance: ', & + maxval(abs(phys_state%effrl - effrl_expected)), ' > ', tolerance * effrl_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%effri - effri_expected)) > tolerance * effri_expected) then + write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effri from expected value exceeds tolerance: ', & + maxval(abs(phys_state%effri - effri_expected)), ' > ', tolerance * effri_expected + compare_data = .false. + end if + + if (maxval(abs(effrs - effrs_expected)) > tolerance * effrs_expected) then + write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of effrs from expected value exceeds tolerance: ', & + maxval(abs(effrs - effrs_expected)), ' > ', tolerance * effrs_expected + compare_data = .false. + end if + + if (abs(phys_state%scalar_var - scalar_expected) > tolerance * scalar_expected) then + write(6, '(a,e16.7,a,e16.7)') & + 'Error: max diff of scalar_var from expected value exceeds tolerance: ', & + abs(phys_state%scalar_var - scalar_expected), ' > ', tolerance * scalar_expected + compare_data = .false. + end if + + if (abs(phys_state%tke - tke_expected) > tolerance * tke_expected) then + write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of tke from expected value exceeds tolerance: ', & + abs(phys_state%tke - tke_expected), ' > ', tolerance * tke_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%fluxsw%sfc_up_sw - sfc_up_sw_expected)) > tolerance * sfc_up_sw_expected) then + write(6, '(a,e16.7,a,e16.7)') & + 'Error: max diff of sfc_up_sw from expected value exceeds tolerance: ', & + abs(phys_state%fluxsw%sfc_up_sw - sfc_up_sw_expected), ' > ', tolerance * sfc_up_sw_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%fluxsw%sfc_down_sw - sfc_down_sw_expected)) > tolerance * sfc_down_sw_expected) then + write(6, '(a,e16.7,a,e16.7)') & + 'Error: max diff of sfc_down_sw from expected value exceeds tolerance: ', & + abs(phys_state%fluxsw%sfc_down_sw - sfc_down_sw_expected), ' > ', tolerance * sfc_down_sw_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%fluxlw%sfc_up_lw - sfc_up_lw_expected)) > tolerance * sfc_up_lw_expected) then + write(6, '(a,e16.7,a,e16.7)') & + 'Error: max diff of sfc_up_lw from expected value exceeds tolerance: ', & + abs(phys_state%fluxlw%sfc_up_lw - sfc_up_lw_expected), ' > ', tolerance * sfc_up_lw_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%fluxlw%sfc_down_lw - sfc_down_lw_expected)) > tolerance * sfc_down_lw_expected) then + write(6, '(a,e16.7,a,e16.7)') & + 'Error: max diff of sfc_down_lw from expected value exceeds tolerance: ', & + abs(phys_state%fluxlw%sfc_down_lw - sfc_down_lw_expected), ' > ', tolerance * sfc_down_lw_expected + compare_data = .false. + end if + + if (lifecycle_counter /= lifecycle_counter_expected) then + write(6, '(a,i0,a,i0)') & + 'Error: lifecycle_counter does not match expected value: ', lifecycle_counter, ' vs ', lifecycle_counter_expected + compare_data = .false. + end if + + end function compare_data + +end module test_host_mod diff --git a/end-to-end-tests/nested_suite/test_host_mod.meta b/end-to-end-tests/nested_suite/test_host_mod.meta new file mode 100644 index 00000000..ab90ebb2 --- /dev/null +++ b/end-to-end-tests/nested_suite/test_host_mod.meta @@ -0,0 +1,47 @@ +[ccpp-table-properties] + name = test_host_mod + type = host +[ccpp-arg-table] + name = test_host_mod + type = host +[ ncols] + standard_name = horizontal_dimension + units = count + type = integer + protected = True + dimensions = () +[ pver ] + standard_name = vertical_layer_dimension + units = count + type = integer + protected = True + dimensions = () +[ phys_state ] + standard_name = physics_state_derived_type + long_name = Physics State DDT + type = physics_state + dimensions = () +[effrs] + standard_name = effective_radius_of_stratiform_cloud_snow_particle + long_name = effective radius of cloud snow particle in meter + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys +[has_ice] + standard_name = flag_indicating_cloud_microphysics_has_ice + long_name = flag indicating that the cloud microphysics produces ice + units = flag + dimensions = () + type = logical +[has_graupel] + standard_name = flag_indicating_cloud_microphysics_has_graupel + long_name = flag indicating that the cloud microphysics produces graupel + units = flag + dimensions = () + type = logical +[lifecycle_counter] + standard_name = lifecycle_counter + units = 1 + dimensions = () + type = integer diff --git a/end-to-end-tests/nested_suite/test_nested_suite_integration.F90 b/end-to-end-tests/nested_suite/test_nested_suite_integration.F90 new file mode 100644 index 00000000..5e9c3009 --- /dev/null +++ b/end-to-end-tests/nested_suite/test_nested_suite_integration.F90 @@ -0,0 +1,91 @@ +program test_nested_suite_integration + use test_prog, only: test_host, & + suite_info, & + cm, & + cs + + implicit none + + character(len=cs), target :: test_parts1(3) = (/ & + 'radiation1 ', & + 'rad_lw_group ', & + 'rad_sw_group '/) + + character(len=cm), target :: test_invars1(18) = (/ & + 'effective_radius_of_stratiform_cloud_rain_particle ', & + 'effective_radius_of_stratiform_cloud_liquid_water_particle', & + 'effective_radius_of_stratiform_cloud_snow_particle ', & + 'effective_radius_of_stratiform_cloud_graupel ', & + 'cloud_graupel_number_concentration ', & + 'scalar_variable_for_testing ', & + 'turbulent_kinetic_energy ', & + 'turbulent_kinetic_energy2 ', & + 'scalar_variable_for_testing_a ', & + 'scalar_variable_for_testing_b ', & + 'scalar_variable_for_testing_c ', & + 'scheme_order_in_suite ', & + 'num_subcycles_for_effr ', & + 'flag_indicating_cloud_microphysics_has_graupel ', & + 'flag_indicating_cloud_microphysics_has_ice ', & + 'surface_downwelling_shortwave_radiation_flux ', & + 'surface_upwelling_shortwave_radiation_flux ', & + 'longwave_radiation_fluxes '/) + + character(len=cm), target :: test_outvars1(14) = (/ & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'effective_radius_of_stratiform_cloud_ice_particle ', & + 'effective_radius_of_stratiform_cloud_liquid_water_particle', & + 'effective_radius_of_stratiform_cloud_rain_particle ', & + 'effective_radius_of_stratiform_cloud_snow_particle ', & + 'cloud_ice_number_concentration ', & + 'scalar_variable_for_testing ', & + 'scheme_order_in_suite ', & + 'surface_downwelling_shortwave_radiation_flux ', & + 'surface_upwelling_shortwave_radiation_flux ', & + 'turbulent_kinetic_energy ', & + 'turbulent_kinetic_energy2 ', & + 'longwave_radiation_fluxes '/) + + character(len=cm), target :: test_reqvars1(22) = (/ & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'effective_radius_of_stratiform_cloud_rain_particle ', & + 'effective_radius_of_stratiform_cloud_ice_particle ', & + 'effective_radius_of_stratiform_cloud_liquid_water_particle', & + 'effective_radius_of_stratiform_cloud_snow_particle ', & + 'effective_radius_of_stratiform_cloud_graupel ', & + 'cloud_graupel_number_concentration ', & + 'cloud_ice_number_concentration ', & + 'scalar_variable_for_testing ', & + 'turbulent_kinetic_energy ', & + 'turbulent_kinetic_energy2 ', & + 'scalar_variable_for_testing_a ', & + 'scalar_variable_for_testing_b ', & + 'scalar_variable_for_testing_c ', & + 'scheme_order_in_suite ', & + 'num_subcycles_for_effr ', & + 'flag_indicating_cloud_microphysics_has_graupel ', & + 'flag_indicating_cloud_microphysics_has_ice ', & + 'surface_downwelling_shortwave_radiation_flux ', & + 'surface_upwelling_shortwave_radiation_flux ', & + 'longwave_radiation_fluxes '/) + + type(suite_info) :: test_suites(1) + logical :: run_okay + + ! Setup expected test suite info + test_suites(1)%suite_name = 'main_suite' + test_suites(1)%suite_parts => test_parts1 + test_suites(1)%suite_input_vars => test_invars1 + test_suites(1)%suite_output_vars => test_outvars1 + test_suites(1)%suite_required_vars => test_reqvars1 + + call test_host(run_okay, test_suites) + + if (run_okay) then + stop 0 + else + stop -1 + end if +end program test_nested_suite_integration diff --git a/end-to-end-tests/opt_arg/CMakeLists.txt b/end-to-end-tests/opt_arg/CMakeLists.txt new file mode 100644 index 00000000..95ede75d --- /dev/null +++ b/end-to-end-tests/opt_arg/CMakeLists.txt @@ -0,0 +1,59 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "opt_arg_scheme") +set(HOST_FILES "data" "main") +set(SUITE_FILES "suite_opt_arg_suite.xml") +set(HOST "test_host") + +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES}) + +# Run ccpp_capgen_ng +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_opt_arg.x ${SCHEME_FORTRAN_FILES} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES}) +target_link_libraries(test_opt_arg.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_opt_arg.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_opt_arg.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_opt_arg + COMMAND test_opt_arg.x) diff --git a/end-to-end-tests/opt_arg/data.F90 b/end-to-end-tests/opt_arg/data.F90 new file mode 100644 index 00000000..621bb428 --- /dev/null +++ b/end-to-end-tests/opt_arg/data.F90 @@ -0,0 +1,21 @@ +module data + + !! \section arg_table_data Argument Table + !! \htmlinclude data.html + !! + use ccpp_kinds, only: kind_phys + + implicit none + + private + + public nx, flag_for_opt_arg, std_arg, opt_arg, opt_arg_2 + + integer, parameter :: nx = 3 + logical :: flag_for_opt_arg + + integer, dimension(nx) :: std_arg + integer, dimension(:), allocatable, target :: opt_arg + real(kind=kind_phys), dimension(:), allocatable, target :: opt_arg_2 + +end module data diff --git a/end-to-end-tests/opt_arg/data.meta b/end-to-end-tests/opt_arg/data.meta new file mode 100644 index 00000000..abbc2086 --- /dev/null +++ b/end-to-end-tests/opt_arg/data.meta @@ -0,0 +1,40 @@ +[ccpp-table-properties] + name = data + type = host + dependencies = +[ccpp-arg-table] + name = data + type = host +[nx] + standard_name = size_of_std_arg + long_name = size of std_arg + units = count + dimensions = () + type = integer +[std_arg] + standard_name = std_arg + long_name = mandatory variable + units = 1 + dimensions = (size_of_std_arg) + type = integer +[opt_arg] + standard_name = opt_arg + long_name = optional variable + units = 1 + dimensions = (size_of_std_arg) + type = integer + active = flag_for_opt_arg +[opt_arg_2] + standard_name = opt_arg_2 + long_name = optional variable with unit conversions + units = km + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + active = flag_for_opt_arg +[flag_for_opt_arg] + standard_name = flag_for_opt_arg + long_name = flag for optional variable + units = 1 + dimensions = () + type = logical diff --git a/end-to-end-tests/opt_arg/main.F90 b/end-to-end-tests/opt_arg/main.F90 new file mode 100644 index 00000000..b9928052 --- /dev/null +++ b/end-to-end-tests/opt_arg/main.F90 @@ -0,0 +1,152 @@ +program test_opt_arg + + use, intrinsic :: iso_fortran_env, only: output_unit, & + error_unit + + use data, only: nx, & + flag_for_opt_arg, & + std_arg, & + opt_arg, & + opt_arg_2 + + use ccpp_static_api, only: ccpp_register, & + ccpp_init, & + ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_final, & + ccpp_physics_final, & + ccpp_final + + implicit none + + character(len=*), parameter :: ccpp_suite = 'opt_arg_suite' + character(len=512) :: errmsg + integer :: errflg + + std_arg = 1 + flag_for_opt_arg = .true. + allocate(opt_arg(nx)) + allocate(opt_arg_2(nx)) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP register step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_register(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_register:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_init(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 1, opt_arg must all be 0 + write(output_unit, '(a)') "After ccpp_init: check std_arg(:)==1, opt_arg(:)==0, opt_arg_2(:)==0" + if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_init: std_arg=", std_arg + if (.not. all(opt_arg == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_init: opt_arg=", opt_arg + if (.not. all(opt_arg_2 == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_init: opt_arg_2=", opt_arg_2 + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_init(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 1, opt_arg must all be 0 + write(output_unit, '(a)') "PASS: After ccpp_physics_init: check std_arg(:)==1 and opt_arg(:)==0" + if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_init: std_arg=", std_arg + if (.not. all(opt_arg == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_init: opt_arg=", opt_arg + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep init step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_timestep_init(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_init:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 1, opt_arg must all be 2 + write(output_unit, '(a)') "PASS: After ccpp_physics_timestep_init: check std_arg(:)==1 and opt_arg(:)==2" + if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_init: std_arg=", std_arg + if (.not. all(opt_arg == 2)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_init: opt_arg=", opt_arg + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics run step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_run(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_run:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 1, opt_arg must all be 3 + write(output_unit, '(a)') "PASS: After ccpp_physics_run: check std_arg(:)==1 and opt_arg(:)==3" + if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_run: std_arg=", std_arg + if (.not. all(opt_arg == 3)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_run: opt_arg=", opt_arg + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + deallocate(opt_arg) + flag_for_opt_arg = .false. + + call ccpp_physics_timestep_final(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_timestep_final:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 7, opt_arg no longer allocated + write(output_unit, '(a)') "PASS: After ccpp_physics_timestep_final: check std_arg(:)==7; opt_arg not allocated" + if (.not. all(std_arg == 7)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_timestep_final: std_arg=", std_arg + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_final(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_physics_final:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + + ! std_arg must all be 7, opt_arg no longer allocated + write(output_unit, '(a)') "PASS: After ccpp_physics_final: check std_arg(:)==7; opt_arg not allocated" + if (.not. all(std_arg == 7)) write(error_unit, '(a,3i3)') "Error after ccpp_physics_final: std_arg=", std_arg + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP finalize step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_final(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + if (errflg/=0) then + write(error_unit, '(a)') "An error occurred in ccpp_final:" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + +end program test_opt_arg diff --git a/end-to-end-tests/opt_arg/main.meta b/end-to-end-tests/opt_arg/main.meta new file mode 100644 index 00000000..b90de81e --- /dev/null +++ b/end-to-end-tests/opt_arg/main.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer diff --git a/end-to-end-tests/opt_arg/opt_arg_scheme.F90 b/end-to-end-tests/opt_arg/opt_arg_scheme.F90 new file mode 100644 index 00000000..963b5264 --- /dev/null +++ b/end-to-end-tests/opt_arg/opt_arg_scheme.F90 @@ -0,0 +1,90 @@ +!>\file opt_arg_scheme.F90 +!! This file contains a opt_arg_scheme CCPP scheme that does nothing +!! except requesting the minimum, mandatory variables. + +module opt_arg_scheme + + use, intrinsic :: iso_fortran_env, only: error_unit + use ccpp_kinds, only: kind_phys + + implicit none + + private + public :: opt_arg_scheme_timestep_init, & + opt_arg_scheme_run, & + opt_arg_scheme_timestep_final + +contains + + !! \section arg_table_opt_arg_scheme_timestep_init Argument Table + !! \htmlinclude opt_arg_scheme_timestep_init.html + !! + subroutine opt_arg_scheme_timestep_init(nx, var, opt_var, opt_var_2, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: nx + integer, intent(in) :: var(:) + integer, optional, intent(out) :: opt_var(:) + real(kind=kind_phys), optional, intent(out) :: opt_var_2(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Initialize opt_var from var if opt_var if present + if (present(opt_var)) then + opt_var = 2 * var + end if + ! Initialize opt_var_2 from var if opt_var_2 present + if (present(opt_var_2)) then + opt_var_2 = 3.0_kind_phys * var + end if + end subroutine opt_arg_scheme_timestep_init + + !! \section arg_table_opt_arg_scheme_run Argument Table + !! \htmlinclude opt_arg_scheme_run.html + !! + subroutine opt_arg_scheme_run(nx, var, opt_var, opt_var_2, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: nx + integer, intent(in) :: var(:) + integer, optional, intent(inout) :: opt_var(:) + real(kind=kind_phys), optional, intent(inout) :: opt_var_2(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Update opt_var from var if opt_var present + if (present(opt_var)) then + opt_var = 3 * var + end if + ! Update opt_var_2 from var if opt_var_2 present + if (present(opt_var_2)) then + opt_var_2 = 4.0_kind_phys * var + end if + end subroutine opt_arg_scheme_run + + !! \section arg_table_opt_arg_scheme_timestep_final Argument Table + !! \htmlinclude opt_arg_scheme_timestep_finalize.html + !! + subroutine opt_arg_scheme_timestep_final(nx, var, opt_var, opt_var_2, errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(in) :: nx + integer, intent(inout) :: var(:) + integer, optional, intent(in) :: opt_var(:) + real(kind=kind_phys), optional, intent(inout) :: opt_var_2(:) + ! Initialize CCPP error handling variables + errmsg = '' + errflg = 0 + ! Update var from opt_var if opt_var is present + if (present(opt_var)) then + var = 4 * opt_var + else + var = 7 * var + end if + ! Update opt_var_2 if present + if (present(opt_var_2)) then + opt_var_2 = opt_var_2 + 5.0_kind_phys + end if + end subroutine opt_arg_scheme_timestep_final + +end module opt_arg_scheme diff --git a/end-to-end-tests/opt_arg/opt_arg_scheme.meta b/end-to-end-tests/opt_arg/opt_arg_scheme.meta new file mode 100644 index 00000000..c0c9a4bf --- /dev/null +++ b/end-to-end-tests/opt_arg/opt_arg_scheme.meta @@ -0,0 +1,157 @@ +[ccpp-table-properties] + name = opt_arg_scheme + type = scheme + dependencies = ccpp_kinds.F90 + +######################################################################## +[ccpp-arg-table] + name = opt_arg_scheme_timestep_init + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[nx] + standard_name = size_of_std_arg + long_name = size of std_arg + units = count + dimensions = () + type = integer + intent = in +[var] + standard_name = std_arg + long_name = mandatory variable + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = in +[opt_var] + standard_name = opt_arg + long_name = optional variable + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = out + optional = True +[opt_var_2] + standard_name = opt_arg_2 + long_name = optional variable with unit conversions + units = m + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + intent = out + optional = True + +######################################################################## +[ccpp-arg-table] + name = opt_arg_scheme_run + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[nx] + standard_name = size_of_std_arg + long_name = size of std_arg + units = count + dimensions = () + type = integer + intent = in +[var] + standard_name = std_arg + long_name = mandatory variable + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = in +[opt_var] + standard_name = opt_arg + long_name = optional variable + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = inout + optional = True +[opt_var_2] + standard_name = opt_arg_2 + long_name = optional variable with unit conversions + units = m + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + intent = inout + optional = True + +######################################################################## +[ccpp-arg-table] + name = opt_arg_scheme_timestep_final + type = scheme +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[nx] + standard_name = size_of_std_arg + long_name = size of std_arg + units = count + dimensions = () + type = integer + intent = in +[var] + standard_name = std_arg + long_name = mandatory variable + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = inout +[opt_var] + standard_name = opt_arg + long_name = optional variable + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = in + optional = True +[opt_var_2] + standard_name = opt_arg_2 + long_name = optional variable with unit conversions + units = m + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + intent = inout + optional = True diff --git a/end-to-end-tests/opt_arg/suite_opt_arg_suite.xml b/end-to-end-tests/opt_arg/suite_opt_arg_suite.xml new file mode 100644 index 00000000..b91ba5e7 --- /dev/null +++ b/end-to-end-tests/opt_arg/suite_opt_arg_suite.xml @@ -0,0 +1,9 @@ + + + + + + opt_arg_scheme + + + diff --git a/end-to-end-tests/utils/test_utils.F90 b/end-to-end-tests/utils/test_utils.F90 new file mode 100644 index 00000000..425a0c61 --- /dev/null +++ b/end-to-end-tests/utils/test_utils.F90 @@ -0,0 +1,100 @@ +module test_utils + + public :: check_list + +contains + logical function check_list(test_list, chk_list, list_desc, suite_name) + ! Check a list () against its expected value () + + ! Dummy arguments + character(len=*), intent(in) :: test_list(:) + character(len=*), intent(in) :: chk_list(:) + character(len=*), intent(in) :: list_desc + character(len=*), optional, intent(in) :: suite_name + + ! Local variables + logical :: found + integer :: num_items + integer :: lindex, tindex + integer, allocatable :: check_unique(:) + character(len=2) :: sep + character(len=256) :: errmsg + + check_list = .true. + errmsg = '' + + ! DH* + write(0,*) '' + do lindex = 1, size(chk_list) + write(errmsg, '(a,i4,1x,a)') 'Debug: chk_list', lindex, trim(chk_list(lindex)) + write(0,*) trim(errmsg) + end do + do lindex = 1, size(test_list) + write(errmsg, '(a,i4,1x,a)') 'Debug: test_list', lindex, trim(test_list(lindex)) + write(0,*) trim(errmsg) + end do + ! *DH + + ! Check the list size + num_items = size(chk_list) + if (size(test_list) /= num_items) then + write(errmsg, '(a,i0,2a)') 'ERROR: Found ', size(test_list), & + ' ', trim(list_desc) + if (present(suite_name)) then + write(errmsg(len_trim(errmsg) + 1:), '(2a)') ' for suite, ', & + trim(suite_name) + end if + write(errmsg(len_trim(errmsg) + 1:), '(a,i0)') ', should be ', num_items + write(6, *) trim(errmsg) + errmsg = '' + check_list = .false. + end if + + ! Now, check the list contents for 1-1 correspondence + if (check_list) then + allocate(check_unique(num_items)) + check_unique = -1 + do lindex = 1, num_items + found = .false. + do tindex = 1, num_items + if (trim(test_list(lindex)) == trim(chk_list(tindex))) then + check_unique(tindex) = lindex + found = .true. + exit + end if + end do + if (.not. found) then + check_list = .false. + write(errmsg, '(5a)') 'ERROR: ', trim(list_desc), ' item, ', & + trim(test_list(lindex)), ', was not found' + if (present(suite_name)) then + write(errmsg(len_trim(errmsg) + 1:), '(2a)') ' in suite, ', & + trim(suite_name) + end if + write(6, *) trim(errmsg) + errmsg = '' + end if + end do + if (check_list .and. any(check_unique < 0)) then + check_list = .false. + write(errmsg, '(3a)') 'ERROR: The following ', trim(list_desc), & + ' items were not found' + if (present(suite_name)) then + write(errmsg(len_trim(errmsg) + 1:), '(2a)') ' in suite, ', & + trim(suite_name) + end if + sep = '; ' + do lindex = 1, num_items + if (check_unique(lindex) < 0) then + write(errmsg(len_trim(errmsg) + 1:), '(2a)') sep, & + trim(chk_list(lindex)) + sep = ', ' + end if + end do + write(6, *) trim(errmsg) + errmsg = '' + end if + end if + + end function check_list +end module test_utils diff --git a/end-to-end-tests/var_compat/CMakeLists.txt b/end-to-end-tests/var_compat/CMakeLists.txt new file mode 100644 index 00000000..50d5c6d7 --- /dev/null +++ b/end-to-end-tests/var_compat/CMakeLists.txt @@ -0,0 +1,69 @@ + +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ +set(SCHEME_FILES "effr_calc" "effrs_calc" "effr_diag" "effr_pre" "effr_post" "rad_lw" "rad_sw") +set(HOST_FILES "module_rad_ddt" "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "var_compatibility_suite.xml") +set(HOST "test_host") + +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES}) + +# Run ccpp_capgen_ng +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_var_compat.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_var_compatibility_integration.F90 +) +target_link_libraries(test_var_compat.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_var_compat.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_var_compat.x PROPERTIES LINKER_LANGUAGE Fortran) + +add_test(NAME test_var_compat + COMMAND test_var_compat.x) diff --git a/end-to-end-tests/var_compat/README.md b/end-to-end-tests/var_compat/README.md new file mode 100644 index 00000000..d5573f3a --- /dev/null +++ b/end-to-end-tests/var_compat/README.md @@ -0,0 +1,23 @@ +# Variable Compatibility Test + +Tests the variable compatibility object (`VarCompatObj`): +- Unit conversions (forward & reverse) +- Vertical array flipping (`top_at_one=true`) +- Kind conversions (`kind_phys <-> 8`) +- And various combinations thereof of the above cases +- Also tests subcycles: + - Nested subcycles + - A subcycle with dynamic iteration length (defined by a standard name) and a subcycle with fixed/integer iteration length + - Multiple subcycles with same standard name defining the iteration length + - Nested subcycles with the same iteration length + +## Building/Running + +To explicitly build/run the variable compatibility test host, run: + +```bash +$ cmake --fresh -S -B -DCCPP_RUN_VAR_COMPATIBILITY_TEST=ON +$ cd +$ make +$ ctest +``` diff --git a/end-to-end-tests/var_compat/effr_calc.F90 b/end-to-end-tests/var_compat/effr_calc.F90 new file mode 100644 index 00000000..b8fc43ed --- /dev/null +++ b/end-to-end-tests/var_compat/effr_calc.F90 @@ -0,0 +1,84 @@ +!Test unit conversions for intent in, inout, out variables +! + +module effr_calc + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: effr_calc_run, effr_calc_init + +contains + !> \section arg_table_effr_calc_init Argument Table + !! \htmlinclude arg_table_effr_calc_init.html + !! + subroutine effr_calc_init(scheme_order, errmsg, errflg) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: scheme_order + + errmsg = '' + errflg = 0 + + if (scheme_order /= 2) then + errflg = 1 + errmsg = 'ERROR: effr_calc_init() needs to be called second' + return + else + scheme_order = scheme_order + 1 + end if + + end subroutine effr_calc_init + + !> \section arg_table_effr_calc_run Argument Table + !! \htmlinclude arg_table_effr_calc_run.html + !! + subroutine effr_calc_run(ncol, nlev, effrr_in, effrg_in, ncg_in, nci_out, & + effrl_inout, effri_out, effrs_inout, ncl_out, & + has_graupel, scalar_var, tke_inout, tke2_inout, & + errmsg, errflg) + + integer, intent(in) :: ncol + integer, intent(in) :: nlev + real(kind=kind_phys), intent(in) :: effrr_in(:, :) + real(kind=kind_phys), intent(in), optional :: effrg_in(:, :) + real(kind=kind_phys), intent(in), optional :: ncg_in(:, :) + real(kind=kind_phys), intent(out), optional :: nci_out(:, :) + real(kind=kind_phys), intent(inout) :: effrl_inout(:, :) + real(kind=kind_phys), intent(out), optional :: effri_out(:, :) + real(kind=8), intent(inout) :: effrs_inout(:, :) + logical, intent(in) :: has_graupel + real(kind=kind_phys), intent(inout) :: scalar_var + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + real(kind=kind_phys), intent(out), optional :: ncl_out(:, :) + real(kind=kind_phys), intent(inout) :: tke_inout + real(kind=kind_phys), intent(inout) :: tke2_inout + + !---------------------------------------------------------------- + + real(kind=kind_phys), parameter :: re_qc_min = 2.5 ! microns + real(kind=kind_phys), parameter :: re_qc_max = 50. ! microns + real(kind=kind_phys), parameter :: re_qi_avg = 75. ! microns + real(kind=kind_phys) :: effrr_local(ncol, nlev) + real(kind=kind_phys) :: effrg_local(ncol, nlev) + real(kind=kind_phys) :: ncg_in_local(ncol, nlev) + real(kind=kind_phys) :: nci_out_local(ncol, nlev) + + errmsg = '' + errflg = 0 + + effrr_local = effrr_in + if (present(effrg_in)) effrg_local = effrg_in + if (present(ncg_in)) ncg_in_local = ncg_in + if (present(nci_out)) nci_out_local = nci_out + effrl_inout = min(max(effrl_inout, re_qc_min), re_qc_max) + if (present(effri_out)) effri_out = re_qi_avg + effrs_inout = effrs_inout + (10.0 / 6.0) ! in micrometer + scalar_var = 2.0 ! in km + + end subroutine effr_calc_run + +end module effr_calc diff --git a/end-to-end-tests/var_compat/effr_calc.meta b/end-to-end-tests/var_compat/effr_calc.meta new file mode 100644 index 00000000..6361eac6 --- /dev/null +++ b/end-to-end-tests/var_compat/effr_calc.meta @@ -0,0 +1,163 @@ +[ccpp-table-properties] + name = effr_calc + type = scheme + dependencies = +######################################################################## +[ccpp-arg-table] + name = effr_calc_init + type = scheme +[ scheme_order ] + standard_name = scheme_order_in_suite + long_name = scheme order in suite definition file + units = None + dimensions = () + type = integer + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +######################################################################## +[ccpp-arg-table] + name = effr_calc_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ nlev ] + standard_name = vertical_layer_dimension + type = integer + units = count + dimensions = () + intent = in +[effrr_in] + standard_name = effective_radius_of_stratiform_cloud_rain_particle + long_name = effective radius of cloud rain particle in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + top_at_one = True +[effrg_in] + standard_name = effective_radius_of_stratiform_cloud_graupel + long_name = effective radius of cloud graupel in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + optional = True +[ncg_in] + standard_name = cloud_graupel_number_concentration + long_name = number concentration of cloud graupel + units = kg-1 + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + optional = True +[nci_out] + standard_name = cloud_ice_number_concentration + long_name = number concentration of cloud ice + units = kg-1 + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = out + optional = True +[effrl_inout] + standard_name = effective_radius_of_stratiform_cloud_liquid_water_particle + long_name = effective radius of cloud liquid water particle in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[effri_out] + standard_name = effective_radius_of_stratiform_cloud_ice_particle + long_name = effective radius of cloud ice water particle in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = out + optional = True +[effrs_inout] + standard_name = effective_radius_of_stratiform_cloud_snow_particle + long_name = effective radius of cloud snow particle in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = 8 + intent = inout + top_at_one = True +[ncl_out] + standard_name = cloud_liquid_number_concentration + long_name = number concentration of cloud liquid + units = kg-1 + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = out + optional = True +[has_graupel] + standard_name = flag_indicating_cloud_microphysics_has_graupel + long_name = flag indicating that the cloud microphysics produces graupel + units = flag + dimensions = () + type = logical + intent = in +[ scalar_var ] + standard_name = scalar_variable_for_testing + long_name = scalar variable for testing + units = km + dimensions = () + type = real + kind = kind_phys + intent = inout +[ tke_inout ] + standard_name = turbulent_kinetic_energy + long_name = turbulent_kinetic_energy + units = m2 s-2 + dimensions = () + type = real + kind = kind_phys + intent = inout +[ tke2_inout ] + standard_name = turbulent_kinetic_energy2 + long_name = turbulent_kinetic_energy2 + units = m+2 s-2 + dimensions = () + type = real + kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/var_compat/effr_diag.F90 b/end-to-end-tests/var_compat/effr_diag.F90 new file mode 100644 index 00000000..75da29c7 --- /dev/null +++ b/end-to-end-tests/var_compat/effr_diag.F90 @@ -0,0 +1,68 @@ +!Test unit conversions for intent in, inout, out variables +! + +module effr_diag + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: effr_diag_run, effr_diag_init + +contains + + !> \section arg_table_effr_diag_init Argument Table + !! \htmlinclude arg_table_effr_diag_init.html + !! + subroutine effr_diag_init(scheme_order, errmsg, errflg) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: scheme_order + + errmsg = '' + errflg = 0 + + if (scheme_order /= 4) then + errflg = 1 + errmsg = 'ERROR: effr_diag_init() needs to be called fourth' + return + else + scheme_order = scheme_order + 1 + end if + + end subroutine effr_diag_init + + !> \section arg_table_effr_diag_run Argument Table + !! \htmlinclude arg_table_effr_diag_run.html + !! + subroutine effr_diag_run(effrr_in, scalar_var, errmsg, errflg) + + real(kind=kind_phys), intent(in) :: effrr_in(:, :) + integer, intent(in) :: scalar_var + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + real(kind=kind_phys) :: effrr_min, effrr_max + + errmsg = '' + errflg = 0 + + call cmp_effr_diag(effrr_in, effrr_min, effrr_max) + + if (scalar_var /= 380) then + errmsg = 'ERROR: effr_diag_run(): scalar_var should be 380' + errflg = 1 + end if + end subroutine effr_diag_run + + subroutine cmp_effr_diag(effr, effr_min, effr_max) + real(kind=kind_phys), intent(in) :: effr(:, :) + real(kind=kind_phys), intent(out) :: effr_min, effr_max + + ! Do some diagnostic calcualtions... + effr_min = minval(effr) + effr_max = maxval(effr) + + end subroutine cmp_effr_diag +end module effr_diag diff --git a/end-to-end-tests/var_compat/effr_diag.meta b/end-to-end-tests/var_compat/effr_diag.meta new file mode 100644 index 00000000..5a5c9e67 --- /dev/null +++ b/end-to-end-tests/var_compat/effr_diag.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = effr_diag + type = scheme + dependencies = +######################################################################## +[ccpp-arg-table] + name = effr_diag_init + type = scheme +[ scheme_order ] + standard_name = scheme_order_in_suite + long_name = scheme order in suite definition file + units = None + dimensions = () + type = integer + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +######################################################################## +[ccpp-arg-table] + name = effr_diag_run + type = scheme +[effrr_in] + standard_name = effective_radius_of_stratiform_cloud_rain_particle + long_name = effective radius of cloud rain particle in micrometer + units = um + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + top_at_one = True +[ scalar_var ] + standard_name = scalar_variable_for_testing_c + long_name = unused scalar variable C + units = m + dimensions = () + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/var_compat/effr_post.F90 b/end-to-end-tests/var_compat/effr_post.F90 new file mode 100644 index 00000000..01357350 --- /dev/null +++ b/end-to-end-tests/var_compat/effr_post.F90 @@ -0,0 +1,61 @@ +!Test unit conversions for intent in, inout, out variables +! + +module effr_post + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: effr_post_run, effr_post_init + +contains + + !> \section arg_table_effr_post_init Argument Table + !! \htmlinclude arg_table_effr_post_init.html + !! + subroutine effr_post_init(scheme_order, errmsg, errflg) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: scheme_order + + errmsg = '' + errflg = 0 + + if (scheme_order /= 3) then + errflg = 1 + errmsg = 'ERROR: effr_post_init() needs to be called third' + return + else + scheme_order = scheme_order + 1 + end if + + end subroutine effr_post_init + + !> \section arg_table_effr_post_run Argument Table + !! \htmlinclude arg_table_effr_post_run.html + !! + subroutine effr_post_run(effrr_inout, scalar_var, errmsg, errflg) + + real(kind=kind_phys), intent(inout) :: effrr_inout(:, :) + real(kind=kind_phys), intent(in) :: scalar_var + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + real(kind=kind_phys) :: effrr_min, effrr_max + + errmsg = '' + errflg = 0 + + ! Do some post-processing on effrr... + effrr_inout(:, :) = effrr_inout(:, :) * 1._kind_phys + + if (scalar_var /= 1013.0) then + errmsg = 'ERROR: effr_post_run(): scalar_var should be 1013.0' + errflg = 1 + end if + + end subroutine effr_post_run + +end module effr_post diff --git a/end-to-end-tests/var_compat/effr_post.meta b/end-to-end-tests/var_compat/effr_post.meta new file mode 100644 index 00000000..703b5ebc --- /dev/null +++ b/end-to-end-tests/var_compat/effr_post.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = effr_post + type = scheme + dependencies = +######################################################################## +[ccpp-arg-table] + name = effr_post_init + type = scheme +[ scheme_order ] + standard_name = scheme_order_in_suite + long_name = scheme order in suite definition file + units = None + dimensions = () + type = integer + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +######################################################################## +[ccpp-arg-table] + name = effr_post_run + type = scheme +[effrr_inout] + standard_name = effective_radius_of_stratiform_cloud_rain_particle + long_name = effective radius of cloud rain particle in micrometer + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ scalar_var ] + standard_name = scalar_variable_for_testing_b + long_name = unused scalar variable B + units = m + dimensions = () + type = real + kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/var_compat/effr_pre.F90 b/end-to-end-tests/var_compat/effr_pre.F90 new file mode 100644 index 00000000..a2fe2f5c --- /dev/null +++ b/end-to-end-tests/var_compat/effr_pre.F90 @@ -0,0 +1,60 @@ +!Test unit conversions for intent in, inout, out variables +! + +module mod_effr_pre + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: effr_pre_run, effr_pre_init + +contains + !> \section arg_table_effr_pre_init Argument Table + !! \htmlinclude arg_table_effr_pre_init.html + !! + subroutine effr_pre_init(scheme_order, errmsg, errflg) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + integer, intent(inout) :: scheme_order + + errmsg = '' + errflg = 0 + + if (scheme_order /= 1) then + errflg = 1 + errmsg = 'ERROR: effr_pre_init() needs to be called first' + return + else + scheme_order = scheme_order + 1 + end if + + end subroutine effr_pre_init + + !> \section arg_table_effr_pre_run Argument Table + !! \htmlinclude arg_table_effr_pre_run.html + !! + subroutine effr_pre_run(effrr_inout, scalar_var, errmsg, errflg) + + real(kind=kind_phys), intent(inout) :: effrr_inout(:, :) + real(kind=kind_phys), intent(in) :: scalar_var + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + real(kind=kind_phys) :: effrr_min, effrr_max + + errmsg = '' + errflg = 0 + + ! Do some pre-processing on effrr... + effrr_inout(:, :) = effrr_inout(:, :) * 1._kind_phys + + if (scalar_var /= 273.15) then + errmsg = 'ERROR: effr_pre_run(): scalar_var should be 273.15' + errflg = 1 + end if + + end subroutine effr_pre_run + +end module mod_effr_pre diff --git a/end-to-end-tests/var_compat/effr_pre.meta b/end-to-end-tests/var_compat/effr_pre.meta new file mode 100644 index 00000000..c47d1abf --- /dev/null +++ b/end-to-end-tests/var_compat/effr_pre.meta @@ -0,0 +1,66 @@ +[ccpp-table-properties] + name = effr_pre + type = scheme + module_name = mod_effr_pre + dependencies = +######################################################################## +[ccpp-arg-table] + name = effr_pre_init + type = scheme +[ scheme_order ] + standard_name = scheme_order_in_suite + long_name = scheme order in suite definition file + units = None + dimensions = () + type = integer + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +######################################################################## +[ccpp-arg-table] + name = effr_pre_run + type = scheme +[effrr_inout] + standard_name = effective_radius_of_stratiform_cloud_rain_particle + long_name = effective radius of cloud rain particle in micrometer + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ scalar_var ] + standard_name = scalar_variable_for_testing_a + long_name = unused scalar variable A + units = m + dimensions = () + type = real + kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/var_compat/effrs_calc.F90 b/end-to-end-tests/var_compat/effrs_calc.F90 new file mode 100644 index 00000000..3aa8d196 --- /dev/null +++ b/end-to-end-tests/var_compat/effrs_calc.F90 @@ -0,0 +1,32 @@ +!Test unit conversions for intent in, inout, out variables +! + +module effrs_calc + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: effrs_calc_run + +contains + !> \section arg_table_effrs_calc_run Argument Table + !! \htmlinclude arg_table_effrs_calc_run.html + !! + subroutine effrs_calc_run(effrs_inout, errmsg, errflg) + + real(kind=kind_phys), intent(inout) :: effrs_inout(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + !---------------------------------------------------------------- + + errmsg = '' + errflg = 0 + + effrs_inout = effrs_inout + (10.E-6_kind_phys / 3._kind_phys) ! in meters + + end subroutine effrs_calc_run + +end module effrs_calc diff --git a/end-to-end-tests/var_compat/effrs_calc.meta b/end-to-end-tests/var_compat/effrs_calc.meta new file mode 100644 index 00000000..e2fd1de9 --- /dev/null +++ b/end-to-end-tests/var_compat/effrs_calc.meta @@ -0,0 +1,25 @@ +[ccpp-table-properties] + name = effrs_calc + type = scheme + +[ccpp-arg-table] + name = effrs_calc_run + type = scheme +[ effrs_inout ] + standard_name = effective_radius_of_stratiform_cloud_snow_particle + units = m + type = real | kind = kind_phys + dimensions = (horizontal_dimension,vertical_layer_dimension) + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + units = none + type = character | kind = len=512 + dimensions = () + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + type = integer + dimensions = () + intent = out diff --git a/end-to-end-tests/var_compat/module_rad_ddt.F90 b/end-to-end-tests/var_compat/module_rad_ddt.F90 new file mode 100644 index 00000000..6e992250 --- /dev/null +++ b/end-to-end-tests/var_compat/module_rad_ddt.F90 @@ -0,0 +1,23 @@ +module mod_rad_ddt + use ccpp_kinds, only: kind_phys + implicit none + + public ty_rad_lw, ty_rad_sw + + !> \section arg_table_ty_rad_lw Argument Table + !! \htmlinclude arg_table_ty_rad_lw.html + !! + type ty_rad_lw + real(kind=kind_phys) :: sfc_up_lw + real(kind=kind_phys) :: sfc_down_lw + end type ty_rad_lw + + !> \section arg_table_ty_rad_sw Argument Table + !! \htmlinclude arg_table_ty_rad_sw.html + !! + type ty_rad_sw + real(kind=kind_phys), pointer :: sfc_up_sw(:) => null() + real(kind=kind_phys), pointer :: sfc_down_sw(:) => null() + end type ty_rad_sw + +end module mod_rad_ddt diff --git a/end-to-end-tests/var_compat/module_rad_ddt.meta b/end-to-end-tests/var_compat/module_rad_ddt.meta new file mode 100644 index 00000000..c4792547 --- /dev/null +++ b/end-to-end-tests/var_compat/module_rad_ddt.meta @@ -0,0 +1,40 @@ +[ccpp-table-properties] + name = ty_rad_lw + type = ddt + dependencies = + module_name = mod_rad_ddt +[ccpp-arg-table] + name = ty_rad_lw + type = ddt +[ sfc_up_lw ] + standard_name = surface_upwelling_longwave_radiation_flux + units = W m2 + dimensions = () + type = real + kind = kind_phys +[ sfc_down_lw ] + standard_name = surface_downwelling_longwave_radiation_flux + units = W m2 + dimensions = () + type = real + kind = kind_phys + +[ccpp-table-properties] + name = ty_rad_sw + type = ddt + module_name = mod_rad_ddt +[ccpp-arg-table] + name = ty_rad_sw + type = ddt +[ sfc_up_sw ] + standard_name = surface_upwelling_shortwave_radiation_flux + units = W m2 + dimensions = (horizontal_dimension) + type = real + kind = kind_phys +[ sfc_down_sw ] + standard_name = surface_downwelling_shortwave_radiation_flux + units = W m2 + dimensions = (horizontal_dimension) + type = real + kind = kind_phys diff --git a/end-to-end-tests/var_compat/rad_lw.F90 b/end-to-end-tests/var_compat/rad_lw.F90 new file mode 100644 index 00000000..ded4861f --- /dev/null +++ b/end-to-end-tests/var_compat/rad_lw.F90 @@ -0,0 +1,35 @@ +module rad_lw + use ccpp_kinds, only: kind_phys + use mod_rad_ddt, only: ty_rad_lw + + implicit none + private + + public :: rad_lw_run + +contains + + !> \section arg_table_rad_lw_run Argument Table + !! \htmlinclude arg_table_rad_lw_run.html + !! + subroutine rad_lw_run(ncol, fluxlw, errmsg, errflg) + + integer, intent(in) :: ncol + type(ty_rad_lw), intent(inout) :: fluxlw(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! Locals + integer :: icol + + errmsg = '' + errflg = 0 + + do icol = 1, ncol + fluxlw(icol)%sfc_up_lw = 300._kind_phys + fluxlw(icol)%sfc_down_lw = 50._kind_phys + end do + + end subroutine rad_lw_run + +end module rad_lw diff --git a/end-to-end-tests/var_compat/rad_lw.meta b/end-to-end-tests/var_compat/rad_lw.meta new file mode 100644 index 00000000..bfab7426 --- /dev/null +++ b/end-to-end-tests/var_compat/rad_lw.meta @@ -0,0 +1,35 @@ +[ccpp-table-properties] + name = rad_lw + type = scheme + dependencies = module_rad_ddt.F90 +[ccpp-arg-table] + name = rad_lw_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[fluxLW] + standard_name = longwave_radiation_fluxes + long_name = longwave radiation fluxes + units = W m-2 + dimensions = (horizontal_dimension) + type = ty_rad_lw + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/var_compat/rad_sw.F90 b/end-to-end-tests/var_compat/rad_sw.F90 new file mode 100644 index 00000000..64756217 --- /dev/null +++ b/end-to-end-tests/var_compat/rad_sw.F90 @@ -0,0 +1,35 @@ +module rad_sw + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: rad_sw_run + +contains + + !> \section arg_table_rad_sw_run Argument Table + !! \htmlinclude arg_table_rad_sw_run.html + !! + subroutine rad_sw_run(ncol, sfc_up_sw, sfc_down_sw, errmsg, errflg) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(inout) :: sfc_up_sw(:) + real(kind=kind_phys), intent(inout) :: sfc_down_sw(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! Locals + integer :: icol + + errmsg = '' + errflg = 0 + + do icol = 1, ncol + sfc_up_sw(icol) = 100._kind_phys + sfc_down_sw(icol) = 400._kind_phys + end do + + end subroutine rad_sw_run + +end module rad_sw diff --git a/end-to-end-tests/var_compat/rad_sw.meta b/end-to-end-tests/var_compat/rad_sw.meta new file mode 100644 index 00000000..af88530f --- /dev/null +++ b/end-to-end-tests/var_compat/rad_sw.meta @@ -0,0 +1,41 @@ +[ccpp-table-properties] + name = rad_sw + type = scheme +[ccpp-arg-table] + name = rad_sw_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ sfc_up_sw ] + standard_name = surface_upwelling_shortwave_radiation_flux + units = W m2 + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[ sfc_down_sw ] + standard_name = surface_downwelling_shortwave_radiation_flux + units = W m2 + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/var_compat/test_host.F90 b/end-to-end-tests/var_compat/test_host.F90 new file mode 100644 index 00000000..0d691174 --- /dev/null +++ b/end-to-end-tests/var_compat/test_host.F90 @@ -0,0 +1,323 @@ +module test_prog + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public test_host + + ! Public data and interfaces + integer, public, parameter :: cs = 32 + integer, public, parameter :: cm = 60 + + !> \section arg_table_suite_info Argument Table + !! \htmlinclude arg_table_suite_info.html + !! + type, public :: suite_info + character(len=cs) :: suite_name = '' + character(len=cs), pointer :: suite_parts(:) => null() + character(len=cm), pointer :: suite_input_vars(:) => null() + character(len=cm), pointer :: suite_output_vars(:) => null() + character(len=cm), pointer :: suite_required_vars(:) => null() + end type suite_info + +contains + + logical function check_suite(test_suite) + use ccpp_static_api, only: ccpp_physics_suite_part_list + use ccpp_static_api, only: ccpp_physics_suite_variables + use test_utils, only: check_list + + ! Dummy argument + type(suite_info), intent(in) :: test_suite + ! Local variables + integer :: sind + logical :: check + integer :: errflg + character(len=512) :: errmsg + character(len=128), allocatable :: test_list(:) + + check_suite = .true. + write(6, *) "Checking suite ", trim(test_suite%suite_name) + ! First, check the suite parts + call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_parts, 'part names', & + suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the input variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.true., output_vars=.false.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_input_vars, & + 'input variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the output variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.false., output_vars=.true.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_output_vars, & + 'output variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check all required variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_required_vars, & + 'required variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + end function check_suite + + !> \section arg_table_test_host Argument Table + !! \htmlinclude arg_table_test_host.html + !! + subroutine test_host(retval, test_suites) + + use test_host_mod, only: ncols + use ccpp_static_api, only: ccpp_register + use ccpp_static_api, only: ccpp_init + use ccpp_static_api, only: ccpp_physics_init + use ccpp_static_api, only: ccpp_physics_timestep_init + use ccpp_static_api, only: ccpp_physics_run + use ccpp_static_api, only: ccpp_physics_timestep_final + use ccpp_static_api, only: ccpp_physics_final + use ccpp_static_api, only: ccpp_final + use ccpp_static_api, only: ccpp_physics_suite_list + use test_host_mod, only: init_data, & + compare_data + use test_utils, only: check_list + + type(suite_info), intent(in) :: test_suites(:) + logical, intent(out) :: retval + + logical :: check + integer :: col_start, col_end + integer :: index, sind + integer :: num_suites + character(len=128), allocatable :: suite_names(:) + character(len=512) :: errmsg + integer :: errflg + + ! Initialize our 'data' + call init_data() + + ! Gather and test the inspection routines + num_suites = size(test_suites) + call ccpp_physics_suite_list(suite_names) + retval = check_list(suite_names, test_suites(:)%suite_name, & + 'suite names') + write(6, *) 'Available suites are:' + do index = 1, size(suite_names) + do sind = 1, num_suites + if (trim(test_suites(sind)%suite_name) == & + trim(suite_names(index))) then + exit + end if + end do + write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & + ' = test_suites(', sind, ')' + end do + if (retval) then + do sind = 1, num_suites + check = check_suite(test_suites(sind)) + retval = retval .and. check + end do + end if + !!! Return here if any check failed + if (.not. retval) then + return + end if + + ! Register CCPP + do sind = 1, num_suites + call ccpp_register( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Initialize CCPP + do sind = 1, num_suites + call ccpp_init( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Use the suite information to setup the run + do sind = 1, num_suites + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + end if + end do + + ! Initialize the timestep + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + exit + end if + if (errflg /= 0) then + exit + end if + end do + + do col_start = 1, ncols, 5 + if (errflg /= 0) then + exit + end if + col_end = min(col_start + 4, ncols) + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + do index = 1, size(test_suites(sind)%suite_parts) + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + col_start=col_start, col_end=col_end, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(5a)') trim(test_suites(sind)%suite_name), & + '/', trim(test_suites(sind)%suite_parts(index)), & + ': ', trim(errmsg) + exit + end if + end do + end do + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_timestep_final, ', & + 'Exiting...' + exit + end if + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & + 'Exiting...' + exit + end if + end do + + do sind = 1, num_suites + if (errflg /= 0) then + exit + end if + if (errflg == 0) then + call ccpp_final( & + suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_final, ', & + 'Exiting...' + exit + end if + end do + + if (errflg == 0) then + ! Run finished without error, check answers + if (compare_data()) then + write(6, *) 'Answers are correct!' + errflg = 0 + else + write(6, *) 'Answers are not correct!' + errflg = -1 + end if + end if + + retval = errflg == 0 + + end subroutine test_host + +end module test_prog diff --git a/end-to-end-tests/var_compat/test_host.meta b/end-to-end-tests/var_compat/test_host.meta new file mode 100644 index 00000000..c151d87b --- /dev/null +++ b/end-to-end-tests/var_compat/test_host.meta @@ -0,0 +1,64 @@ +[ccpp-table-properties] + name = test_host + type = control + +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/end-to-end-tests/var_compat/test_host_data.F90 b/end-to-end-tests/var_compat/test_host_data.F90 new file mode 100644 index 00000000..ece60034 --- /dev/null +++ b/end-to-end-tests/var_compat/test_host_data.F90 @@ -0,0 +1,103 @@ +module test_host_data + + use ccpp_kinds, only: kind_phys + use mod_rad_ddt, only: ty_rad_lw, & + ty_rad_sw + + implicit none + private + + !> \section arg_table_physics_state Argument Table + !! \htmlinclude arg_table_physics_state.html + type physics_state + real(kind=kind_phys), dimension(:, :), allocatable :: & + effrr, & ! effective radius of cloud rain + effrl, & ! effective radius of cloud liquid water + effri, & ! effective radius of cloud ice + effrg, & ! effective radius of cloud graupel + ncg, & ! number concentration of cloud graupel + nci ! number concentration of cloud ice + real(kind=kind_phys) :: scalar_var + type(ty_rad_lw), dimension(:), allocatable :: & + fluxlw ! Longwave radiation fluxes + type(ty_rad_sw) :: & + fluxsw ! Shortwave radiation fluxes + real(kind=kind_phys) :: scalar_vara + real(kind=kind_phys) :: scalar_varb + real(kind=kind_phys) :: tke, tke2 + integer :: scalar_varc + integer :: scheme_order + integer :: num_subcycles + end type physics_state + + public :: physics_state + public :: allocate_physics_state + +contains + + subroutine allocate_physics_state(cols, levels, state, has_graupel, has_ice) + integer, intent(in) :: cols + integer, intent(in) :: levels + type(physics_state), intent(out) :: state + logical, intent(in) :: has_graupel + logical, intent(in) :: has_ice + + if (allocated(state%effrr)) then + deallocate(state%effrr) + end if + allocate(state%effrr(cols, levels)) + + if (allocated(state%effrl)) then + deallocate(state%effrl) + end if + allocate(state%effrl(cols, levels)) + + if (has_ice) then + if (allocated(state%effri)) then + deallocate(state%effri) + end if + allocate(state%effri(cols, levels)) + end if + + if (has_graupel) then + if (allocated(state%effrg)) then + deallocate(state%effrg) + end if + allocate(state%effrg(cols, levels)) + + if (allocated(state%ncg)) then + deallocate(state%ncg) + end if + allocate(state%ncg(cols, levels)) + end if + + if (has_ice) then + if (allocated(state%nci)) then + deallocate(state%nci) + end if + allocate(state%nci(cols, levels)) + end if + + if (allocated(state%fluxlw)) then + deallocate(state%fluxlw) + end if + allocate(state%fluxlw(cols)) + + if (associated(state%fluxsw%sfc_up_sw)) then + nullify(state%fluxsw%sfc_up_sw) + end if + allocate(state%fluxsw%sfc_up_sw(cols)) + + if (associated(state%fluxsw%sfc_down_sw)) then + nullify(state%fluxsw%sfc_down_sw) + end if + allocate(state%fluxsw%sfc_down_sw(cols)) + + ! Initialize scheme counter. + state%scheme_order = 1 + ! Initialize subcycle counter. + state%num_subcycles = 3 + + end subroutine allocate_physics_state + +end module test_host_data diff --git a/end-to-end-tests/var_compat/test_host_data.meta b/end-to-end-tests/var_compat/test_host_data.meta new file mode 100644 index 00000000..65ce3d9d --- /dev/null +++ b/end-to-end-tests/var_compat/test_host_data.meta @@ -0,0 +1,128 @@ +#[ccpp-table-properties] +# name = test_host_data +# type = host +# dependencies = module_rad_ddt.F90 +#[ccpp-arg-table] +# name = test_host_data +# type = host +# +[ccpp-table-properties] + name = physics_state + type = ddt + dependencies = module_rad_ddt.F90 +[ccpp-arg-table] + name = physics_state + type = ddt +[effrr] + standard_name = effective_radius_of_stratiform_cloud_rain_particle + long_name = effective radius of cloud rain particle in meter + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys +[effrl] + standard_name = effective_radius_of_stratiform_cloud_liquid_water_particle + long_name = effective radius of cloud liquid water particle in meter + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys +[effri] + standard_name = effective_radius_of_stratiform_cloud_ice_particle + long_name = effective radius of cloud ice water particle in meter + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + active = (flag_indicating_cloud_microphysics_has_ice) +[effrg] + standard_name = effective_radius_of_stratiform_cloud_graupel + long_name = effective radius of cloud graupel in meter + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + active = (flag_indicating_cloud_microphysics_has_graupel) +[ncg] + standard_name = cloud_graupel_number_concentration + long_name = number concentration of cloud graupel + units = kg-1 + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + active = (flag_indicating_cloud_microphysics_has_graupel) +[nci] + standard_name = cloud_ice_number_concentration + long_name = number concentration of cloud ice + units = kg-1 + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys + intent = in + active = (flag_indicating_cloud_microphysics_has_ice) +[scalar_var] + standard_name = scalar_variable_for_testing + long_name = unused scalar variable + units = m + dimensions = () + type = real + kind = kind_phys +[ tke ] + standard_name = turbulent_kinetic_energy + long_name = turbulent_kinetic_energy + units = J kg-1 + dimensions = () + type = real + kind = kind_phys +[ tke2 ] + standard_name = turbulent_kinetic_energy2 + long_name = turbulent_kinetic_energy2 + units = m2 s-2 + dimensions = () + type = real + kind = kind_phys +[fluxSW] + standard_name = shortwave_radiation_fluxes + long_name = shortwave radiation fluxes + units = W m-2 + dimensions = () + type = ty_rad_sw +[fluxLW] + standard_name = longwave_radiation_fluxes + long_name = longwave radiation fluxes + units = W m-2 + dimensions = (horizontal_dimension) + type = ty_rad_lw +[scalar_varA] + standard_name = scalar_variable_for_testing_a + long_name = unused scalar variable A + units = m + dimensions = () + type = real + kind = kind_phys +[scalar_varB] + standard_name = scalar_variable_for_testing_b + long_name = unused scalar variable B + units = m + dimensions = () + type = real + kind = kind_phys +[scalar_varC] + standard_name = scalar_variable_for_testing_c + long_name = unused scalar variable C + units = m + dimensions = () + type = integer +[scheme_order] + standard_name = scheme_order_in_suite + long_name = scheme order in suite definition file + units = None + dimensions = () + type = integer +[num_subcycles] + standard_name = num_subcycles_for_effr + long_name = Number of times to subcycle the effr calculation + units = None + dimensions = () + type = integer diff --git a/end-to-end-tests/var_compat/test_host_mod.F90 b/end-to-end-tests/var_compat/test_host_mod.F90 new file mode 100644 index 00000000..cba820d0 --- /dev/null +++ b/end-to-end-tests/var_compat/test_host_mod.F90 @@ -0,0 +1,132 @@ +module test_host_mod + + use ccpp_kinds, only: kind_phys + use test_host_data, only: physics_state, & + allocate_physics_state + + implicit none + public + + !> \section arg_table_test_host_mod Argument Table + !! \htmlinclude arg_table_test_host_host.html + !! + integer, parameter :: ncols = 12 + integer, parameter :: pver = 4 + type(physics_state), target :: phys_state + real(kind=kind_phys) :: effrs(ncols, pver) + logical, parameter :: has_ice = .true. + logical, parameter :: has_graupel = .true. + + public :: init_data + public :: compare_data + +contains + + subroutine init_data() + + ! Allocate and initialize state + call allocate_physics_state(ncols, pver, phys_state, has_graupel, has_ice) + phys_state%effrr = 1.0E-3 ! 1000 microns, in meter + phys_state%effrl = 1.0E-4 ! 100 microns, in meter + phys_state%scalar_var = 1.0 ! in m + phys_state%scalar_vara = 273.15 ! in K + phys_state%scalar_varb = 1013.0 ! in mb + phys_state%scalar_varc = 380 ! in ppmv + effrs = 5.0E-4 ! 500 microns, in meter + if (has_graupel) then + phys_state%effrg = 2.5E-4 ! 250 microns, in meter + phys_state%ncg = 40 + end if + if (has_ice) then + phys_state%effri = 5.0E-5 ! 50 microns, in meter + phys_state%nci = 80 + end if + phys_state%tke = 10.0 !J kg-1 + phys_state%tke2 = 42.0 !J kg-1 + + end subroutine init_data + + logical function compare_data() + + real(kind=kind_phys), parameter :: effrr_expected = 1.0E-3 ! 1000 microns, in meter + real(kind=kind_phys), parameter :: effrl_expected = 5.0E-5 ! 50 microns, in meter + real(kind=kind_phys), parameter :: effri_expected = 7.5E-5 ! 75 microns, in meter + real(kind=kind_phys), parameter :: effrs_expected = 5.3E-4 ! 530 microns, in meter + real(kind=kind_phys), parameter :: scalar_expected = 2.0E3 ! 2 km, in meter + real(kind=kind_phys), parameter :: tke_expected = 10.0 ! 10 J kg-1 + real(kind=kind_phys), parameter :: tolerance = 1.0E-6 ! used as scaling factor for expected value + real(kind=kind_phys), parameter :: sfc_up_sw_expected = 100. ! W/m2 + real(kind=kind_phys), parameter :: sfc_down_sw_expected = 400. ! W/m2 + real(kind=kind_phys), parameter :: sfc_up_lw_expected = 300. ! W/m2 + real(kind=kind_phys), parameter :: sfc_down_lw_expected = 50. ! W/m2 + + compare_data = .true. + + if (maxval(abs(phys_state%effrr - effrr_expected)) > tolerance * effrr_expected) then + write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effrr from expected value exceeds tolerance: ', & + maxval(abs(phys_state%effrr - effrr_expected)), ' > ', tolerance * effrr_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%effrl - effrl_expected)) > tolerance * effrl_expected) then + write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effrl from expected value exceeds tolerance: ', & + maxval(abs(phys_state%effrl - effrl_expected)), ' > ', tolerance * effrl_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%effri - effri_expected)) > tolerance * effri_expected) then + write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of phys_state%effri from expected value exceeds tolerance: ', & + maxval(abs(phys_state%effri - effri_expected)), ' > ', tolerance * effri_expected + compare_data = .false. + end if + + if (maxval(abs(effrs - effrs_expected)) > tolerance * effrs_expected) then + write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of effrs from expected value exceeds tolerance: ', & + maxval(abs(effrs - effrs_expected)), ' > ', tolerance * effrs_expected + compare_data = .false. + end if + + if (abs(phys_state%scalar_var - scalar_expected) > tolerance * scalar_expected) then + write(6, '(a,e16.7,a,e16.7)') & + 'Error: max diff of scalar_var from expected value exceeds tolerance: ', & + abs(phys_state%scalar_var - scalar_expected), ' > ', tolerance * scalar_expected + compare_data = .false. + end if + + if (abs(phys_state%tke - tke_expected) > tolerance * tke_expected) then + write(6, '(a,e16.7,a,e16.7)') 'Error: max diff of tke from expected value exceeds tolerance: ', & + abs(phys_state%tke - tke_expected), ' > ', tolerance * tke_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%fluxsw%sfc_up_sw - sfc_up_sw_expected)) > tolerance * sfc_up_sw_expected) then + write(6, '(a,e16.7,a,e16.7)') & + 'Error: max diff of sfc_up_sw from expected value exceeds tolerance: ', & + abs(phys_state%fluxsw%sfc_up_sw - sfc_up_sw_expected), ' > ', tolerance * sfc_up_sw_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%fluxsw%sfc_down_sw - sfc_down_sw_expected)) > tolerance * sfc_down_sw_expected) then + write(6, '(a,e16.7,a,e16.7)') & + 'Error: max diff of sfc_down_sw from expected value exceeds tolerance: ', & + abs(phys_state%fluxsw%sfc_down_sw - sfc_down_sw_expected), ' > ', tolerance * sfc_down_sw_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%fluxlw%sfc_up_lw - sfc_up_lw_expected)) > tolerance * sfc_up_lw_expected) then + write(6, '(a,e16.7,a,e16.7)') & + 'Error: max diff of sfc_up_lw from expected value exceeds tolerance: ', & + abs(phys_state%fluxlw%sfc_up_lw - sfc_up_lw_expected), ' > ', tolerance * sfc_up_lw_expected + compare_data = .false. + end if + + if (maxval(abs(phys_state%fluxlw%sfc_down_lw - sfc_down_lw_expected)) > tolerance * sfc_down_lw_expected) then + write(6, '(a,e16.7,a,e16.7)') & + 'Error: max diff of sfc_down_lw from expected value exceeds tolerance: ', & + abs(phys_state%fluxlw%sfc_down_lw - sfc_down_lw_expected), ' > ', tolerance * sfc_down_lw_expected + compare_data = .false. + end if + + end function compare_data + +end module test_host_mod diff --git a/end-to-end-tests/var_compat/test_host_mod.meta b/end-to-end-tests/var_compat/test_host_mod.meta new file mode 100644 index 00000000..a5df9381 --- /dev/null +++ b/end-to-end-tests/var_compat/test_host_mod.meta @@ -0,0 +1,42 @@ +[ccpp-table-properties] + name = test_host_mod + type = host +[ccpp-arg-table] + name = test_host_mod + type = host +[ ncols] + standard_name = horizontal_dimension + units = count + type = integer + protected = True + dimensions = () +[ pver ] + standard_name = vertical_layer_dimension + units = count + type = integer + protected = True + dimensions = () +[ phys_state ] + standard_name = physics_state_derived_type + long_name = Physics State DDT + type = physics_state + dimensions = () +[effrs] + standard_name = effective_radius_of_stratiform_cloud_snow_particle + long_name = effective radius of cloud snow particle in meter + units = m + dimensions = (horizontal_dimension,vertical_layer_dimension) + type = real + kind = kind_phys +[has_ice] + standard_name = flag_indicating_cloud_microphysics_has_ice + long_name = flag indicating that the cloud microphysics produces ice + units = flag + dimensions = () + type = logical +[has_graupel] + standard_name = flag_indicating_cloud_microphysics_has_graupel + long_name = flag indicating that the cloud microphysics produces graupel + units = flag + dimensions = () + type = logical diff --git a/end-to-end-tests/var_compat/test_var_compatibility_integration.F90 b/end-to-end-tests/var_compat/test_var_compatibility_integration.F90 new file mode 100644 index 00000000..4115face --- /dev/null +++ b/end-to-end-tests/var_compat/test_var_compatibility_integration.F90 @@ -0,0 +1,88 @@ +program test_var_compatibility_integration + use test_prog, only: test_host, & + suite_info, & + cm, & + cs + + implicit none + + character(len=cs), target :: test_parts1(1) = (/ 'radiation ' /) + + character(len=cm), target :: test_invars1(18) = (/ & + 'effective_radius_of_stratiform_cloud_rain_particle ', & + 'effective_radius_of_stratiform_cloud_liquid_water_particle', & + 'effective_radius_of_stratiform_cloud_snow_particle ', & + 'effective_radius_of_stratiform_cloud_graupel ', & + 'cloud_graupel_number_concentration ', & + 'scalar_variable_for_testing ', & + 'turbulent_kinetic_energy ', & + 'turbulent_kinetic_energy2 ', & + 'scalar_variable_for_testing_a ', & + 'scalar_variable_for_testing_b ', & + 'scalar_variable_for_testing_c ', & + 'scheme_order_in_suite ', & + 'num_subcycles_for_effr ', & + 'flag_indicating_cloud_microphysics_has_graupel ', & + 'flag_indicating_cloud_microphysics_has_ice ', & + 'surface_downwelling_shortwave_radiation_flux ', & + 'surface_upwelling_shortwave_radiation_flux ', & + 'longwave_radiation_fluxes '/) + + character(len=cm), target :: test_outvars1(14) = (/ & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'effective_radius_of_stratiform_cloud_ice_particle ', & + 'effective_radius_of_stratiform_cloud_liquid_water_particle', & + 'effective_radius_of_stratiform_cloud_rain_particle ', & + 'effective_radius_of_stratiform_cloud_snow_particle ', & + 'cloud_ice_number_concentration ', & + 'scalar_variable_for_testing ', & + 'scheme_order_in_suite ', & + 'surface_downwelling_shortwave_radiation_flux ', & + 'surface_upwelling_shortwave_radiation_flux ', & + 'turbulent_kinetic_energy ', & + 'turbulent_kinetic_energy2 ', & + 'longwave_radiation_fluxes '/) + + character(len=cm), target :: test_reqvars1(22) = (/ & + 'ccpp_error_code ', & + 'ccpp_error_message ', & + 'effective_radius_of_stratiform_cloud_rain_particle ', & + 'effective_radius_of_stratiform_cloud_ice_particle ', & + 'effective_radius_of_stratiform_cloud_liquid_water_particle', & + 'effective_radius_of_stratiform_cloud_snow_particle ', & + 'effective_radius_of_stratiform_cloud_graupel ', & + 'cloud_graupel_number_concentration ', & + 'cloud_ice_number_concentration ', & + 'scalar_variable_for_testing ', & + 'turbulent_kinetic_energy ', & + 'turbulent_kinetic_energy2 ', & + 'scalar_variable_for_testing_a ', & + 'scalar_variable_for_testing_b ', & + 'scalar_variable_for_testing_c ', & + 'scheme_order_in_suite ', & + 'num_subcycles_for_effr ', & + 'flag_indicating_cloud_microphysics_has_graupel ', & + 'flag_indicating_cloud_microphysics_has_ice ', & + 'surface_downwelling_shortwave_radiation_flux ', & + 'surface_upwelling_shortwave_radiation_flux ', & + 'longwave_radiation_fluxes '/) + + type(suite_info) :: test_suites(1) + logical :: run_okay + + ! Setup expected test suite info + test_suites(1)%suite_name = 'var_compatibility_suite' + test_suites(1)%suite_parts => test_parts1 + test_suites(1)%suite_input_vars => test_invars1 + test_suites(1)%suite_output_vars => test_outvars1 + test_suites(1)%suite_required_vars => test_reqvars1 + + call test_host(run_okay, test_suites) + + if (run_okay) then + stop 0 + else + stop -1 + end if +end program test_var_compatibility_integration diff --git a/end-to-end-tests/var_compat/var_compatibility_suite.xml b/end-to-end-tests/var_compat/var_compatibility_suite.xml new file mode 100644 index 00000000..07d950ef --- /dev/null +++ b/end-to-end-tests/var_compat/var_compatibility_suite.xml @@ -0,0 +1,21 @@ + + + + + + effr_pre + + + effr_calc + + + effr_post + + + effrs_calc + + effr_diag + rad_lw + rad_sw + + diff --git a/end-to-end-tests/var_compat/var_compatibility_test_reports.py b/end-to-end-tests/var_compat/var_compatibility_test_reports.py new file mode 100755 index 00000000..ada612c7 --- /dev/null +++ b/end-to-end-tests/var_compat/var_compatibility_test_reports.py @@ -0,0 +1,116 @@ +#! /usr/bin/env python3 +""" +----------------------------------------------------------------------- + Description: Test capgen database report python interface + + Assumptions: + + Command line arguments: build_dir database_filepath + + Usage: python test_reports +----------------------------------------------------------------------- +""" +import os +import unittest + +from test_stub import BaseTests + +_BUILD_DIR = os.path.join(os.path.abspath(os.environ['BUILD_DIR']), "test", "var_compatibility_test") +_DATABASE = os.path.abspath(os.path.join(_BUILD_DIR, "ccpp", "datatable.xml")) + +_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +_FRAMEWORK_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, os.pardir)) +_SCRIPTS_DIR = os.path.join(_FRAMEWORK_DIR, "scripts") +_SRC_DIR = os.path.join(_FRAMEWORK_DIR, "src") + +# Check data +_HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] +_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_compatibility_suite_cap.F90")] +_UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), + os.path.join(_SRC_DIR, "ccpp_constituent_prop_mod.F90"), + os.path.join(_SRC_DIR, "ccpp_scheme_utils.F90"), + os.path.join(_SRC_DIR, "ccpp_hashable.F90"), + os.path.join(_SRC_DIR, "ccpp_hash_table.F90")] +_CCPP_FILES = _UTILITY_FILES + \ + [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90"), + os.path.join(_BUILD_DIR, "ccpp", "ccpp_var_compatibility_suite_cap.F90")] +_PROCESS_LIST = [""] +_MODULE_LIST = ["effr_calc", "effrs_calc", "effr_diag", "effr_post", "mod_effr_pre", "rad_lw", "rad_sw"] +_SUITE_LIST = ["var_compatibility_suite"] +_DEPENDENCIES = [ os.path.join(_TEST_DIR, "module_rad_ddt.F90")] +_INPUT_VARS_VAR_ACTION = ["horizontal_loop_begin", "horizontal_loop_end", "horizontal_dimension", "vertical_layer_dimension", + "effective_radius_of_stratiform_cloud_liquid_water_particle", + "effective_radius_of_stratiform_cloud_rain_particle", + "effective_radius_of_stratiform_cloud_snow_particle", + "effective_radius_of_stratiform_cloud_graupel", + "cloud_graupel_number_concentration", + "scalar_variable_for_testing", + "turbulent_kinetic_energy", + "turbulent_kinetic_energy2", + "scalar_variable_for_testing_a", + "scalar_variable_for_testing_b", + "scalar_variable_for_testing_c", + "scheme_order_in_suite", + "flag_indicating_cloud_microphysics_has_graupel", + "flag_indicating_cloud_microphysics_has_ice", + "surface_downwelling_shortwave_radiation_flux", + "surface_upwelling_shortwave_radiation_flux", + "longwave_radiation_fluxes", + "num_subcycles_for_effr"] +_OUTPUT_VARS_VAR_ACTION = ["ccpp_error_code", "ccpp_error_message", + "effective_radius_of_stratiform_cloud_ice_particle", + "effective_radius_of_stratiform_cloud_liquid_water_particle", + "effective_radius_of_stratiform_cloud_snow_particle", + "cloud_ice_number_concentration", + "effective_radius_of_stratiform_cloud_rain_particle", + "turbulent_kinetic_energy", + "turbulent_kinetic_energy2", + "scalar_variable_for_testing", + "scalar_variable_for_testing", + "surface_downwelling_shortwave_radiation_flux", + "surface_upwelling_shortwave_radiation_flux", + "longwave_radiation_fluxes", + "scheme_order_in_suite"] +_REQUIRED_VARS_VAR_ACTION = _INPUT_VARS_VAR_ACTION + _OUTPUT_VARS_VAR_ACTION + + +class TestVarCompatibilityHostDataTables(unittest.TestCase, BaseTests.TestHostDataTables): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + + +class CommandLineVarCompatibilityHostDatafileRequiredFiles(unittest.TestCase, BaseTests.TestHostCommandLineDataFiles): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" + + +class TestCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuite): + database = _DATABASE + required_vars = _REQUIRED_VARS_VAR_ACTION + input_vars = _INPUT_VARS_VAR_ACTION + output_vars = _OUTPUT_VARS_VAR_ACTION + suite_name = "var_compatibility_suite" + + +class CommandLineCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuiteCommandLine): + database = _DATABASE + required_vars = _REQUIRED_VARS_VAR_ACTION + input_vars = _INPUT_VARS_VAR_ACTION + output_vars = _OUTPUT_VARS_VAR_ACTION + suite_name = "var_compatibility_suite" + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" diff --git a/unit-tests/__init__.py b/unit-tests/__init__.py new file mode 100644 index 00000000..35c2ae0f --- /dev/null +++ b/unit-tests/__init__.py @@ -0,0 +1 @@ +"""Unit and integration tests for ccpp-capgen-ng.""" diff --git a/unit-tests/conftest.py b/unit-tests/conftest.py new file mode 100644 index 00000000..638eb4eb --- /dev/null +++ b/unit-tests/conftest.py @@ -0,0 +1,30 @@ +"""pytest configuration for capgen-ng unit tests. + +Adds the capgen-ng package root to sys.path so that ``import metadata`` and +``import generator`` work regardless of where pytest is invoked from. + +Layout assumed:: + + / + capgen-ng/ <-- the package being tested + unit-tests/ <-- this file's parent directory + conftest.py + test_*.py +""" + +import os +import sys + +# this directory (unit-tests/) — needed so tests can ``from +# import …``. The directory name contains a hyphen so it can't be +# used as a Python module, but adding it to sys.path makes each +# top-level test_*.py importable as a flat module. +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +# repository root (parent of unit-tests/) +_REPO_ROOT = os.path.dirname(_TESTS_DIR) +# capgen-ng/ package directory (sibling of unit-tests/) +_CAPGEN_DIR = os.path.join(_REPO_ROOT, 'capgen-ng') + +for _path in (_TESTS_DIR, _CAPGEN_DIR, _REPO_ROOT): + if _path not in sys.path: + sys.path.insert(0, _path) diff --git a/unit-tests/run_tests.py b/unit-tests/run_tests.py new file mode 100644 index 00000000..a4769948 --- /dev/null +++ b/unit-tests/run_tests.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Convenience script to run all capgen-ng unit tests. + +Usage:: + + python unit-tests/run_tests.py # run all tests + python unit-tests/run_tests.py -v # verbose output + python unit-tests/run_tests.py --doctest # include doctests + +This script sets up ``sys.path`` and then delegates to ``unittest.main`` so +that the tests can be run without installing the package. +""" + +import os +import sys +import unittest + +# ---- path setup ------------------------------------------------------------ +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.dirname(_TESTS_DIR) +_CAPGEN_DIR = os.path.join(_REPO_ROOT, 'capgen-ng') + +for _p in (_CAPGEN_DIR, _REPO_ROOT): + if _p not in sys.path: + sys.path.insert(0, _p) + +# ---- optional --doctest flag ----------------------------------------------- +_include_doctests = '--doctest' in sys.argv +if _include_doctests: + sys.argv.remove('--doctest') + +if __name__ == '__main__': + loader = unittest.TestLoader() + suite = loader.discover(start_dir=_TESTS_DIR, pattern='test_*.py') + + if _include_doctests: + import doctest + import metadata.metadata_table as _mt + suite.addTests(doctest.DocTestSuite(_mt)) + + runner = unittest.TextTestRunner(verbosity=2 if '-v' in sys.argv else 1) + result = runner.run(suite) + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/unit-tests/sample_files/bad_ctrl_in_host_table.meta b/unit-tests/sample_files/bad_ctrl_in_host_table.meta new file mode 100644 index 00000000..4cf16a82 --- /dev/null +++ b/unit-tests/sample_files/bad_ctrl_in_host_table.meta @@ -0,0 +1,17 @@ +# Host table that declares suite_name — it should be in a control table instead. + +[ccpp-table-properties] + name = misplaced_host + type = host + +[ccpp-arg-table] + name = misplaced_host + type = host + +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 diff --git a/unit-tests/sample_files/bad_ctrl_missing_suite_name.meta b/unit-tests/sample_files/bad_ctrl_missing_suite_name.meta new file mode 100644 index 00000000..2554d704 --- /dev/null +++ b/unit-tests/sample_files/bad_ctrl_missing_suite_name.meta @@ -0,0 +1,60 @@ +# Control table with all required vars except suite_name. +# Paired with bad_ctrl_in_host_table.meta to test the "wrong table" error. + +[ccpp-table-properties] + name = ctrl_no_suite_name + type = control + +[ccpp-arg-table] + name = ctrl_no_suite_name + type = control + +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = physics thread budget + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/bad_ctrl_missing_vars.meta b/unit-tests/sample_files/bad_ctrl_missing_vars.meta new file mode 100644 index 00000000..24a7b878 --- /dev/null +++ b/unit-tests/sample_files/bad_ctrl_missing_vars.meta @@ -0,0 +1,18 @@ +# Control table that is missing all required variables except suite_name. +# Used to test that the generator reports all missing vars in one pass. + +[ccpp-table-properties] + name = incomplete_ctrl + type = control + +[ccpp-arg-table] + name = incomplete_ctrl + type = control + +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 diff --git a/unit-tests/sample_files/bad_ctrl_nonscalar.meta b/unit-tests/sample_files/bad_ctrl_nonscalar.meta new file mode 100644 index 00000000..ebfa5b07 --- /dev/null +++ b/unit-tests/sample_files/bad_ctrl_nonscalar.meta @@ -0,0 +1,66 @@ +# Control table where ccpp_error_code has dimensions (must be scalar). + +[ccpp-table-properties] + name = nonscalar_ctrl + type = control + +[ccpp-arg-table] + name = nonscalar_ctrl + type = control + +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = physics thread budget + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = (horizontal_dimension) + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/bad_ctrl_wrong_type.meta b/unit-tests/sample_files/bad_ctrl_wrong_type.meta new file mode 100644 index 00000000..f3574687 --- /dev/null +++ b/unit-tests/sample_files/bad_ctrl_wrong_type.meta @@ -0,0 +1,66 @@ +# Control table where horizontal_loop_begin is declared as real instead of integer. + +[ccpp-table-properties] + name = wrong_type_ctrl + type = control + +[ccpp-arg-table] + name = wrong_type_ctrl + type = control + +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range + units = index + dimensions = () + type = real +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = physics thread budget + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/bad_dim_loop_begin.meta b/unit-tests/sample_files/bad_dim_loop_begin.meta new file mode 100644 index 00000000..e7ba7060 --- /dev/null +++ b/unit-tests/sample_files/bad_dim_loop_begin.meta @@ -0,0 +1,33 @@ +# Scheme variable that uses horizontal_loop_begin as a dimension — forbidden. + +[ccpp-table-properties] + name = bad_dim_scheme + type = scheme + +[ccpp-arg-table] + name = bad_dim_scheme_run + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ data ] + standard_name = some_data + long_name = data array with forbidden dimension + units = 1 + dimensions = (horizontal_loop_begin) + type = real + kind = kind_phys + intent = in diff --git a/unit-tests/sample_files/bad_dim_loop_extent.meta b/unit-tests/sample_files/bad_dim_loop_extent.meta new file mode 100644 index 00000000..1c8cad2b --- /dev/null +++ b/unit-tests/sample_files/bad_dim_loop_extent.meta @@ -0,0 +1,23 @@ +# Host variable that uses horizontal_loop_extent as a dimension — forbidden. + +[ccpp-table-properties] + name = bad_dim_host + type = host + +[ccpp-arg-table] + name = bad_dim_host + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer +[ data ] + standard_name = some_data + long_name = data array dimensioned by loop extent + units = 1 + dimensions = (horizontal_loop_extent) + type = real + kind = kind_phys diff --git a/unit-tests/sample_files/bad_duplicate_stdname.meta b/unit-tests/sample_files/bad_duplicate_stdname.meta new file mode 100644 index 00000000..07dc00a8 --- /dev/null +++ b/unit-tests/sample_files/bad_duplicate_stdname.meta @@ -0,0 +1,21 @@ +# Scheme with two variables sharing the same standard name — must be rejected. + +[ccpp-table-properties] + name = dup_scheme + type = scheme + +[ccpp-arg-table] + name = dup_scheme_run + type = scheme +[ a_var ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer + intent = in +[ b_var ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer + intent = in diff --git a/unit-tests/sample_files/bad_finalize_phase.meta b/unit-tests/sample_files/bad_finalize_phase.meta new file mode 100644 index 00000000..edbc30db --- /dev/null +++ b/unit-tests/sample_files/bad_finalize_phase.meta @@ -0,0 +1,23 @@ +# Scheme with old 'finalize' phase name — must be rejected. +# The generator should emit a hard error directing the user to rename to 'final'. + +[ccpp-table-properties] + name = old_scheme + type = scheme + +[ccpp-arg-table] + name = old_scheme_finalize + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/bad_invalid_type.meta b/unit-tests/sample_files/bad_invalid_type.meta new file mode 100644 index 00000000..d728f68e --- /dev/null +++ b/unit-tests/sample_files/bad_invalid_type.meta @@ -0,0 +1,14 @@ +# Table with 'type = banana' — must be rejected with a clear error. + +[ccpp-table-properties] + name = bad_table + type = banana + +[ccpp-arg-table] + name = bad_table + type = banana +[ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/bad_module_type.meta b/unit-tests/sample_files/bad_module_type.meta new file mode 100644 index 00000000..1a2ff906 --- /dev/null +++ b/unit-tests/sample_files/bad_module_type.meta @@ -0,0 +1,15 @@ +# This file uses the old 'type = module' which must be rejected. +# The generator should emit a helpful error directing the user to 'type = host'. + +[ccpp-table-properties] + name = physics_module + type = module + +[ccpp-arg-table] + name = physics_module + type = module +[ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_chunked_data.meta b/unit-tests/sample_files/control_chunked_data.meta new file mode 100644 index 00000000..04dfdf98 --- /dev/null +++ b/unit-tests/sample_files/control_chunked_data.meta @@ -0,0 +1,79 @@ +# Control metadata for the chunked_data integration test. + +[ccpp-table-properties] + name = chunked_ctrl + type = control + +[ccpp-arg-table] + name = chunked_ctrl + type = control + +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ nchunk ] + standard_name = ccpp_chunk_number + long_name = current chunk number + units = index + dimensions = () + type = integer +[ thrd_no ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_no ] + standard_name = instance_number + long_name = current model instance index + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_full.meta b/unit-tests/sample_files/control_full.meta new file mode 100644 index 00000000..3f419372 --- /dev/null +++ b/unit-tests/sample_files/control_full.meta @@ -0,0 +1,73 @@ +# Full control metadata for generator tests. + +[ccpp-table-properties] + name = host_ctrl + type = control + +[ccpp-arg-table] + name = host_ctrl + type = control + +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_no_instance.meta b/unit-tests/sample_files/control_no_instance.meta new file mode 100644 index 00000000..282b6a48 --- /dev/null +++ b/unit-tests/sample_files/control_no_instance.meta @@ -0,0 +1,67 @@ +# Single-instance host: control table omits instance_number. The paired +# host metadata (host_no_instance.meta) likewise omits number_of_instances. +# Used to exercise the no-instance-API code paths in the generator. +[ccpp-table-properties] + name = ccpp_control + type = control + +[ccpp-arg-table] + name = ccpp_control + type = control +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = total number of physics threads + units = 1 + dimensions = () + type = integer +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_opt_arg.meta b/unit-tests/sample_files/control_opt_arg.meta new file mode 100644 index 00000000..bbbf2e4b --- /dev/null +++ b/unit-tests/sample_files/control_opt_arg.meta @@ -0,0 +1,73 @@ +# Minimal control metadata for the opt_arg integration test. + +[ccpp-table-properties] + name = opt_arg_ctrl + type = control + +[ccpp-arg-table] + name = opt_arg_ctrl + type = control + +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thrd_no ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_simple.meta b/unit-tests/sample_files/control_simple.meta new file mode 100644 index 00000000..81a47798 --- /dev/null +++ b/unit-tests/sample_files/control_simple.meta @@ -0,0 +1,70 @@ +[ccpp-table-properties] + name = ccpp_control + type = control + +[ccpp-arg-table] + name = ccpp_control + type = control +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = total number of physics threads + units = 1 + dimensions = () + type = integer +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=256 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_unit_conv.meta b/unit-tests/sample_files/control_unit_conv.meta new file mode 100644 index 00000000..2f9fd066 --- /dev/null +++ b/unit-tests/sample_files/control_unit_conv.meta @@ -0,0 +1,66 @@ +# Control metadata for the unit_conv integration test. + +[ccpp-table-properties] + name = unit_conv_ctrl + type = control + +[ccpp-arg-table] + name = unit_conv_ctrl + type = control + +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = physics thread budget + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/ddt_chunked_data.meta b/unit-tests/sample_files/ddt_chunked_data.meta new file mode 100644 index 00000000..0ad6cea9 --- /dev/null +++ b/unit-tests/sample_files/ddt_chunked_data.meta @@ -0,0 +1,16 @@ +# DDT definition for the chunked_data integration test. + +[ccpp-table-properties] + name = chunked_data_type + type = ddt + +[ccpp-arg-table] + name = chunked_data_type + type = ddt + +[ array_data ] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer diff --git a/unit-tests/sample_files/ddt_nested_inner.meta b/unit-tests/sample_files/ddt_nested_inner.meta new file mode 100644 index 00000000..53f8dbe4 --- /dev/null +++ b/unit-tests/sample_files/ddt_nested_inner.meta @@ -0,0 +1,22 @@ +# Inner DDT used as a field inside ddt_nested_outer.meta. + +[ccpp-table-properties] + name = ddt_inner_type + type = ddt + +[ccpp-arg-table] + name = ddt_inner_type + type = ddt +[ inner_value ] + standard_name = inner_real_value + long_name = a real value inside the inner DDT + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys +[ inner_flag ] + standard_name = inner_integer_flag + long_name = an integer flag inside the inner DDT + units = flag + dimensions = () + type = integer diff --git a/unit-tests/sample_files/ddt_nested_outer.meta b/unit-tests/sample_files/ddt_nested_outer.meta new file mode 100644 index 00000000..ad2a4a53 --- /dev/null +++ b/unit-tests/sample_files/ddt_nested_outer.meta @@ -0,0 +1,22 @@ +# Outer DDT whose second field is itself a DDT (tests nested DDT flattening). +# Field 'inner_ddt' is of type 'ddt_inner_type', defined in ddt_nested_inner.meta. + +[ccpp-table-properties] + name = ddt_outer_type + type = ddt + +[ccpp-arg-table] + name = ddt_outer_type + type = ddt +[ scalar_field ] + standard_name = outer_scalar_field + long_name = a scalar field on the outer DDT + units = 1 + dimensions = () + type = integer +[ inner_ddt ] + standard_name = inner_ddt_instance + long_name = nested inner DDT + units = none + dimensions = () + type = ddt_inner_type diff --git a/unit-tests/sample_files/ddt_simple.meta b/unit-tests/sample_files/ddt_simple.meta new file mode 100644 index 00000000..f3731a7c --- /dev/null +++ b/unit-tests/sample_files/ddt_simple.meta @@ -0,0 +1,21 @@ +[ccpp-table-properties] + name = gfs_statein_type + type = ddt + +[ccpp-arg-table] + name = gfs_statein_type + type = ddt +[ phii ] + standard_name = geopotential_at_interface + long_name = geopotential at model layer interfaces + units = m2 s-2 + dimensions = (horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys +[ phil ] + standard_name = geopotential + long_name = geopotential at model layer centers + units = m2 s-2 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys diff --git a/unit-tests/sample_files/ddt_subcycle_stdname.meta b/unit-tests/sample_files/ddt_subcycle_stdname.meta new file mode 100644 index 00000000..8c6d215b --- /dev/null +++ b/unit-tests/sample_files/ddt_subcycle_stdname.meta @@ -0,0 +1,17 @@ +# DDT definition holding the subcycle-count field used by +# suite_subcycle_stdname_ddt.xml. The standard name is exposed via the +# DDT instance declared in host_subcycle_stdname_ddt.meta. + +[ccpp-table-properties] + name = physics_state_subcycle + type = ddt + +[ccpp-arg-table] + name = physics_state_subcycle + type = ddt +[ n_sub ] + standard_name = num_subcycles_for_test + long_name = number of subcycles for the test scheme + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/host_chunked_data.meta b/unit-tests/sample_files/host_chunked_data.meta new file mode 100644 index 00000000..225ea68f --- /dev/null +++ b/unit-tests/sample_files/host_chunked_data.meta @@ -0,0 +1,31 @@ +# Host metadata for the chunked_data integration test. +# Tests a scheme that uses ccpp_chunk_number as a control variable and +# accesses a DDT-based data array. Chunk loop bounds are provided as +# scalars (the host manages chunking externally). + +[ccpp-table-properties] + name = chunked_data_mod + type = host + +[ccpp-arg-table] + name = chunked_data_mod + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ chunked_data_instance ] + standard_name = chunked_data_type_instance + long_name = instance of chunked_data_type + units = DDT + dimensions = () + type = chunked_data_type +[ ninstances ] + standard_name = number_of_instances + long_name = number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/host_full.meta b/unit-tests/sample_files/host_full.meta new file mode 100644 index 00000000..c4d7ce32 --- /dev/null +++ b/unit-tests/sample_files/host_full.meta @@ -0,0 +1,56 @@ +# Full host metadata for generator tests. +# Provides all variables needed by temp_calc_adjust scheme +# plus extras for dimension resolution. + +[ccpp-table-properties] + name = host_phys + type = host + +[ccpp-arg-table] + name = host_phys + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ im ] + standard_name = horizontal_loop_extent + long_name = horizontal loop extent (ub - lb + 1) + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + long_name = number of vertical layers + units = count + dimensions = () + type = integer +[ nlevp1 ] + standard_name = vertical_interface_dimension + long_name = number of vertical layer interfaces + units = count + dimensions = () + type = integer +[ dt ] + standard_name = time_step_for_physics + long_name = physics time step + units = s + dimensions = () + type = real + kind = kind_phys +[ gt0 ] + standard_name = air_temperature + long_name = model layer mean temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys +[ ninstances ] + standard_name = number_of_instances + long_name = number of model instances (ensemble members) + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/host_no_instance.meta b/unit-tests/sample_files/host_no_instance.meta new file mode 100644 index 00000000..92d5a383 --- /dev/null +++ b/unit-tests/sample_files/host_no_instance.meta @@ -0,0 +1,49 @@ +# Single-instance host data: same vars as host_full.meta minus +# number_of_instances. Pairs with control_no_instance.meta (which +# omits instance_number). +[ccpp-table-properties] + name = host_phys + type = host + +[ccpp-arg-table] + name = host_phys + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ im ] + standard_name = horizontal_loop_extent + long_name = horizontal loop extent (ub - lb + 1) + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + long_name = number of vertical layers + units = count + dimensions = () + type = integer +[ nlevp1 ] + standard_name = vertical_interface_dimension + long_name = number of vertical layer interfaces + units = count + dimensions = () + type = integer +[ dt ] + standard_name = time_step_for_physics + long_name = physics time step + units = s + dimensions = () + type = real + kind = kind_phys +[ gt0 ] + standard_name = air_temperature + long_name = model layer mean temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys diff --git a/unit-tests/sample_files/host_opt_arg.meta b/unit-tests/sample_files/host_opt_arg.meta new file mode 100644 index 00000000..5fb01b7d --- /dev/null +++ b/unit-tests/sample_files/host_opt_arg.meta @@ -0,0 +1,52 @@ +# Host metadata for the opt_arg integration test. +# Provides a mandatory integer array, an optional integer array, +# an optional real array (km; scheme expects m — Case 4), and an +# active flag controlling whether the optional vars are present. + +[ccpp-table-properties] + name = opt_arg_data + type = host + +[ccpp-arg-table] + name = opt_arg_data + type = host + +[ nx ] + standard_name = size_of_std_arg + long_name = size of std_arg array + units = count + dimensions = () + type = integer +[ std_arg ] + standard_name = std_arg + long_name = mandatory integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer +[ opt_arg ] + standard_name = opt_arg + long_name = optional integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + active = flag_for_opt_arg +[ opt_arg_2 ] + standard_name = opt_arg_2 + long_name = optional real array in km + units = km + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + active = flag_for_opt_arg +[ flag_for_opt_arg ] + standard_name = flag_for_opt_arg + long_name = flag controlling whether optional vars are present + units = flag + dimensions = () + type = logical +[ ninstances ] + standard_name = number_of_instances + long_name = number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/host_simple.meta b/unit-tests/sample_files/host_simple.meta new file mode 100644 index 00000000..3527e210 --- /dev/null +++ b/unit-tests/sample_files/host_simple.meta @@ -0,0 +1,25 @@ +[ccpp-table-properties] + name = physics_data + type = host + +[ccpp-arg-table] + name = physics_data + type = host +[ ncols ] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + long_name = number of vertical layers + units = count + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/host_subcycle_stdname.meta b/unit-tests/sample_files/host_subcycle_stdname.meta new file mode 100644 index 00000000..5bd4bd5d --- /dev/null +++ b/unit-tests/sample_files/host_subcycle_stdname.meta @@ -0,0 +1,18 @@ +# Supplementary host metadata: adds num_subcycles_for_test (used as a +# subcycle loop= bound in suite_subcycle_stdname.xml). Pairs +# with host_full.meta / control_full.meta which provide all the other +# required vars (loop bounds, errflg, errmsg, etc.). + +[ccpp-table-properties] + name = host_phys_subcycle_helper + type = host + +[ccpp-arg-table] + name = host_phys_subcycle_helper + type = host +[ n_sub ] + standard_name = num_subcycles_for_test + long_name = number of subcycles for the test scheme + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/host_subcycle_stdname_ddt.meta b/unit-tests/sample_files/host_subcycle_stdname_ddt.meta new file mode 100644 index 00000000..211eb9e2 --- /dev/null +++ b/unit-tests/sample_files/host_subcycle_stdname_ddt.meta @@ -0,0 +1,19 @@ +# Supplementary host metadata: declares an instance of the +# physics_state_subcycle DDT, which exposes num_subcycles_for_test as a +# component. The subcycle bound in suite_subcycle_stdname_ddt.xml +# resolves to ``phys_state%n_sub`` (access path), not the bare +# component name. + +[ccpp-table-properties] + name = test_host_with_ddt_mod + type = host + +[ccpp-arg-table] + name = test_host_with_ddt_mod + type = host +[ phys_state ] + standard_name = physics_state_subcycle_ddt_instance + long_name = physics state DDT instance carrying the subcycle counter + units = ddt + dimensions = () + type = physics_state_subcycle diff --git a/unit-tests/sample_files/host_unit_conv.meta b/unit-tests/sample_files/host_unit_conv.meta new file mode 100644 index 00000000..3fd44700 --- /dev/null +++ b/unit-tests/sample_files/host_unit_conv.meta @@ -0,0 +1,51 @@ +# Host metadata for the unit_conv integration test. +# Provides horizontal loop variables, data_array in m (mandatory), +# and data_array_opt in m (optional, controlled by flag_for_opt_array). + +[ccpp-table-properties] + name = unit_conv_data + type = host + +[ccpp-arg-table] + name = unit_conv_data + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ im ] + standard_name = horizontal_loop_extent + long_name = horizontal loop extent + units = count + dimensions = () + type = integer +[ data_array ] + standard_name = data_array + long_name = mandatory data array in m + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys +[ data_array_opt ] + standard_name = data_array_opt + long_name = optional data array in m + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + active = flag_for_opt_array +[ flag_for_opt_array ] + standard_name = flag_for_opt_array + long_name = flag controlling whether optional array is present + units = flag + dimensions = () + type = logical +[ ninstances ] + standard_name = number_of_instances + long_name = number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/host_with_constituents.meta b/unit-tests/sample_files/host_with_constituents.meta new file mode 100644 index 00000000..75328806 --- /dev/null +++ b/unit-tests/sample_files/host_with_constituents.meta @@ -0,0 +1,36 @@ +# Host metadata that exposes the constituent object via the type=host table. +# Required when at least one register-phase scheme produces dynamic +# constituents (intent=out type=ccpp_constituent_properties_t). + +[ccpp-table-properties] + name = host_consts + type = host + +[ccpp-arg-table] + name = host_consts + type = host + +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + long_name = number of vertical layers + units = count + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = number of model instances + units = count + dimensions = () + type = integer +[ host_consts_obj ] + standard_name = ccpp_model_constituents_object + long_name = host-owned model constituent object + units = none + dimensions = () + type = ccpp_model_constituents_t diff --git a/unit-tests/sample_files/host_with_ddt_instance.meta b/unit-tests/sample_files/host_with_ddt_instance.meta new file mode 100644 index 00000000..60528d16 --- /dev/null +++ b/unit-tests/sample_files/host_with_ddt_instance.meta @@ -0,0 +1,16 @@ +# Host table that declares a DDT instance (tests the DDT instance pattern). +# The type 'gfs_statein_type' references the DDT defined in ddt_simple.meta. + +[ccpp-table-properties] + name = CCPP_data + type = host + +[ccpp-arg-table] + name = CCPP_data + type = host +[ gfs_statein ] + standard_name = gfs_statein + long_name = GFS state-in DDT instance + units = none + dimensions = (number_of_instances) + type = gfs_statein_type diff --git a/unit-tests/sample_files/host_with_dependencies.meta b/unit-tests/sample_files/host_with_dependencies.meta new file mode 100644 index 00000000..cf998279 --- /dev/null +++ b/unit-tests/sample_files/host_with_dependencies.meta @@ -0,0 +1,59 @@ +# Host metadata declaring file dependencies via the table-properties +# block. Used to verify that host-table dependencies make it into +# datatable.xml's section (regression test for a bug +# where the generator only walked scheme_tables for deps). + +[ccpp-table-properties] + name = host_with_deps + type = host + dependencies_path = /tmp/fake_phys + dependencies = mp/some_mp_params.F90, radiation/some_rad_param.f + dependencies = chemistry/some_chem.F90 + +[ccpp-arg-table] + name = host_with_deps + type = host +[ ncols ] + standard_name = horizontal_dimension + long_name = full horizontal dimension + units = count + dimensions = () + type = integer +[ im ] + standard_name = horizontal_loop_extent + long_name = horizontal loop extent + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + long_name = number of vertical layers + units = count + dimensions = () + type = integer +[ nlevp1 ] + standard_name = vertical_interface_dimension + long_name = number of vertical layer interfaces + units = count + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = number of model instances + units = count + dimensions = () + type = integer +[ dt ] + standard_name = time_step_for_physics + long_name = physics time step + units = s + dimensions = () + type = real + kind = kind_phys +[ gt0 ] + standard_name = air_temperature + long_name = model layer mean temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys diff --git a/unit-tests/sample_files/host_with_nested_ddt.meta b/unit-tests/sample_files/host_with_nested_ddt.meta new file mode 100644 index 00000000..07eab552 --- /dev/null +++ b/unit-tests/sample_files/host_with_nested_ddt.meta @@ -0,0 +1,16 @@ +# Host table with an instance of ddt_outer_type, which itself contains a +# ddt_inner_type field. Used to test recursive DDT flattening. + +[ccpp-table-properties] + name = nested_host_mod + type = host + +[ccpp-arg-table] + name = nested_host_mod + type = host +[ outer_inst ] + standard_name = outer_ddt_instance + long_name = instance of the outer DDT (scalar, no instance dimension) + units = none + dimensions = () + type = ddt_outer_type diff --git a/unit-tests/sample_files/scheme_chunked_data.meta b/unit-tests/sample_files/scheme_chunked_data.meta new file mode 100644 index 00000000..f37e46a6 --- /dev/null +++ b/unit-tests/sample_files/scheme_chunked_data.meta @@ -0,0 +1,152 @@ +# Scheme metadata for the chunked_data integration test. + +[ccpp-table-properties] + name = chunked_data_scheme + type = scheme + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_init + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ data_array ] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_timestep_init + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ data_array ] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_run + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ nchunk ] + standard_name = ccpp_chunk_number + long_name = current chunk number + units = index + dimensions = () + type = integer + intent = in +[ data_array ] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_timestep_final + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ data_array ] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in + +######################################################################## +[ccpp-arg-table] + name = chunked_data_scheme_final + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ data_array ] + standard_name = chunked_data_array + long_name = chunked data array + units = 1 + dimensions = (horizontal_dimension) + type = integer + intent = in diff --git a/unit-tests/sample_files/scheme_consume_constituent.meta b/unit-tests/sample_files/scheme_consume_constituent.meta new file mode 100644 index 00000000..ab6b216b --- /dev/null +++ b/unit-tests/sample_files/scheme_consume_constituent.meta @@ -0,0 +1,41 @@ +# cam-sima-style scheme that consumes a base constituent and produces +# a tendency. No host or earlier-scheme provider for either — both +# are auto-resolved by capgen-ng via the ccpp_constituents / +# ccpp_constituent_tendencies arrays owned by the suite cap. + +[ccpp-table-properties] + name = consume_constituent + type = scheme + +[ccpp-arg-table] + name = consume_constituent_run + type = scheme +[ cldliq ] + standard_name = cloud_liquid_water_mixing_ratio + long_name = base constituent (read-only) + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in + advected = .true. +[ tend_cldliq ] + standard_name = tendency_of_cloud_liquid_water_mixing_ratio + long_name = tendency of base constituent + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = .true. +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_interstitial_consumer.meta b/unit-tests/sample_files/scheme_interstitial_consumer.meta new file mode 100644 index 00000000..5dd72b84 --- /dev/null +++ b/unit-tests/sample_files/scheme_interstitial_consumer.meta @@ -0,0 +1,32 @@ +# Scheme that consumes the suite-owned interstitial variable. + +[ccpp-table-properties] + name = interstitial_consumer + type = scheme + +[ccpp-arg-table] + name = interstitial_consumer_run + type = scheme +[ diag_in ] + standard_name = diagnostic_interstitial_field + long_name = suite-owned diagnostic field + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_interstitial_producer.meta b/unit-tests/sample_files/scheme_interstitial_producer.meta new file mode 100644 index 00000000..54bb7490 --- /dev/null +++ b/unit-tests/sample_files/scheme_interstitial_producer.meta @@ -0,0 +1,32 @@ +# Scheme that produces a suite-owned interstitial variable. + +[ccpp-table-properties] + name = interstitial_producer + type = scheme + +[ccpp-arg-table] + name = interstitial_producer_run + type = scheme +[ diag_out ] + standard_name = diagnostic_interstitial_field + long_name = suite-owned diagnostic field + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_module_name_override.meta b/unit-tests/sample_files/scheme_module_name_override.meta new file mode 100644 index 00000000..f5902861 --- /dev/null +++ b/unit-tests/sample_files/scheme_module_name_override.meta @@ -0,0 +1,32 @@ +# Scheme metadata that declares ``module_name`` in [ccpp-table-properties] +# distinct from the table ``name``. Used to verify capgen-ng emits +# ``use mod_alt_name, only: ...`` (the explicit module name) rather than +# ``use scheme_alt_name``. + +[ccpp-table-properties] + name = scheme_alt_name + type = scheme + module_name = mod_alt_name + +[ccpp-arg-table] + name = scheme_alt_name_run + type = scheme +[ im ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_multipart.meta b/unit-tests/sample_files/scheme_multipart.meta new file mode 100644 index 00000000..82e04c20 --- /dev/null +++ b/unit-tests/sample_files/scheme_multipart.meta @@ -0,0 +1,87 @@ +# Scheme metadata with three phases: init, run, final. +# Tests the redesign phase naming: 'final' (not 'finalize'). + +[ccpp-table-properties] + name = temp_calc_adjust + type = scheme + +[ccpp-arg-table] + name = temp_calc_adjust_init + type = scheme +[ im ] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = temp_calc_adjust_run + type = scheme +[ im ] + standard_name = horizontal_dimension + long_name = horizontal loop extent + units = count + dimensions = () + type = integer + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = physics time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp ] + standard_name = air_temperature + long_name = model layer mean temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = temp_calc_adjust_final + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_multipart_correct.F90 b/unit-tests/sample_files/scheme_multipart_correct.F90 new file mode 100644 index 00000000..e56fec95 --- /dev/null +++ b/unit-tests/sample_files/scheme_multipart_correct.F90 @@ -0,0 +1,34 @@ +module temp_calc_adjust + + implicit none + private + +contains + + subroutine temp_calc_adjust_init(im, errmsg, errflg) + integer, intent(in) :: im + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + errmsg = '' + errflg = 0 + end subroutine temp_calc_adjust_init + + subroutine temp_calc_adjust_run(im, timestep, temp, & + errmsg, errflg) + integer, intent(in) :: im + real, intent(in) :: timestep + real, intent(inout) :: temp(:,:) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + errmsg = '' + errflg = 0 + end subroutine temp_calc_adjust_run + + subroutine temp_calc_adjust_final(errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + errmsg = '' + errflg = 0 + end subroutine temp_calc_adjust_final + +end module temp_calc_adjust diff --git a/unit-tests/sample_files/scheme_multipart_wrong_args.F90 b/unit-tests/sample_files/scheme_multipart_wrong_args.F90 new file mode 100644 index 00000000..043eea28 --- /dev/null +++ b/unit-tests/sample_files/scheme_multipart_wrong_args.F90 @@ -0,0 +1,28 @@ +module temp_calc_adjust + + implicit none + private + +contains + + ! init has wrong arg count: missing errflg + subroutine temp_calc_adjust_init(im, errmsg) + integer, intent(in) :: im + character(len=*), intent(out) :: errmsg + end subroutine temp_calc_adjust_init + + ! run has a renamed arg (tempo instead of temp) + subroutine temp_calc_adjust_run(im, timestep, tempo, errmsg, errflg) + integer, intent(in) :: im + real, intent(in) :: timestep + real, intent(inout) :: tempo(:,:) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + end subroutine temp_calc_adjust_run + + subroutine temp_calc_adjust_final(errmsg, errflg) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + end subroutine temp_calc_adjust_final + +end module temp_calc_adjust diff --git a/unit-tests/sample_files/scheme_opt_arg.meta b/unit-tests/sample_files/scheme_opt_arg.meta new file mode 100644 index 00000000..af443aac --- /dev/null +++ b/unit-tests/sample_files/scheme_opt_arg.meta @@ -0,0 +1,163 @@ +# Scheme metadata for the opt_arg integration test. +# Tests Case 2 (optional integer, no transform) and Case 4 +# (optional real in m; host has km → unit conversion km→m). + +[ccpp-table-properties] + name = opt_arg_scheme + type = scheme + +######################################################################## +[ccpp-arg-table] + name = opt_arg_scheme_timestep_init + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ nx ] + standard_name = size_of_std_arg + long_name = size of std_arg array + units = count + dimensions = () + type = integer + intent = in +[ var ] + standard_name = std_arg + long_name = mandatory integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = in +[ opt_var ] + standard_name = opt_arg + long_name = optional integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = out + optional = True +[ opt_var_2 ] + standard_name = opt_arg_2 + long_name = optional real array in m + units = m + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + intent = out + optional = True + +######################################################################## +[ccpp-arg-table] + name = opt_arg_scheme_run + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ nx ] + standard_name = size_of_std_arg + long_name = size of std_arg array + units = count + dimensions = () + type = integer + intent = in +[ var ] + standard_name = std_arg + long_name = mandatory integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = inout +[ opt_var ] + standard_name = opt_arg + long_name = optional integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = inout + optional = True +[ opt_var_2 ] + standard_name = opt_arg_2 + long_name = optional real array in m + units = m + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + intent = inout + optional = True + +######################################################################## +[ccpp-arg-table] + name = opt_arg_scheme_timestep_final + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ nx ] + standard_name = size_of_std_arg + long_name = size of std_arg array + units = count + dimensions = () + type = integer + intent = in +[ var ] + standard_name = std_arg + long_name = mandatory integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = inout +[ opt_var ] + standard_name = opt_arg + long_name = optional integer array + units = 1 + dimensions = (size_of_std_arg) + type = integer + intent = in + optional = True +[ opt_var_2 ] + standard_name = opt_arg_2 + long_name = optional real array in m + units = m + dimensions = (size_of_std_arg) + type = real + kind = kind_phys + intent = inout + optional = True diff --git a/unit-tests/sample_files/scheme_register_constituents.meta b/unit-tests/sample_files/scheme_register_constituents.meta new file mode 100644 index 00000000..6dd9e6a1 --- /dev/null +++ b/unit-tests/sample_files/scheme_register_constituents.meta @@ -0,0 +1,33 @@ +# Scheme that registers dynamic constituents during ``_register``. +# The register-phase intent=out arg is an allocatable array of +# ccpp_constituent_properties_t, the special type the suite cap detects +# and feeds through the two-pass merge into the host's +# ccpp_model_constituents_object. + +[ccpp-table-properties] + name = register_constituents + type = scheme + +[ccpp-arg-table] + name = register_constituents_register + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_register_test + long_name = per-scheme constituent array + units = none + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_register_dim_consumer.meta b/unit-tests/sample_files/scheme_register_dim_consumer.meta new file mode 100644 index 00000000..8826e739 --- /dev/null +++ b/unit-tests/sample_files/scheme_register_dim_consumer.meta @@ -0,0 +1,31 @@ +# Scheme that uses the suite-owned scalar dimension produced by +# register_dim_producer's _register phase to dimension a suite-owned +# interstitial array. + +[ccpp-table-properties] + name = register_dim_consumer + type = scheme + +[ccpp-arg-table] + name = register_dim_consumer_run + type = scheme +[ interstitial_var ] + standard_name = output_only_interstitial_variable + long_name = suite-owned interstitial dimensioned by register-set scalar + units = 1 + dimensions = (dimension_for_interstitial_variable) + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_register_dim_producer.meta b/unit-tests/sample_files/scheme_register_dim_producer.meta new file mode 100644 index 00000000..03479ba2 --- /dev/null +++ b/unit-tests/sample_files/scheme_register_dim_producer.meta @@ -0,0 +1,32 @@ +# Scheme that declares a suite-owned scalar dimension during _register. +# Mirrors the temp_calc_adjust_register pattern: an integer scalar set in +# the register phase that is later used as the upper bound of an +# interstitial array in another scheme's run phase. + +[ccpp-table-properties] + name = register_dim_producer + type = scheme + +[ccpp-arg-table] + name = register_dim_producer_register + type = scheme +[ dim_inter ] + standard_name = dimension_for_interstitial_variable + long_name = size of suite-owned interstitial dimension + units = count + dimensions = () + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_suite_init_final.meta b/unit-tests/sample_files/scheme_suite_init_final.meta new file mode 100644 index 00000000..346b62b2 --- /dev/null +++ b/unit-tests/sample_files/scheme_suite_init_final.meta @@ -0,0 +1,42 @@ +# Scheme used for the suite-level / integration test. +# Declares minimal init and final phases with only the standard error +# args, so the resolved suite-init/-final calls don't pull in any +# host-side state and the generated cap stays compact. + +[ccpp-table-properties] + name = suite_init_final_scheme + type = scheme + +[ccpp-arg-table] + name = suite_init_final_scheme_init + type = scheme +[errmsg] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[errflg] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = suite_init_final_scheme_final + type = scheme +[errmsg] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[errflg] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_top_at_one.meta b/unit-tests/sample_files/scheme_top_at_one.meta new file mode 100644 index 00000000..578dc1a9 --- /dev/null +++ b/unit-tests/sample_files/scheme_top_at_one.meta @@ -0,0 +1,34 @@ +# Scheme that requests air_temperature with top_at_one = True. +# Pairs with host_full.meta (which declares air_temperature with the +# default top_at_one = False), so the resolver should emit a vertical-flip +# transform on the host-side subscript. + +[ccpp-table-properties] + name = top_at_one_scheme + type = scheme + +[ccpp-arg-table] + name = top_at_one_scheme_run + type = scheme +[ temp ] + standard_name = air_temperature + long_name = model layer mean temperature (top-down) + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + top_at_one = True +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_files/scheme_unit_conv_1.meta b/unit-tests/sample_files/scheme_unit_conv_1.meta new file mode 100644 index 00000000..726c7562 --- /dev/null +++ b/unit-tests/sample_files/scheme_unit_conv_1.meta @@ -0,0 +1,45 @@ +# Scheme 1 for the unit_conv integration test. +# Expects data_array in m (no conversion — Case 1) and +# data_array_opt in m (optional, no conversion — Case 2). + +[ccpp-table-properties] + name = unit_conv_scheme_1 + type = scheme + +######################################################################## +[ccpp-arg-table] + name = unit_conv_scheme_1_run + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ data_array ] + standard_name = data_array + long_name = data array in m + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[ data_array_opt ] + standard_name = data_array_opt + long_name = optional data array in m + units = m + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout + optional = True diff --git a/unit-tests/sample_files/scheme_unit_conv_2.meta b/unit-tests/sample_files/scheme_unit_conv_2.meta new file mode 100644 index 00000000..1f50f5ad --- /dev/null +++ b/unit-tests/sample_files/scheme_unit_conv_2.meta @@ -0,0 +1,45 @@ +# Scheme 2 for the unit_conv integration test. +# Expects data_array in km (unit conversion m→km — Case 3) and +# data_array_opt in km (optional + unit conversion — Case 4). + +[ccpp-table-properties] + name = unit_conv_scheme_2 + type = scheme + +######################################################################## +[ccpp-arg-table] + name = unit_conv_scheme_2_run + type = scheme + +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character + kind = len=* + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag + units = 1 + dimensions = () + type = integer + intent = out +[ data_array ] + standard_name = data_array + long_name = data array in km + units = km + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout +[ data_array_opt ] + standard_name = data_array_opt + long_name = optional data array in km + units = km + dimensions = (horizontal_dimension) + type = real + kind = kind_phys + intent = inout + optional = True diff --git a/unit-tests/sample_suite_files/another_suite.xml b/unit-tests/sample_suite_files/another_suite.xml new file mode 100644 index 00000000..72346933 --- /dev/null +++ b/unit-tests/sample_suite_files/another_suite.xml @@ -0,0 +1,10 @@ + + + + + + another_scheme + + more_scheme + + diff --git a/unit-tests/sample_suite_files/another_suite2.xml b/unit-tests/sample_suite_files/another_suite2.xml new file mode 100644 index 00000000..def97177 --- /dev/null +++ b/unit-tests/sample_suite_files/another_suite2.xml @@ -0,0 +1,16 @@ + + + + + + another_scheme + + more_scheme + + + + another_scheme + + more_scheme + + diff --git a/unit-tests/sample_suite_files/nested_full_suite.xml b/unit-tests/sample_suite_files/nested_full_suite.xml new file mode 100644 index 00000000..2979f3ff --- /dev/null +++ b/unit-tests/sample_suite_files/nested_full_suite.xml @@ -0,0 +1,10 @@ + + + + + g1_scheme1 + + + + + diff --git a/unit-tests/sample_suite_files/subsuite1.xml b/unit-tests/sample_suite_files/subsuite1.xml new file mode 100644 index 00000000..c58ed752 --- /dev/null +++ b/unit-tests/sample_suite_files/subsuite1.xml @@ -0,0 +1,7 @@ + + + + + scheme_subsuite1 + + diff --git a/unit-tests/sample_suite_files/subsuite_inline.xml b/unit-tests/sample_suite_files/subsuite_inline.xml new file mode 100644 index 00000000..706ea801 --- /dev/null +++ b/unit-tests/sample_suite_files/subsuite_inline.xml @@ -0,0 +1,9 @@ + + + + + scheme1i + scheme2i + scheme1i + + diff --git a/unit-tests/sample_suite_files/suite_bad_v2_duplicate_group.xml b/unit-tests/sample_suite_files/suite_bad_v2_duplicate_group.xml new file mode 100644 index 00000000..8ea72077 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_bad_v2_duplicate_group.xml @@ -0,0 +1,16 @@ + + + + + + effr_pre + + + scheme9 + + + scheme3 + + + + diff --git a/unit-tests/sample_suite_files/suite_bad_v2_suite_tag.xml b/unit-tests/sample_suite_files/suite_bad_v2_suite_tag.xml new file mode 100644 index 00000000..6bc4f424 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_bad_v2_suite_tag.xml @@ -0,0 +1,7 @@ + + + + + subsuite_inline + + diff --git a/unit-tests/sample_suite_files/suite_bad_version01.xml b/unit-tests/sample_suite_files/suite_bad_version01.xml new file mode 100644 index 00000000..ecee0c63 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_bad_version01.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/unit-tests/sample_suite_files/suite_bad_version02.xml b/unit-tests/sample_suite_files/suite_bad_version02.xml new file mode 100644 index 00000000..55aff67a --- /dev/null +++ b/unit-tests/sample_suite_files/suite_bad_version02.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/unit-tests/sample_suite_files/suite_bad_version03.xml b/unit-tests/sample_suite_files/suite_bad_version03.xml new file mode 100644 index 00000000..794bfe7b --- /dev/null +++ b/unit-tests/sample_suite_files/suite_bad_version03.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/unit-tests/sample_suite_files/suite_bad_version04.xml b/unit-tests/sample_suite_files/suite_bad_version04.xml new file mode 100644 index 00000000..aaaab154 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_bad_version04.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/unit-tests/sample_suite_files/suite_chunked_data.xml b/unit-tests/sample_suite_files/suite_chunked_data.xml new file mode 100644 index 00000000..731fb6dd --- /dev/null +++ b/unit-tests/sample_suite_files/suite_chunked_data.xml @@ -0,0 +1,9 @@ + + + + + + chunked_data_scheme + + + diff --git a/unit-tests/sample_suite_files/suite_consume_constituent.xml b/unit-tests/sample_suite_files/suite_consume_constituent.xml new file mode 100644 index 00000000..3c0297cb --- /dev/null +++ b/unit-tests/sample_suite_files/suite_consume_constituent.xml @@ -0,0 +1,6 @@ + + + + consume_constituent + + diff --git a/unit-tests/sample_suite_files/suite_good_v1_test01.xml b/unit-tests/sample_suite_files/suite_good_v1_test01.xml new file mode 100644 index 00000000..0eee366b --- /dev/null +++ b/unit-tests/sample_suite_files/suite_good_v1_test01.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/unit-tests/sample_suite_files/suite_good_v1_test02.xml b/unit-tests/sample_suite_files/suite_good_v1_test02.xml new file mode 100644 index 00000000..3d355cc5 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_good_v1_test02.xml @@ -0,0 +1,11 @@ + + + + + scheme1 + scheme2 + scheme3 + scheme2 + scheme1 + + diff --git a/unit-tests/sample_suite_files/suite_good_v2_test01.xml b/unit-tests/sample_suite_files/suite_good_v2_test01.xml new file mode 100644 index 00000000..73c30732 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_good_v2_test01.xml @@ -0,0 +1,9 @@ + + + + + scheme5 + + scheme9 + + diff --git a/unit-tests/sample_suite_files/suite_good_v2_test01_exp.xml b/unit-tests/sample_suite_files/suite_good_v2_test01_exp.xml new file mode 100644 index 00000000..dcecf218 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_good_v2_test01_exp.xml @@ -0,0 +1,11 @@ + + + + + scheme5 + scheme1i + scheme2i + scheme1i + scheme9 + + diff --git a/unit-tests/sample_suite_files/suite_good_v2_test02.xml b/unit-tests/sample_suite_files/suite_good_v2_test02.xml new file mode 100644 index 00000000..c7b5b8c9 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_good_v2_test02.xml @@ -0,0 +1,10 @@ + + + + + + scheme6 + + + + diff --git a/unit-tests/sample_suite_files/suite_good_v2_test02_exp.xml b/unit-tests/sample_suite_files/suite_good_v2_test02_exp.xml new file mode 100644 index 00000000..6b100283 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_good_v2_test02_exp.xml @@ -0,0 +1,13 @@ + + + + + + scheme6 + + + another_scheme + + more_scheme + + diff --git a/unit-tests/sample_suite_files/suite_good_v2_test03.xml b/unit-tests/sample_suite_files/suite_good_v2_test03.xml new file mode 100644 index 00000000..eff15298 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_good_v2_test03.xml @@ -0,0 +1,19 @@ + + + + + scheme13 + + effr_pre + + + main_calc + + + main_post + + + + + + diff --git a/unit-tests/sample_suite_files/suite_good_v2_test03_exp.xml b/unit-tests/sample_suite_files/suite_good_v2_test03_exp.xml new file mode 100644 index 00000000..5f9a9987 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_good_v2_test03_exp.xml @@ -0,0 +1,30 @@ + + + + + scheme13 + + effr_pre + + + main_calc + + + main_post + + + another_scheme + + more_scheme + + another_scheme + + more_scheme + + + g1_scheme1 + + + scheme_subsuite1 + + diff --git a/unit-tests/sample_suite_files/suite_good_v2_test04.xml b/unit-tests/sample_suite_files/suite_good_v2_test04.xml new file mode 100644 index 00000000..abb87008 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_good_v2_test04.xml @@ -0,0 +1,18 @@ + + + + + + effr_pre + + + main_calc + + + main_post + + + + + + diff --git a/unit-tests/sample_suite_files/suite_good_v2_test04_exp.xml b/unit-tests/sample_suite_files/suite_good_v2_test04_exp.xml new file mode 100644 index 00000000..af103e7d --- /dev/null +++ b/unit-tests/sample_suite_files/suite_good_v2_test04_exp.xml @@ -0,0 +1,26 @@ + + + + + + effr_pre + + + main_calc + + + main_post + + + another_scheme + + more_scheme + + another_scheme + + more_scheme + + + scheme_subsuite1 + + diff --git a/unit-tests/sample_suite_files/suite_interstitial.xml b/unit-tests/sample_suite_files/suite_interstitial.xml new file mode 100644 index 00000000..2938e9ee --- /dev/null +++ b/unit-tests/sample_suite_files/suite_interstitial.xml @@ -0,0 +1,7 @@ + + + + interstitial_producer + interstitial_consumer + + diff --git a/unit-tests/sample_suite_files/suite_invalid_group_fortran_id.xml b/unit-tests/sample_suite_files/suite_invalid_group_fortran_id.xml new file mode 100644 index 00000000..0fbb40dc --- /dev/null +++ b/unit-tests/sample_suite_files/suite_invalid_group_fortran_id.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/unit-tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml b/unit-tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml new file mode 100644 index 00000000..e236cbd0 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_invalid_scheme_fortran_id.xml @@ -0,0 +1,8 @@ + + + + + scheme-1 + scheme2 + + diff --git a/unit-tests/sample_suite_files/suite_invalid_suite_fortran_id.xml b/unit-tests/sample_suite_files/suite_invalid_suite_fortran_id.xml new file mode 100644 index 00000000..ae8a28a4 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_invalid_suite_fortran_id.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/unit-tests/sample_suite_files/suite_missing_file.xml b/unit-tests/sample_suite_files/suite_missing_file.xml new file mode 100644 index 00000000..8c7b85e6 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_missing_file.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/unit-tests/sample_suite_files/suite_missing_group.xml b/unit-tests/sample_suite_files/suite_missing_group.xml new file mode 100644 index 00000000..a33078e6 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_missing_group.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/unit-tests/sample_suite_files/suite_missing_loaded_suite.xml b/unit-tests/sample_suite_files/suite_missing_loaded_suite.xml new file mode 100644 index 00000000..cf21a590 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_missing_loaded_suite.xml @@ -0,0 +1,16 @@ + + + + + + scheme23 + + + scheme9 + + + scheme3 + + + + diff --git a/unit-tests/sample_suite_files/suite_missing_version.xml b/unit-tests/sample_suite_files/suite_missing_version.xml new file mode 100644 index 00000000..463c56d9 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_missing_version.xml @@ -0,0 +1,8 @@ + + + + + scheme1 + scheme2 + + diff --git a/unit-tests/sample_suite_files/suite_module_name_override.xml b/unit-tests/sample_suite_files/suite_module_name_override.xml new file mode 100644 index 00000000..6df5b815 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_module_name_override.xml @@ -0,0 +1,6 @@ + + + + scheme_alt_name + + diff --git a/unit-tests/sample_suite_files/suite_nested_subcycle.xml b/unit-tests/sample_suite_files/suite_nested_subcycle.xml new file mode 100644 index 00000000..7fc99bdd --- /dev/null +++ b/unit-tests/sample_suite_files/suite_nested_subcycle.xml @@ -0,0 +1,10 @@ + + + + + + temp_calc_adjust + + + + diff --git a/unit-tests/sample_suite_files/suite_opt_arg.xml b/unit-tests/sample_suite_files/suite_opt_arg.xml new file mode 100644 index 00000000..496c6fbf --- /dev/null +++ b/unit-tests/sample_suite_files/suite_opt_arg.xml @@ -0,0 +1,9 @@ + + + + + + opt_arg_scheme + + + diff --git a/unit-tests/sample_suite_files/suite_recurse_level2.xml b/unit-tests/sample_suite_files/suite_recurse_level2.xml new file mode 100644 index 00000000..35fed8b2 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_recurse_level2.xml @@ -0,0 +1,10 @@ + + + + + scheme13 + scheme3 + + scheme43 + + diff --git a/unit-tests/sample_suite_files/suite_recurse_level2a.xml b/unit-tests/sample_suite_files/suite_recurse_level2a.xml new file mode 100644 index 00000000..2e018e39 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_recurse_level2a.xml @@ -0,0 +1,10 @@ + + + + + scheme13 + scheme3 + scheme43 + + + diff --git a/unit-tests/sample_suite_files/suite_recurse_level3.xml b/unit-tests/sample_suite_files/suite_recurse_level3.xml new file mode 100644 index 00000000..f38a9d8f --- /dev/null +++ b/unit-tests/sample_suite_files/suite_recurse_level3.xml @@ -0,0 +1,10 @@ + + + + + scheme13 + scheme3 + + scheme43 + + diff --git a/unit-tests/sample_suite_files/suite_recurse_level3a.xml b/unit-tests/sample_suite_files/suite_recurse_level3a.xml new file mode 100644 index 00000000..a11182dc --- /dev/null +++ b/unit-tests/sample_suite_files/suite_recurse_level3a.xml @@ -0,0 +1,10 @@ + + + + + scheme13 + scheme3 + scheme43 + + + diff --git a/unit-tests/sample_suite_files/suite_recurse_top1.xml b/unit-tests/sample_suite_files/suite_recurse_top1.xml new file mode 100644 index 00000000..8e8c4f1a --- /dev/null +++ b/unit-tests/sample_suite_files/suite_recurse_top1.xml @@ -0,0 +1,18 @@ + + + + + scheme13 + + scheme23 + + + scheme9 + + + scheme3 + + + scheme43 + + diff --git a/unit-tests/sample_suite_files/suite_recurse_top2.xml b/unit-tests/sample_suite_files/suite_recurse_top2.xml new file mode 100644 index 00000000..87ce7057 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_recurse_top2.xml @@ -0,0 +1,18 @@ + + + + + scheme13 + + scheme23 + + + scheme9 + + + scheme3 + + scheme43 + + + diff --git a/unit-tests/sample_suite_files/suite_register_constituents.xml b/unit-tests/sample_suite_files/suite_register_constituents.xml new file mode 100644 index 00000000..de03eb38 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_register_constituents.xml @@ -0,0 +1,6 @@ + + + + register_constituents + + diff --git a/unit-tests/sample_suite_files/suite_register_dim.xml b/unit-tests/sample_suite_files/suite_register_dim.xml new file mode 100644 index 00000000..0fc85811 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_register_dim.xml @@ -0,0 +1,7 @@ + + + + register_dim_producer + register_dim_consumer + + diff --git a/unit-tests/sample_suite_files/suite_subcycle_stdname.xml b/unit-tests/sample_suite_files/suite_subcycle_stdname.xml new file mode 100644 index 00000000..0f2472a5 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_subcycle_stdname.xml @@ -0,0 +1,8 @@ + + + + + temp_calc_adjust + + + diff --git a/unit-tests/sample_suite_files/suite_subcycle_stdname_ddt.xml b/unit-tests/sample_suite_files/suite_subcycle_stdname_ddt.xml new file mode 100644 index 00000000..1c39345d --- /dev/null +++ b/unit-tests/sample_suite_files/suite_subcycle_stdname_ddt.xml @@ -0,0 +1,8 @@ + + + + + temp_calc_adjust + + + diff --git a/unit-tests/sample_suite_files/suite_test_simple.xml b/unit-tests/sample_suite_files/suite_test_simple.xml new file mode 100644 index 00000000..9a8319ae --- /dev/null +++ b/unit-tests/sample_suite_files/suite_test_simple.xml @@ -0,0 +1,6 @@ + + + + temp_calc_adjust + + diff --git a/unit-tests/sample_suite_files/suite_test_subcycle.xml b/unit-tests/sample_suite_files/suite_test_subcycle.xml new file mode 100644 index 00000000..d7d9a6ba --- /dev/null +++ b/unit-tests/sample_suite_files/suite_test_subcycle.xml @@ -0,0 +1,8 @@ + + + + + temp_calc_adjust + + + diff --git a/unit-tests/sample_suite_files/suite_top_at_one.xml b/unit-tests/sample_suite_files/suite_top_at_one.xml new file mode 100644 index 00000000..cc67b1b5 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_top_at_one.xml @@ -0,0 +1,6 @@ + + + + top_at_one_scheme + + diff --git a/unit-tests/sample_suite_files/suite_unit_conv.xml b/unit-tests/sample_suite_files/suite_unit_conv.xml new file mode 100644 index 00000000..f59cf22a --- /dev/null +++ b/unit-tests/sample_suite_files/suite_unit_conv.xml @@ -0,0 +1,10 @@ + + + + + + unit_conv_scheme_1 + unit_conv_scheme_2 + + + diff --git a/unit-tests/sample_suite_files/suite_with_init_final.xml b/unit-tests/sample_suite_files/suite_with_init_final.xml new file mode 100644 index 00000000..8f11e61a --- /dev/null +++ b/unit-tests/sample_suite_files/suite_with_init_final.xml @@ -0,0 +1,8 @@ + + + suite_init_final_scheme + + temp_calc_adjust + + suite_init_final_scheme + diff --git a/unit-tests/test_ccpp_datafile.py b/unit-tests/test_ccpp_datafile.py new file mode 100644 index 00000000..592cf08f --- /dev/null +++ b/unit-tests/test_ccpp_datafile.py @@ -0,0 +1,334 @@ +"""Tests for the ccpp_datafile query CLI. + +Covers each of the 17 CLI flags end-to-end by: + 1. building a real datatable.xml via the writer in generator.datatable, + 2. invoking datatable_report / datatable_pretty_print on it, + 3. asserting the textual output. +""" + +import doctest +import os +import shutil +import sys +import tempfile +import unittest + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen-ng') +for _p in (_CAPGEN_DIR, _TESTS_DIR): + if _p not in sys.path: + sys.path.insert(0, _p) + +import ccpp_datafile as cdf +from ccpp_datafile import ( + DatatableReport, + datatable_pretty_print, + datatable_report, +) +from generator.datatable import write_datatable + +from test_suite_resolver import ( + _load_full_host_dict, + _load_scheme_store, + _parse_suite, +) +from generator.suite_resolver import resolve_suite + + +def _build_datatable(tmpdir, host_name='test_host', + host_file_paths=None, utility_paths=None, + suite_file_paths=None, dependency_paths=None, + suite_meta_paths=None, expanded_sdf_paths=None, + protect_first_host_var=False): + """Build a real datatable.xml in *tmpdir* and return its path.""" + hd = _load_full_host_dict() + if protect_first_host_var: + first = next(iter(hd)) + hd[first].protected = True + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + sr = resolve_suite(suite, store, hd) + return write_datatable( + [sr], + store, + utility_paths or ['/out/ccpp_kinds.F90'], + suite_file_paths or ['/out/ccpp_test_simple_cap.F90', + '/out/ccpp_test_simple_physics_cap.F90'], + tmpdir, + host_file_paths=host_file_paths or ['/out/ccpp_static_api.F90'], + dependency_paths=dependency_paths or [], + suite_meta_paths=suite_meta_paths, + expanded_sdf_paths=expanded_sdf_paths, + host_dict=hd, + host_name=host_name, + ) + + +class _DTBase(unittest.TestCase): + """Shared fixture: build one datatable.xml per test class.""" + + @classmethod + def setUpClass(cls): + cls._tmpdir = tempfile.mkdtemp() + cls._datatable = _build_datatable(cls._tmpdir) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls._tmpdir) + + +class TestDatatableReportFileActions(_DTBase): + + def test_host_files(self): + out = datatable_report(self._datatable, + DatatableReport('host_files'), ',') + self.assertEqual(out, '/out/ccpp_static_api.F90') + + def test_suite_files(self): + out = datatable_report(self._datatable, + DatatableReport('suite_files'), ',') + items = out.split(',') + self.assertIn('/out/ccpp_test_simple_cap.F90', items) + self.assertIn('/out/ccpp_test_simple_physics_cap.F90', items) + + def test_utility_files(self): + out = datatable_report(self._datatable, + DatatableReport('utility_files'), ',') + self.assertEqual(out, '/out/ccpp_kinds.F90') + + def test_capgen_files_returns_all(self): + out = datatable_report(self._datatable, + DatatableReport('capgen_files'), ',') + items = out.split(',') + self.assertIn('/out/ccpp_kinds.F90', items) + self.assertIn('/out/ccpp_static_api.F90', items) + self.assertIn('/out/ccpp_test_simple_cap.F90', items) + + def test_separator_honored(self): + out = datatable_report(self._datatable, + DatatableReport('suite_files'), ';') + self.assertIn(';', out) + self.assertNotIn(',', out) + + +class TestDatatableReportInspectionFiles(unittest.TestCase): + """--inspection-files returns meta + expanded SDF paths and excludes them + from --capgen-files.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._datatable = _build_datatable( + self._tmpdir, + suite_meta_paths=['/out/ccpp_test_simple.meta'], + expanded_sdf_paths=['/out/ccpp_test_simple_expanded.xml'], + ) + + def tearDown(self): + shutil.rmtree(self._tmpdir) + + def test_inspection_files_returns_meta_and_expanded(self): + out = datatable_report(self._datatable, + DatatableReport('inspection_files'), ',') + items = out.split(',') + self.assertIn('/out/ccpp_test_simple.meta', items) + self.assertIn('/out/ccpp_test_simple_expanded.xml', items) + + def test_inspection_files_excluded_from_capgen_files(self): + out = datatable_report(self._datatable, + DatatableReport('capgen_files'), ',') + items = out.split(',') + self.assertNotIn('/out/ccpp_test_simple.meta', items) + self.assertNotIn('/out/ccpp_test_simple_expanded.xml', items) + + def test_inspection_files_empty_when_none_given(self): + # Build a datatable with no inspection paths; the section is still + # present, and --inspection-files returns an empty string. + with tempfile.TemporaryDirectory() as d: + path = _build_datatable(d) + out = datatable_report(path, + DatatableReport('inspection_files'), ',') + self.assertEqual(out, '') + + +class TestDatatableReportSchemeActions(_DTBase): + + def test_process_list_empty(self): + # capgen-ng does not emit attrs. + out = datatable_report(self._datatable, + DatatableReport('process_list'), ',') + self.assertEqual(out, '') + + def test_module_list_includes_scheme_modules(self): + out = datatable_report(self._datatable, + DatatableReport('module_list'), ',') + modules = out.split(',') + self.assertIn('temp_calc_adjust', modules) + + def test_dependencies_empty_when_none(self): + out = datatable_report(self._datatable, + DatatableReport('dependencies'), ',') + self.assertEqual(out, '') + + +class TestDatatableReportDependenciesPopulated(unittest.TestCase): + """--dependencies returns the sorted, dedup'd dependency list.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._datatable = _build_datatable( + self._tmpdir, + dependency_paths=['/dep/b.F90', '/dep/a.F90', '/dep/a.F90'], + ) + + def tearDown(self): + shutil.rmtree(self._tmpdir) + + def test_dependencies_sorted_dedup(self): + out = datatable_report(self._datatable, + DatatableReport('dependencies'), ',') + self.assertEqual(out, '/dep/a.F90,/dep/b.F90') + + +class TestDatatableReportSuiteList(_DTBase): + + def test_suite_list(self): + out = datatable_report(self._datatable, + DatatableReport('suite_list'), ',') + self.assertEqual(out, 'test_simple') + + +class TestDatatableReportVariableActions(_DTBase): + + def test_required_variables_includes_call_list_vars(self): + out = datatable_report( + self._datatable, + DatatableReport('required_variables', 'test_simple'), ',') + names = out.split(',') + self.assertIn('air_temperature', names) + + def test_input_variables_excludes_out_only(self): + # An intent=out var must not appear in --input-variables. + out = datatable_report( + self._datatable, + DatatableReport('input_variables', 'test_simple'), ',') + names = set(out.split(',')) + # ccpp_error_code is intent=out → not in input list. + self.assertNotIn('ccpp_error_code', names) + + def test_output_variables_excludes_in_only(self): + out = datatable_report( + self._datatable, + DatatableReport('output_variables', 'test_simple'), ',') + names = set(out.split(',')) + self.assertIn('ccpp_error_code', names) + + def test_host_variables_returns_host_names(self): + out = datatable_report(self._datatable, + DatatableReport('host_variables'), ',') + names = set(out.split(',')) + self.assertIn('air_temperature', names) + + def test_unknown_suite_returns_empty(self): + out = datatable_report( + self._datatable, + DatatableReport('required_variables', 'no_such_suite'), ',') + self.assertEqual(out, '') + + +class TestExcludeProtected(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._datatable = _build_datatable( + self._tmpdir, protect_first_host_var=True, + ) + hd = _load_full_host_dict() + self._protected_name = next(iter(hd)) + + def tearDown(self): + shutil.rmtree(self._tmpdir) + + def test_protected_excluded_from_host_when_flag_on(self): + out = datatable_report( + self._datatable, + DatatableReport('host_variables'), ',', exclude_protected=True) + names = set(out.split(',')) + self.assertNotIn(self._protected_name, names) + + def test_protected_included_when_flag_off(self): + out = datatable_report( + self._datatable, + DatatableReport('host_variables'), ',', exclude_protected=False) + names = set(out.split(',')) + self.assertIn(self._protected_name, names) + + +class TestShowAction(_DTBase): + + def test_show_returns_string(self): + out = datatable_pretty_print(self._datatable, indent=2, line_wrap=-1) + self.assertIsInstance(out, str) + self.assertIn('; it must not now.""" + self.assertIsNone(self._capgen_files().find('suite_meta_files')) + + +class TestInspectionFilesSection(unittest.TestCase): + """ collects non-Fortran artifacts (meta + expanded SDF).""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + path, _, _ = _write( + self._tmpdir, + suite_meta_paths=['/out/ccpp_test_simple.meta'], + expanded_sdf_paths=['/out/ccpp_test_simple_expanded.xml'], + ) + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _inspection(self): + return self._root.find('inspection_files') + + def test_inspection_files_element_present(self): + self.assertIsNotNone(self._inspection()) + + def test_suite_meta_files_subsection(self): + meta = self._inspection().find('suite_meta_files') + self.assertIsNotNone(meta) + files = [f.text for f in meta.findall('file')] + self.assertEqual(files, ['/out/ccpp_test_simple.meta']) + + def test_expanded_sdf_files_subsection(self): + exp = self._inspection().find('expanded_sdf_files') + self.assertIsNotNone(exp) + files = [f.text for f in exp.findall('file')] + self.assertEqual(files, ['/out/ccpp_test_simple_expanded.xml']) + + def test_empty_subsections_when_no_paths(self): + """Section + both subsections are always written, even when empty.""" + with tempfile.TemporaryDirectory() as d: + path, _, _ = _write(d) # no meta / expanded paths + root = ET.parse(path).getroot() + insp = root.find('inspection_files') + self.assertIsNotNone(insp) + self.assertIsNotNone(insp.find('suite_meta_files')) + self.assertIsNotNone(insp.find('expanded_sdf_files')) + self.assertEqual(len(insp.find('suite_meta_files').findall('file')), 0) + self.assertEqual(len(insp.find('expanded_sdf_files').findall('file')), 0) + + +class TestSchemesSection(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + path, _, _ = _write(self._tmpdir) + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _schemes(self): + return self._root.find('schemes') + + def test_schemes_element_present(self): + self.assertIsNotNone(self._schemes()) + + def test_scheme_name_present(self): + names = [s.get('name') for s in self._schemes().findall('scheme')] + self.assertIn('temp_calc_adjust', names) + + def test_no_duplicate_scheme_elements(self): + names = [s.get('name') for s in self._schemes().findall('scheme')] + self.assertEqual(len(names), len(set(names))) + + def test_run_phase_element(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + phase_tags = [child.tag for child in scheme] + self.assertIn('run', phase_tags) + + def test_run_phase_attributes(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + run_elem = scheme.find('run') + self.assertIsNotNone(run_elem) + self.assertEqual(run_elem.get('name'), 'temp_calc_adjust') + self.assertEqual(run_elem.get('subroutine_name'), 'temp_calc_adjust_run') + self.assertEqual(run_elem.get('module'), 'temp_calc_adjust') + + def test_call_list_present(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + run_elem = scheme.find('run') + call_list = run_elem.find('call_list') + self.assertIsNotNone(call_list) + + def test_call_list_has_vars(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + run_elem = scheme.find('run') + call_list = run_elem.find('call_list') + vars_ = call_list.findall('var') + self.assertGreater(len(vars_), 0) + + def test_var_has_required_attributes(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + run_elem = scheme.find('run') + call_list = run_elem.find('call_list') + for v in call_list.findall('var'): + self.assertIsNotNone(v.get('name'), "var missing 'name'") + self.assertIsNotNone(v.get('intent'), "var missing 'intent'") + self.assertIsNotNone(v.get('local_name'), "var missing 'local_name'") + + def test_error_vars_present_in_call_list(self): + scheme = next( + s for s in self._schemes().findall('scheme') + if s.get('name') == 'temp_calc_adjust' + ) + run_elem = scheme.find('run') + call_list = run_elem.find('call_list') + std_names = [v.get('name') for v in call_list.findall('var')] + self.assertIn('ccpp_error_message', std_names) + self.assertIn('ccpp_error_code', std_names) + + +class TestApiSection(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + path, _, _ = _write(self._tmpdir) + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_api_element_present(self): + self.assertIsNotNone(self._root.find('api')) + + def test_suites_element_present(self): + api = self._root.find('api') + self.assertIsNotNone(api.find('suites')) + + def test_suite_name(self): + suites = self._root.find('api').find('suites') + names = [s.get('name') for s in suites.findall('suite')] + self.assertIn('test_simple', names) + + def test_group_name(self): + suites = self._root.find('api').find('suites') + suite = next(s for s in suites.findall('suite') if s.get('name') == 'test_simple') + groups = suite.findall('group') + self.assertGreater(len(groups), 0) + group_names = [g.get('name') for g in groups] + self.assertIn('physics', group_names) + + def test_scheme_listed_in_group(self): + suites = self._root.find('api').find('suites') + suite = next(s for s in suites.findall('suite') if s.get('name') == 'test_simple') + group = next(g for g in suite.findall('group') if g.get('name') == 'physics') + scheme_names = [s.text for s in group.findall('scheme')] + self.assertIn('temp_calc_adjust', scheme_names) + + def test_no_duplicate_schemes_in_group(self): + suites = self._root.find('api').find('suites') + suite = next(s for s in suites.findall('suite') if s.get('name') == 'test_simple') + group = next(g for g in suite.findall('group') if g.get('name') == 'physics') + names = [s.text for s in group.findall('scheme')] + self.assertEqual(len(names), len(set(names))) + + +class TestDependenciesSection(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + path, _, _ = _write(self._tmpdir) + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_dependencies_element_present(self): + self.assertIsNotNone(self._root.find('dependencies')) + + def test_dependencies_empty_by_default(self): + deps = self._root.find('dependencies') + self.assertEqual(len(list(deps)), 0) + + +class TestDependenciesPopulated(unittest.TestCase): + """write_datatable writes children when paths are given.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + sr, store = _resolve() + self._path = write_datatable( + [sr], + store, + [], + [], + self._tmpdir, + dependency_paths=['/path/to/a.F90', '/path/to/b.F90', '/path/to/a.F90'], + ) + self._root = ET.parse(self._path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_dependency_elements_present(self): + deps = self._root.find('dependencies') + texts = [d.text for d in deps.findall('dependency')] + self.assertIn('/path/to/a.F90', texts) + self.assertIn('/path/to/b.F90', texts) + + def test_dependencies_deduplicated(self): + deps = self._root.find('dependencies') + texts = [d.text for d in deps.findall('dependency')] + self.assertEqual(texts.count('/path/to/a.F90'), 1) + + def test_dependencies_sorted(self): + deps = self._root.find('dependencies') + texts = [d.text for d in deps.findall('dependency')] + self.assertEqual(texts, sorted(texts)) + + +class TestSubcycleDatatable(unittest.TestCase): + """Schemes inside subcycles appear once in the datatable.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + path, _, _ = _write(self._tmpdir, suite_xml='suite_test_subcycle.xml') + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_scheme_deduped_in_schemes(self): + names = [s.get('name') for s in self._root.find('schemes').findall('scheme')] + self.assertEqual(names.count('temp_calc_adjust'), 1) + + def test_scheme_deduped_in_api_group(self): + suites = self._root.find('api').find('suites') + suite = suites.findall('suite')[0] + group = suite.findall('group')[0] + names = [s.text for s in group.findall('scheme')] + self.assertEqual(names.count('temp_calc_adjust'), 1) + + +class TestDiagnosticNameEmission(unittest.TestCase): + """diagnostic_name / diagnostic_name_fixed must reach attributes.""" + + def _find_var(self, root, scheme, phase, std_name): + for s in root.find('schemes').findall('scheme'): + if s.get('name') != scheme: + continue + ph = s.find(phase) + if ph is None: + continue + for v in ph.find('call_list').findall('var'): + if v.get('name') == std_name: + return v + return None + + def test_explicit_diagnostic_name_emitted(self): + with tempfile.TemporaryDirectory() as d: + path, _, store = _write(d) + # Find a scheme variable, set an explicit diagnostic_name on its + # MetaVar, then re-write the datatable. + mvars = store.variables_for('temp_calc_adjust', 'run') + target = next(mv for mv in mvars + if mv.standard_name == 'air_temperature') + target._diagnostic_name = 'temperature' + sr, _ = _resolve() + path = write_datatable( + [sr], store, + ['/out/ccpp_kinds.F90'], + ['/out/ccpp_test_simple_cap.F90'], + d, + ) + root = ET.parse(path).getroot() + v = self._find_var(root, 'temp_calc_adjust', 'run', + 'air_temperature') + self.assertIsNotNone(v) + self.assertEqual(v.get('diagnostic_name'), 'temperature') + + def test_diagnostic_name_fixed_emitted(self): + with tempfile.TemporaryDirectory() as d: + sr, store = _resolve() + mvars = store.variables_for('temp_calc_adjust', 'run') + target = next(mv for mv in mvars + if mv.standard_name == 'air_temperature') + target.diagnostic_name_fixed = 'Q' + path = write_datatable( + [sr], store, + ['/out/ccpp_kinds.F90'], + ['/out/ccpp_test_simple_cap.F90'], + d, + ) + root = ET.parse(path).getroot() + v = self._find_var(root, 'temp_calc_adjust', 'run', + 'air_temperature') + self.assertEqual(v.get('diagnostic_name_fixed'), 'Q') + # diagnostic_name attr is suppressed (since _fixed wins). + self.assertIsNone(v.get('diagnostic_name')) + + def test_default_diagnostic_name_is_local_name(self): + """No explicit attrs → diagnostic_name attribute equals local_name.""" + with tempfile.TemporaryDirectory() as d: + path, _, _ = _write(d) + root = ET.parse(path).getroot() + v = self._find_var(root, 'temp_calc_adjust', 'run', + 'air_temperature') + self.assertIsNotNone(v) + self.assertEqual(v.get('diagnostic_name'), v.get('local_name')) + + +class TestHostFilesPopulated(unittest.TestCase): + """host_file_paths arg populates .""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._path, _, _ = _write( + self._tmpdir, + host_file_paths=['/out/ccpp_static_api.F90'], + ) + self._root = ET.parse(self._path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_host_files_contains_static_api(self): + host_files = self._root.find('capgen_files').find('host_files') + names = [f.text for f in host_files.findall('file')] + self.assertEqual(names, ['/out/ccpp_static_api.F90']) + + +class TestVarDictionariesSection(unittest.TestCase): + """write_datatable emits when host_dict is given.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._hd = _load_full_host_dict() + self._path, self._sr, _ = _write( + self._tmpdir, + host_dict=self._hd, + host_name='test_host', + ) + self._root = ET.parse(self._path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _vd(self): + return self._root.find('var_dictionaries') + + def test_section_emitted_when_host_dict_provided(self): + self.assertIsNotNone(self._vd()) + + def test_no_section_when_host_dict_absent(self): + with tempfile.TemporaryDirectory() as d: + path, _, _ = _write(d) # host_dict=None + root = ET.parse(path).getroot() + self.assertIsNone(root.find('var_dictionaries')) + + def test_host_dictionary_present(self): + host_d = next( + (vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'host'), + None, + ) + self.assertIsNotNone(host_d) + self.assertEqual(host_d.get('name'), 'test_host') + + def test_host_dictionary_has_vars(self): + host_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'host') + names = {v.get('name') for v in host_d.find('variables').findall('var')} + self.assertIn('air_temperature', names) + + def test_api_dict_parent_is_host_name(self): + api_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'api') + self.assertEqual(api_d.get('parent'), 'test_host') + + def test_suite_dict_parent_is_api(self): + suite_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'suite') + self.assertEqual(suite_d.get('parent'), 'ccpp_api') + self.assertEqual(suite_d.get('name'), 'test_simple') + + def test_group_dict_parent_is_suite(self): + group_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'group') + self.assertEqual(group_d.get('parent'), 'test_simple') + + def test_group_call_list_parent_is_group(self): + call_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'group_call_list') + gname = call_d.get('name').replace('_call_list', '') + self.assertEqual(call_d.get('parent'), gname) + + def test_group_call_list_has_vars(self): + call_d = next(vd for vd in self._vd().findall('var_dictionary') + if vd.get('type') == 'group_call_list') + vars_ = call_d.find('variables').findall('var') + self.assertGreater(len(vars_), 0) + for v in vars_: + self.assertIsNotNone(v.get('name')) + # intent absent for the rare control-only entry is OK + # but every var must have a name + + +class TestVarDictionariesProtectedAttr(unittest.TestCase): + """Protected host vars carry protected='True'; others omit the attr.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + hd = _load_full_host_dict() + # Flip one entry to protected for the test. + self._protected_std = next(iter(hd)) + hd[self._protected_std].protected = True + self._hd = hd + path, _, _ = _write(self._tmpdir, host_dict=hd) + self._root = ET.parse(path).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_protected_attr_emitted_when_true(self): + host_d = next(vd for vd in self._root.find('var_dictionaries') + .findall('var_dictionary') + if vd.get('type') == 'host') + v = next(v for v in host_d.find('variables').findall('var') + if v.get('name') == self._protected_std) + self.assertEqual(v.get('protected'), 'True') + + def test_protected_attr_absent_when_false(self): + host_d = next(vd for vd in self._root.find('var_dictionaries') + .findall('var_dictionary') + if vd.get('type') == 'host') + non_prot = next(v for v in host_d.find('variables').findall('var') + if v.get('name') != self._protected_std) + self.assertIsNone(non_prot.get('protected')) + + +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(dt_mod)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_host_constituents.py b/unit-tests/test_host_constituents.py new file mode 100644 index 00000000..c57b8566 --- /dev/null +++ b/unit-tests/test_host_constituents.py @@ -0,0 +1,548 @@ +"""Unit tests for ``generator.host_constituents``.""" + +import doctest +import os +import unittest + +from generator.suite_resolver import resolve_suite +from generator.host_constituents import ( + _any_constituent_state, + _all_index_names, + _suites_with_register_consts, + _generate_host_constituents, +) + + +def _resolve_consumer(): + """Resolve the consume_constituent fixture; return (sr, host_dict).""" + from test_suite_resolver import ( + _load_constituent_host_dict, + _load_constituent_consumer_store, + _parse_suite, + ) + hd = _load_constituent_host_dict() + store = _load_constituent_consumer_store() + suite = _parse_suite('suite_consume_constituent.xml') + return resolve_suite(suite, store, hd), hd + + +def _resolve_register(): + """Resolve the register_constituents fixture; return (sr, host_dict).""" + from test_suite_resolver import ( + _load_constituent_host_dict, + _load_constituent_scheme_store, + _parse_suite, + ) + hd = _load_constituent_host_dict() + store = _load_constituent_scheme_store() + suite = _parse_suite('suite_register_constituents.xml') + return resolve_suite(suite, store, hd), hd + + +def _resolve_simple(): + """Resolve the no-constituent fixture; return (sr, host_dict).""" + from test_suite_resolver import ( + _load_full_host_dict, + _load_scheme_store, + _parse_suite, + ) + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + return resolve_suite(suite, store, hd), hd + + +def _render_consumer(): + sr, hd = _resolve_consumer() + return '\n'.join(_generate_host_constituents([sr], host_dict=hd)) + + +def _render_register(): + sr, hd = _resolve_register() + return '\n'.join(_generate_host_constituents([sr], host_dict=hd)) + + +class TestAggregationHelpers(unittest.TestCase): + """``_any_constituent_state`` / ``_all_index_names`` / ``_suites_with_register_consts``.""" + + def test_any_when_consumer_only(self): + sr, _hd = _resolve_consumer() + self.assertTrue(_any_constituent_state([sr])) + + def test_any_when_register_only(self): + sr, _hd = _resolve_register() + self.assertTrue(_any_constituent_state([sr])) + + def test_any_when_neither(self): + sr, _hd = _resolve_simple() + self.assertFalse(_any_constituent_state([sr])) + + def test_index_names_aggregated(self): + consumer, _ch = _resolve_consumer() + register, _rh = _resolve_register() + names = _all_index_names([consumer, register]) + # Only the consumer side names a constituent in this fixture + # (register-phase scheme produces dyn_const3, but that std name + # is never read back via index_of_). + self.assertEqual(names, ['cloud_liquid_water_mixing_ratio']) + + def test_register_suites_listed(self): + consumer, _ch = _resolve_consumer() + register, _rh = _resolve_register() + self.assertEqual( + _suites_with_register_consts([consumer, register]), + ['reg_consts'], + ) + + +class TestModuleSkippedWhenNoConstituents(unittest.TestCase): + """``_generate_host_constituents`` returns ``None`` when nothing touches + constituent state — the module is not emitted at all.""" + + def test_returns_none(self): + sr, _hd = _resolve_simple() + self.assertIsNone(_generate_host_constituents([sr])) + + +class TestModuleHeaderAndUses(unittest.TestCase): + """Module declaration and external USEs are correct.""" + + def setUp(self): + self.text = _render_consumer() + + def test_module_name(self): + self.assertIn('module ccpp_host_constituents', self.text) + self.assertIn('end module ccpp_host_constituents', self.text) + + def test_use_kind(self): + self.assertIn('use ccpp_kinds, only: kind_phys', self.text) + + def test_use_constituent_prop_mod(self): + self.assertIn('use ccpp_constituent_prop_mod', self.text) + self.assertIn('ccpp_model_constituents_t', self.text) + self.assertIn('ccpp_constituent_properties_t', self.text) + self.assertIn('ccpp_constituent_prop_ptr_t', self.text) + + +class TestStateDeclarations(unittest.TestCase): + """Module-level state declarations: obj, pointers, integers.""" + + def setUp(self): + self.consumer_text = '\n'.join( + _render_consumer().splitlines() + ) + self.register_text = '\n'.join( + _render_register().splitlines() + ) + + def test_constituent_obj_declared(self): + # Per-instance allocatable array. + self.assertIn( + 'type(ccpp_model_constituents_t), target, allocatable :: ' + 'ccpp_model_constituents_obj(:)', + self.consumer_text, + ) + + def test_obj_is_public(self): + self.assertIn('public :: ccpp_model_constituents_obj', self.consumer_text) + + def test_no_module_level_pointers(self): + # Under per-instance design, ccpp_constituents / ..._tendencies / + # ..._properties / number_of_ccpp_constituents are NOT module-level + # variables — they're accessed as members of the obj(inst) array. + self.assertNotIn('pointer :: ccpp_constituents', self.consumer_text) + self.assertNotIn( + 'pointer :: ccpp_constituent_tendencies', self.consumer_text, + ) + self.assertNotIn( + 'pointer :: ccpp_constituent_properties', self.consumer_text, + ) + self.assertNotIn( + 'integer :: number_of_ccpp_constituents', self.consumer_text, + ) + + def test_index_of_X_declared(self): + self.assertIn( + 'integer :: index_of_cloud_liquid_water_mixing_ratio = 0', + self.consumer_text, + ) + self.assertIn( + 'public :: index_of_cloud_liquid_water_mixing_ratio', + self.consumer_text, + ) + + def test_per_suite_buffer_declared_for_producer(self): + self.assertIn( + 'type(ccpp_constituent_properties_t), allocatable, target :: ' + 'reg_consts_dynamic_constituents(:)', + self.register_text, + ) + self.assertIn( + 'public :: reg_consts_dynamic_constituents', + self.register_text, + ) + + def test_no_per_suite_buffer_when_no_producer(self): + # consumer fixture has no register-phase producer schemes — no + # ``_dynamic_constituents`` array declaration, no public + # of any such buffer. (The ccpp_deallocate_dynamic_constituents + # subroutine name is unrelated and is expected to appear.) + self.assertNotIn('allocatable, target :: ', self.consumer_text) + self.assertNotIn('_dynamic_constituents(:)', self.consumer_text) + + +class TestRegisterConstituentsRoutine(unittest.TestCase): + """``ccpp_register_constituents`` merges host + per-suite buffers.""" + + def setUp(self): + # Use a SuiteResolution list containing BOTH consumer and register + # fixtures so the routine iterates a real per-suite buffer. + self.text = _render_register() + + def test_takes_host_constituents_and_instance(self): + # instance_number is in the signature when the host declares it. + self.assertIn( + 'subroutine ccpp_register_constituents(host_constituents, ' + 'inst_num, errflg, errmsg)', + self.text, + ) + self.assertIn( + 'type(ccpp_constituent_properties_t), target, intent(in) :: ' + 'host_constituents(:)', + self.text, + ) + self.assertIn('integer, intent(in) :: inst_num', self.text) + + def test_allocates_obj_array_on_first_call(self): + # Idempotent allocation: only the first instance to call sees + # an unallocated array; subsequent instances skip. + self.assertIn( + 'if (.not. allocated(ccpp_model_constituents_obj)) then', + self.text, + ) + self.assertIn( + 'allocate(ccpp_model_constituents_obj(ninstances))', self.text, + ) + + def test_initializes_table_per_instance(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%initialize_table(num_consts)', + self.text, + ) + self.assertIn( + 'num_consts = num_consts + size(reg_consts_dynamic_constituents, 1)', + self.text, + ) + + def test_iterates_host_then_suite_per_instance(self): + body = self.text.split('subroutine ccpp_register_constituents')[1].split( + 'end subroutine ccpp_register_constituents' + )[0] + host_pos = body.find('host_constituents(index)') + suite_pos = body.find('reg_consts_dynamic_constituents(index)') + self.assertGreater(host_pos, 0) + self.assertGreater(suite_pos, host_pos) + # All %new_field calls go through obj(inst_num). + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%new_field(const_prop', + body, + ) + + def test_lock_table_called_per_instance(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%lock_table(' + 'errcode=errflg, errmsg=errmsg)', + self.text, + ) + + +class TestInitializeConstituentsRoutine(unittest.TestCase): + """``ccpp_initialize_constituents`` locks data + binds pointers + populates indices.""" + + def setUp(self): + self.text = _render_consumer() + + def test_takes_dimensions_and_instance(self): + self.assertIn( + 'subroutine ccpp_initialize_constituents(ncols, num_layers, ' + 'inst_num, errflg, errmsg)', + self.text, + ) + + def test_calls_lock_data_per_instance(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%lock_data(' + 'ncols, num_layers, errcode=errflg, errmsg=errmsg)', + self.text, + ) + + def test_uses_scheme_utils(self): + body = self.text.split('subroutine ccpp_initialize_constituents')[1].split( + 'end subroutine ccpp_initialize_constituents' + )[0] + self.assertIn( + 'use ccpp_scheme_utils, only: ccpp_initialize_constituent_ptr', + body, + ) + + def test_initialises_scheme_utils_pointer_per_instance(self): + body = self.text.split('subroutine ccpp_initialize_constituents')[1].split( + 'end subroutine ccpp_initialize_constituents' + )[0] + # Need a local pointer variable to pass a non-pointer target as a + # pointer dummy. + self.assertIn( + 'type(ccpp_model_constituents_t), pointer :: const_obj_ptr', + body, + ) + self.assertIn( + 'const_obj_ptr => ccpp_model_constituents_obj(inst_num)', body, + ) + self.assertIn( + 'call ccpp_initialize_constituent_ptr(const_obj_ptr)', body, + ) + + def test_no_module_pointer_binding(self): + # The old module-level ccpp_constituents pointers don't exist; + # this routine must NOT try to bind them. + body = self.text.split('subroutine ccpp_initialize_constituents')[1].split( + 'end subroutine ccpp_initialize_constituents' + )[0] + self.assertNotIn('ccpp_constituents =>', body) + self.assertNotIn('ccpp_constituent_tendencies =>', body) + self.assertNotIn('ccpp_constituent_properties =>', body) + self.assertNotIn( + 'number_of_ccpp_constituents =', body, + ) + + def test_queries_index_of_X_per_instance(self): + self.assertIn( + "call ccpp_model_constituents_obj(inst_num)%const_index(" + "index_of_cloud_liquid_water_mixing_ratio, " + "'cloud_liquid_water_mixing_ratio', errcode=errflg, errmsg=errmsg)", + self.text, + ) + + def test_int_unassigned_imported(self): + # Used by the post-const_index validation block; must be in the + # module-level USE so the contained subroutine can reference it. + self.assertIn( + 'use ccpp_constituent_prop_mod, only:', self.text, + ) + self.assertIn('int_unassigned', self.text) + + def test_post_const_index_validation_emitted(self): + # %const_index doesn't error on a miss — it returns int_unassigned + # and errcode=0. The generator must check the integer afterward + # and fail with a descriptive message so the host sees the bad + # registration at init time instead of crashing on a -huge(1) + # subscript later in run-phase scheme calls. + body = self.text.split('subroutine ccpp_initialize_constituents')[1].split( + 'end subroutine ccpp_initialize_constituents' + )[0] + self.assertIn( + 'if (index_of_cloud_liquid_water_mixing_ratio == int_unassigned) then', + body, + ) + self.assertIn('errflg = 1', body) + self.assertIn( + "errmsg = 'ccpp_initialize_constituents: constituent " + "''cloud_liquid_water_mixing_ratio'' is referenced by a " + "scheme but is not in the registered constituent table", + body, + ) + + +class TestIsSchemeConstituent(unittest.TestCase): + """``ccpp_is_scheme_constituent`` + the module-scope std-name parameter array.""" + + def setUp(self): + self.text = _render_consumer() + + def test_subroutine_signature(self): + self.assertIn( + 'subroutine ccpp_is_scheme_constituent(var_name, ' + 'constituent_exists, errflg, errmsg)', + self.text, + ) + + def test_uses_known_array(self): + self.assertIn( + 'constituent_exists = any(ccpp_model_const_stdnames == var_name)', + self.text, + ) + + def test_param_array_declared(self): + self.assertIn( + "character(len=31), parameter :: ccpp_model_const_stdnames(1) = (/ &", + self.text, + ) + self.assertIn("'cloud_liquid_water_mixing_ratio'", self.text) + + def test_param_array_public(self): + self.assertIn('public :: ccpp_model_const_stdnames', self.text) + + +class TestIsSchemeConstituentNoIndices(unittest.TestCase): + """When the suite registers constituents but none are referenced via + ``index_of_``, ``ccpp_is_scheme_constituent`` returns ``.false.`` + unconditionally — no parameter array is emitted.""" + + def setUp(self): + self.text = _render_register() + + def test_falls_back_to_constant_false(self): + self.assertIn('constituent_exists = .false.', self.text) + + def test_no_param_array(self): + self.assertNotIn('ccpp_model_const_stdnames', self.text) + + +class TestWrapperSubroutines(unittest.TestCase): + """Thin wrappers for number / gather / update / const_index.""" + + def setUp(self): + self.text = _render_consumer() + + def test_number_constituents(self): + self.assertIn( + 'subroutine ccpp_number_constituents(num_flds, advected, ' + 'inst_num, errflg, errmsg)', + self.text, + ) + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%num_constituents(' + 'num_flds, advected=advected, errcode=errflg, errmsg=errmsg)', + self.text, + ) + + def test_gather_constituents(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%copy_in(' + 'const_array, errcode=errflg, errmsg=errmsg)', + self.text, + ) + + def test_update_constituents(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%copy_out(' + 'const_array, errcode=errflg, errmsg=errmsg)', + self.text, + ) + + def test_const_get_index(self): + # Keyword args ensure unambiguous mapping to the DDT's signature + # (index, standard_name, errcode, errmsg). + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%const_index(' + 'standard_name=stdname, index=const_index, ' + 'errcode=errflg, errmsg=errmsg)', + self.text, + ) + + +class TestAccessorFunctions(unittest.TestCase): + """Pointer-returning accessor functions.""" + + def setUp(self): + self.text = _render_consumer() + + def test_constituents_array(self): + self.assertIn( + 'function ccpp_constituents_array(inst_num) result(const_ptr)', + self.text, + ) + self.assertIn( + 'const_ptr => ccpp_model_constituents_obj(inst_num)%field_data_ptr()', + self.text, + ) + + def test_advected_constituents_array(self): + self.assertIn( + 'function ccpp_advected_constituents_array(inst_num) result(const_ptr)', + self.text, + ) + self.assertIn( + 'const_ptr => ccpp_model_constituents_obj(inst_num)' + '%advected_constituents_ptr()', + self.text, + ) + + def test_model_const_properties(self): + self.assertIn( + 'function ccpp_model_const_properties(inst_num) result(const_ptr)', + self.text, + ) + self.assertIn( + 'const_ptr => ccpp_model_constituents_obj(inst_num)' + '%constituent_props_ptr()', + self.text, + ) + + +class TestDeallocateRoutine(unittest.TestCase): + """``ccpp_deallocate_dynamic_constituents`` is per-instance with + last-to-leave teardown of shared buffers + the obj array.""" + + def setUp(self): + self.text = _render_register() + self.body = self.text.split( + 'subroutine ccpp_deallocate_dynamic_constituents' + )[1].split('end subroutine ccpp_deallocate_dynamic_constituents')[0] + + def test_takes_instance_number(self): + self.assertIn( + 'subroutine ccpp_deallocate_dynamic_constituents(inst_num)', + self.text, + ) + self.assertIn('integer, intent(in) :: inst_num', self.body) + + def test_per_instance_reset(self): + self.assertIn( + 'call ccpp_model_constituents_obj(inst_num)%reset()', self.body, + ) + + def test_short_circuit_when_unallocated(self): + # If no instance has registered yet, the call is a no-op. + self.assertIn( + 'if (.not. allocated(ccpp_model_constituents_obj)) return', + self.body, + ) + + def test_last_to_leave_check(self): + self.assertIn('all_done = .true.', self.body) + self.assertIn('do i = 1, size(ccpp_model_constituents_obj, 1)', self.body) + self.assertIn( + 'if (ccpp_model_constituents_obj(i)%const_props_locked()) then', + self.body, + ) + self.assertIn('all_done = .false.', self.body) + + def test_last_to_leave_teardown(self): + self.assertIn('if (all_done) then', self.body) + self.assertIn('deallocate(ccpp_model_constituents_obj)', self.body) + # The per-suite buffer is NOT torn down here — that's owned by + # the suite-cap lifecycle (deallocated in _final's + # last-to-leave block). Tearing it down here would break the + # next ccpp_register call (suite_state guard skips re-fill). + self.assertNotIn( + 'deallocate(reg_consts_dynamic_constituents)', self.body, + ) + + def test_no_module_pointer_nullify(self): + # The module-level pointer set was removed; deallocate routine + # must not try to nullify them. + self.assertNotIn('nullify(ccpp_constituents)', self.body) + self.assertNotIn('nullify(ccpp_constituent_tendencies)', self.body) + self.assertNotIn('nullify(ccpp_constituent_properties)', self.body) + self.assertNotIn('number_of_ccpp_constituents = 0', self.body) + + +def load_tests(loader, tests, ignore): + import generator.host_constituents as hc + tests.addTests(doctest.DocTestSuite(hc)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_integration.py b/unit-tests/test_integration.py new file mode 100644 index 00000000..c7817478 --- /dev/null +++ b/unit-tests/test_integration.py @@ -0,0 +1,2065 @@ +"""End-to-end integration tests for capgen_ng.capgen(). + +These tests invoke the full pipeline — metadata loading, variable resolution, +and file generation — and verify that all expected output files are produced +with correct content. No Fortran compiler is involved; correctness is checked +at the text/XML level. +""" + +import os +import tempfile +import unittest +import xml.etree.ElementTree as ET + +from ccpp_capgen_ng import capgen + +_TESTS_DIR = os.path.dirname(__file__) +_SAMPLES_DIR = os.path.join(_TESTS_DIR, 'sample_files') +_SUITE_DIR = os.path.join(_TESTS_DIR, 'sample_suite_files') + + +def _sf(name): + return os.path.join(_SAMPLES_DIR, name) + + +def _suite_file(name): + return os.path.join(_SUITE_DIR, name) + + +# --------------------------------------------------------------------------- +# Helper: run capgen and return output directory + file map +# --------------------------------------------------------------------------- + +def _run_simple(tmpdir, suite_xml='suite_test_simple.xml', kind_types=None): + """Run capgen with the simple test suite and return the output dir.""" + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file(suite_xml)], + output_root=tmpdir, + kind_types=kind_types or {}, + ) + return tmpdir + + +def _run_subcycle(tmpdir): + return _run_simple(tmpdir, suite_xml='suite_test_subcycle.xml') + + +# --------------------------------------------------------------------------- +# Test: output files exist +# --------------------------------------------------------------------------- + +class TestOutputFilesExist(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _path(self, name): + return os.path.join(self._tmpdir, name) + + def test_static_api_exists(self): + self.assertTrue(os.path.isfile(self._path('ccpp_static_api.F90'))) + + def test_suite_cap_exists(self): + self.assertTrue(os.path.isfile(self._path('ccpp_test_simple_cap.F90'))) + + def test_group_cap_exists(self): + self.assertTrue(os.path.isfile(self._path('ccpp_test_simple_physics_cap.F90'))) + + def test_suite_data_exists(self): + self.assertTrue(os.path.isfile(self._path('ccpp_test_simple_data.F90'))) + + def test_datatable_exists(self): + self.assertTrue(os.path.isfile(self._path('datatable.xml'))) + + def test_ccpp_kinds_always_generated(self): + # ccpp_kinds.F90 is always written, even when no --kind-type is given. + self.assertTrue(os.path.isfile(self._path('ccpp_kinds.F90'))) + + def test_ccpp_kinds_default_kind_phys(self): + """With no --kind-type, kind_phys defaults to REAL64 from iso_fortran_env.""" + with open(self._path('ccpp_kinds.F90')) as fh: + text = fh.read() + self.assertIn('use iso_fortran_env, only: REAL64', text) + self.assertIn('kind_phys = REAL64', text) + + def test_ccpp_kinds_with_explicit_kind_types(self): + with tempfile.TemporaryDirectory() as d: + _run_simple(d, kind_types={ + 'kind_phys': ('iso_fortran_env', 'REAL64'), + }) + self.assertTrue(os.path.isfile(os.path.join(d, 'ccpp_kinds.F90'))) + + def test_ccpp_kinds_default_injected_when_other_kinds_given(self): + """kind_phys default is injected even when other --kind-type args are present.""" + with tempfile.TemporaryDirectory() as d: + _run_simple(d, kind_types={ + 'kind_dyn': ('iso_fortran_env', 'REAL32'), + }) + with open(os.path.join(d, 'ccpp_kinds.F90')) as fh: + text = fh.read() + self.assertIn('kind_dyn', text) + self.assertIn('kind_phys', text) + + +# --------------------------------------------------------------------------- +# Test: kind_spec declared in metadata is folded into ccpp_kinds.F90 +# --------------------------------------------------------------------------- + +def _inject_kind_spec_lines(src_meta, dest_meta, lines): + """Copy *src_meta* to *dest_meta*, inserting *lines* into the first + ``[ccpp-table-properties]`` block immediately after its header.""" + with open(src_meta) as fh: + text = fh.read() + marker = '[ccpp-table-properties]' + idx = text.find(marker) + if idx < 0: + raise RuntimeError("no ccpp-table-properties marker in " + src_meta) + nl = text.find('\n', idx) + insertion = ''.join(' ' + line + '\n' for line in lines) + new_text = text[:nl + 1] + insertion + text[nl + 1:] + with open(dest_meta, 'w') as fh: + fh.write(new_text) + + +class TestMetadataKindSpec(unittest.TestCase): + """Integration tests for kind_spec declared in [ccpp-table-properties].""" + + def _scheme_with_kind_spec(self, dest_dir, lines): + """Write a copy of scheme_multipart.meta with extra ``kind_spec`` lines.""" + dest = os.path.join(dest_dir, 'scheme_multipart.meta') + _inject_kind_spec_lines(_sf('scheme_multipart.meta'), dest, lines) + return dest + + def test_metadata_kind_spec_added_to_ccpp_kinds(self): + """A ``kind_spec`` declared in scheme metadata appears in ccpp_kinds.F90.""" + with tempfile.TemporaryDirectory() as d: + scheme_meta = self._scheme_with_kind_spec( + d, ['kind_spec = temp_kinds:kind_temp=>temp_r8'], + ) + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[scheme_meta], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=d, + kind_types={}, + ) + with open(os.path.join(d, 'ccpp_kinds.F90')) as fh: + text = fh.read() + # Use lines may be column-aligned when multiple modules are + # present; match each token rather than the full line. + self.assertRegex(text, r'use\s+temp_kinds\b[^\n]*only:\s*temp_r8') + self.assertRegex(text, r'kind_temp\s*=\s*temp_r8') + # kind_phys default still injected. + self.assertIn('kind_phys', text) + self.assertIn('use iso_fortran_env', text) + + def test_metadata_kind_spec_shorthand_added_to_ccpp_kinds(self): + """Shorthand ``module:spec`` republishes spec under its own name.""" + with tempfile.TemporaryDirectory() as d: + scheme_meta = self._scheme_with_kind_spec( + d, ['kind_spec = host_kinds:kind_r4'], + ) + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[scheme_meta], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=d, + kind_types={}, + ) + with open(os.path.join(d, 'ccpp_kinds.F90')) as fh: + text = fh.read() + self.assertRegex(text, r'use\s+host_kinds\b[^\n]*only:\s*kind_r4') + self.assertRegex(text, r'kind_r4\s*=\s*kind_r4') + + def test_cli_and_metadata_identical_kind_spec_no_conflict(self): + """CLI --kind-type and metadata kind_spec for the same kind are accepted when identical.""" + with tempfile.TemporaryDirectory() as d: + scheme_meta = self._scheme_with_kind_spec( + d, ['kind_spec = temp_kinds:kind_temp=>temp_r8'], + ) + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[scheme_meta], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=d, + kind_types={'kind_temp': ('temp_kinds', 'temp_r8')}, + ) + with open(os.path.join(d, 'ccpp_kinds.F90')) as fh: + text = fh.read() + # The kind appears exactly once in ccpp_kinds.F90. + self.assertEqual(text.count('kind_temp ='), 1) + + def test_cli_and_metadata_conflicting_kind_spec_raises(self): + """CLI and metadata declaring the same kind with different specs is a hard error.""" + with tempfile.TemporaryDirectory() as d: + scheme_meta = self._scheme_with_kind_spec( + d, ['kind_spec = temp_kinds:kind_temp=>temp_r8'], + ) + from metadata.parse_tools import CCPPError + with self.assertRaises(CCPPError) as cm: + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[scheme_meta], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=d, + kind_types={'kind_temp': ('other_kinds', 'r8')}, + ) + self.assertIn('kind_temp', str(cm.exception)) + + +# --------------------------------------------------------------------------- +# Test: host_constituents module + framework file inclusion +# --------------------------------------------------------------------------- + +def _run_with_constituents(tmpdir, suite_xml='suite_consume_constituent.xml'): + """Run capgen with a constituent-using fixture and return tmpdir.""" + capgen( + host_name='host_consts', + host_files=[_sf('host_with_constituents.meta'), + _sf('control_full.meta')], + scheme_files=[_sf('scheme_consume_constituent.meta')], + suite_files=[_suite_file(suite_xml)], + output_root=tmpdir, + kind_types={}, + ) + return tmpdir + + +class TestHostConstituentsEmittedEndToEnd(unittest.TestCase): + """Full pipeline emits ccpp_host_constituents.F90 and lists every + framework F90 dependency in datatable.xml's .""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_with_constituents(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_host_constituents_file_exists(self): + self.assertTrue(os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_host_constituents.F90') + )) + + def _utility_paths(self): + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + return [e.text for e in tree.getroot().findall( + './capgen_files/utilities/file' + )] + + def test_datatable_lists_host_constituents(self): + utils = self._utility_paths() + names = [os.path.basename(p) for p in utils] + self.assertIn('ccpp_host_constituents.F90', names) + + def test_datatable_lists_framework_constituent_module(self): + utils = self._utility_paths() + names = [os.path.basename(p) for p in utils] + self.assertIn('ccpp_constituent_prop_mod.F90', names) + + def test_datatable_lists_framework_dependencies(self): + utils = self._utility_paths() + names = [os.path.basename(p) for p in utils] + # Transitive deps of ccpp_constituent_prop_mod. + self.assertIn('ccpp_hashable.F90', names) + self.assertIn('ccpp_hash_table.F90', names) + # Used by cam-sima schemes (ccpp_constituent_index). + self.assertIn('ccpp_scheme_utils.F90', names) + + def test_framework_paths_absolute_and_existing(self): + utils = self._utility_paths() + framework_names = { + 'ccpp_constituent_prop_mod.F90', + 'ccpp_hashable.F90', + 'ccpp_hash_table.F90', + 'ccpp_scheme_utils.F90', + } + for path in utils: + if os.path.basename(path) in framework_names: + self.assertTrue(os.path.isabs(path), + 'framework path not absolute: ' + path) + self.assertTrue(os.path.isfile(path), + 'framework file missing: ' + path) + + def test_framework_paths_resolve_under_capgen_ng_src(self): + """Every framework F90 listed must resolve under capgen-ng/src/. + Capgen-ng ships self-contained — no parent-dir fallback. If any + framework file lands outside capgen-ng/src/ this test fails so + downstream consumers (vendoring just capgen-ng/) don't silently + miss a required dependency.""" + from ccpp_capgen_ng import _FRAMEWORK_SRC_DIR + framework_names = { + 'ccpp_constituent_prop_mod.F90', + 'ccpp_hashable.F90', + 'ccpp_hash_table.F90', + 'ccpp_scheme_utils.F90', + } + utils = self._utility_paths() + canonical = os.path.abspath(_FRAMEWORK_SRC_DIR) + for path in utils: + if os.path.basename(path) in framework_names: + self.assertEqual( + os.path.abspath(os.path.dirname(path)), canonical, + 'framework F90 outside capgen-ng/src/: ' + path, + ) + + +class TestResolveFrameworkF90FilesMissingRaises(unittest.TestCase): + """``_resolve_framework_f90_files`` raises CCPPError listing the + missing file(s) when a required framework F90 is not present under + capgen-ng/src/. Catches deployment errors immediately instead of + leaving the host build to fail with an opaque "Cannot open module + file" message at compile time.""" + + def test_missing_file_raises_with_actionable_message(self): + import ccpp_capgen_ng + from metadata.parse_tools import CCPPError + # Append a never-vendored sentinel to the framework-F90 list, + # then restore on teardown via a try/finally so we don't leak + # state into other tests. + original = list(ccpp_capgen_ng._FRAMEWORK_F90_FILES) + ccpp_capgen_ng._FRAMEWORK_F90_FILES.append('definitely_missing.F90') + try: + with self.assertRaises(CCPPError) as cm: + ccpp_capgen_ng._resolve_framework_f90_files() + finally: + ccpp_capgen_ng._FRAMEWORK_F90_FILES[:] = original + msg = str(cm.exception) + # Names the missing file, the search dir, and what to do. + self.assertIn('definitely_missing.F90', msg) + self.assertIn(ccpp_capgen_ng._FRAMEWORK_SRC_DIR, msg) + self.assertIn('Vendor', msg) + + +class TestNoHostConstituentsWhenAbsent(unittest.TestCase): + """The host-constituents module is NOT emitted (and the framework F90 + files are NOT listed) when no suite touches constituent state.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_host_constituents_file_absent(self): + self.assertFalse(os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_host_constituents.F90') + )) + + def test_no_framework_dependencies_in_utilities(self): + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + utils = [e.text for e in tree.getroot().findall( + './capgen_files/utilities/file' + )] + names = [os.path.basename(p) for p in utils] + # Only ccpp_kinds.F90 — no constituent framework files. + self.assertEqual(names, ['ccpp_kinds.F90']) + + +# --------------------------------------------------------------------------- +# Test: static API content +# --------------------------------------------------------------------------- + +class TestStaticApiContent(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_module_declaration(self): + self.assertIn('module ccpp_static_api', self.text) + + def test_ccpp_register_always_present(self): + # ccpp_register is mandatory in the new design and always emitted, + # even when no scheme has a register phase (state transition only). + self.assertIn('subroutine ccpp_register', self.text) + + def test_ccpp_init_present(self): + self.assertIn('subroutine ccpp_init', self.text) + + def test_ccpp_final_present(self): + self.assertIn('subroutine ccpp_final', self.text) + + def test_ccpp_physics_run_present(self): + self.assertIn('subroutine ccpp_physics_run', self.text) + + def test_dispatches_to_test_simple(self): + self.assertIn("case('test_simple')", self.text) + + def test_uses_suite_cap_module(self): + self.assertIn('use ccpp_test_simple_cap', self.text) + + +# --------------------------------------------------------------------------- +# Test: suite cap content +# --------------------------------------------------------------------------- + +class TestSuiteCapContent(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_module_declaration(self): + self.assertIn('module ccpp_test_simple_cap', self.text) + + def test_register_subroutine_always_present(self): + # _register is mandatory in the new design — emitted even when + # no scheme has a register phase (state-transition skeleton only). + self.assertIn('subroutine test_simple_register', self.text) + + def test_init_subroutine(self): + self.assertIn('subroutine test_simple_init', self.text) + + def test_final_subroutine(self): + self.assertIn('subroutine test_simple_final', self.text) + + def test_physics_run_subroutine(self): + self.assertIn('subroutine test_simple_physics_run', self.text) + + def test_dispatches_to_group(self): + self.assertIn("case('physics')", self.text) + + def test_state_alloc_called(self): + self.assertIn('state_alloc', self.text) + + def test_state_dealloc_called(self): + self.assertIn('state_dealloc', self.text) + + +# --------------------------------------------------------------------------- +# Test: group cap content +# --------------------------------------------------------------------------- + +class TestGroupCapContent(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_module_declaration(self): + self.assertIn('module ccpp_test_simple_physics_cap', self.text) + + def test_state_machine_params(self): + self.assertIn('CCPP_GROUP_UNINITIALIZED', self.text) + self.assertIn('CCPP_GROUP_INITIALIZED', self.text) + + def test_scheme_call_present(self): + self.assertIn('call temp_calc_adjust_run', self.text) + + def test_init_guard(self): + self.assertIn('CCPP_GROUP_INITIALIZED', self.text) + self.assertIn('ccpp_group_state', self.text) + + def test_state_alloc_subroutine(self): + self.assertIn('subroutine ccpp_test_simple_physics_state_alloc', self.text) + + def test_state_dealloc_subroutine(self): + self.assertIn('subroutine ccpp_test_simple_physics_state_dealloc', self.text) + + def test_ends_with_newline(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + raw = fh.read() + self.assertTrue(raw.endswith('\n')) + + +# --------------------------------------------------------------------------- +# Test: datatable.xml content +# --------------------------------------------------------------------------- + +class TestDatatableContent(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + self._root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_root_version(self): + self.assertEqual(self._root.get('version'), '1.0') + + def test_suite_file_in_capgen_files(self): + suite_files = self._root.find('capgen_files').find('suite_files') + names = [os.path.basename(f.text) for f in suite_files.findall('file')] + self.assertIn('ccpp_test_simple_cap.F90', names) + self.assertIn('ccpp_test_simple_physics_cap.F90', names) + + def test_static_api_in_host_files(self): + host_files = self._root.find('capgen_files').find('host_files') + self.assertIsNotNone(host_files) + names = [os.path.basename(f.text) for f in host_files.findall('file')] + self.assertIn('ccpp_static_api.F90', names) + + def test_ccpp_kinds_in_utilities(self): + # ccpp_kinds.F90 is always generated and must be discoverable by + # CMake via (matches the original ccpp_capgen behavior). + utils = self._root.find('capgen_files').find('utilities') + names = [os.path.basename(f.text) for f in utils.findall('file')] + self.assertIn('ccpp_kinds.F90', names) + + def test_ccpp_kinds_utilities_path_is_absolute_and_exists(self): + """The path stored in must be absolute and resolve to a real file.""" + utils = self._root.find('capgen_files').find('utilities') + kinds_paths = [ + f.text for f in utils.findall('file') + if os.path.basename(f.text) == 'ccpp_kinds.F90' + ] + self.assertEqual(len(kinds_paths), 1) + self.assertTrue(os.path.isabs(kinds_paths[0])) + self.assertTrue(os.path.isfile(kinds_paths[0])) + + def test_suite_meta_in_inspection_files(self): + inspection = self._root.find('inspection_files') + self.assertIsNotNone(inspection) + meta_files = inspection.find('suite_meta_files') + self.assertIsNotNone(meta_files) + names = [os.path.basename(f.text) for f in meta_files.findall('file')] + self.assertIn('ccpp_test_simple.meta', names) + + def test_suite_meta_not_in_capgen_files(self): + capgen_files = self._root.find('capgen_files') + self.assertIsNone(capgen_files.find('suite_meta_files')) + + def test_expanded_sdf_in_inspection_files(self): + inspection = self._root.find('inspection_files') + self.assertIsNotNone(inspection) + exp_files = inspection.find('expanded_sdf_files') + self.assertIsNotNone(exp_files) + names = [os.path.basename(f.text) for f in exp_files.findall('file')] + self.assertIn('ccpp_test_simple_expanded.xml', names) + # The path stored must resolve to a real file on disk. + for f in exp_files.findall('file'): + self.assertTrue(os.path.isabs(f.text)) + self.assertTrue(os.path.isfile(f.text)) + + def test_scheme_in_schemes(self): + names = [s.get('name') for s in self._root.find('schemes').findall('scheme')] + self.assertIn('temp_calc_adjust', names) + + def test_suite_in_api(self): + suites = self._root.find('api').find('suites') + names = [s.get('name') for s in suites.findall('suite')] + self.assertIn('test_simple', names) + + def test_group_in_api_suite(self): + suites = self._root.find('api').find('suites') + suite = next(s for s in suites.findall('suite') if s.get('name') == 'test_simple') + group_names = [g.get('name') for g in suite.findall('group')] + self.assertIn('physics', group_names) + + +# --------------------------------------------------------------------------- +# Test: subcycle suite +# --------------------------------------------------------------------------- + +class TestSubcycleIntegration(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_subcycle(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_exists(self): + self.assertTrue( + os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_test_subcycle_physics_cap.F90') + ) + ) + + def test_do_loop_in_group_cap(self): + with open( + os.path.join(self._tmpdir, 'ccpp_test_subcycle_physics_cap.F90') + ) as fh: + text = fh.read() + # ``suite_test_subcycle.xml`` declares ```` — + # the integer literal must flow through verbatim into the + # generated do-loop bound (no host-dict lookup, no symbolic + # translation). Compare against the literal value, not just the + # presence of any ``do ccpp_loop_counter = 1,``. + self.assertIn('do ccpp_loop_counter = 1, 3', text) + self.assertIn('end do', text) + + def test_datatable_for_subcycle_suite(self): + root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + suites = root.find('api').find('suites') + names = [s.get('name') for s in suites.findall('suite')] + self.assertIn('test_subcycle', names) + + +# --------------------------------------------------------------------------- +# Test: multiple suites in one capgen run +# --------------------------------------------------------------------------- + +class TestMultipleSuites(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[ + _suite_file('suite_test_simple.xml'), + _suite_file('suite_test_subcycle.xml'), + ], + output_root=self._tmpdir, + kind_types={}, + ) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_both_suite_caps_exist(self): + self.assertTrue( + os.path.isfile(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) + ) + self.assertTrue( + os.path.isfile(os.path.join(self._tmpdir, 'ccpp_test_subcycle_cap.F90')) + ) + + def test_static_api_dispatches_both(self): + with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + text = fh.read() + self.assertIn("case('test_simple')", text) + self.assertIn("case('test_subcycle')", text) + + def test_datatable_has_both_suites(self): + root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + suites = root.find('api').find('suites') + names = [s.get('name') for s in suites.findall('suite')] + self.assertIn('test_simple', names) + self.assertIn('test_subcycle', names) + + def test_scheme_deduplicated_in_datatable(self): + root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + scheme_names = [ + s.get('name') for s in root.find('schemes').findall('scheme') + ] + self.assertEqual(scheme_names.count('temp_calc_adjust'), 1) + + +# --------------------------------------------------------------------------- +# Test: multi-instance — number_of_instances flows end-to-end +# --------------------------------------------------------------------------- + +class TestMultiInstanceIntegration(unittest.TestCase): + """host_full.meta provides ninstances → generated code is multi-instance aware.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_ccpp_init_minimal_signature(self): + # New minimal lifecycle signature: drops ninstances entirely. + with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + text = fh.read() + self.assertIn( + 'subroutine ccpp_init(suite_name, errflg, errmsg, inst_num)', + text, + ) + + def test_suite_init_minimal_signature(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'subroutine test_simple_init(inst_num, errmsg, errflg)', + text, + ) + + def test_register_passes_ninstances_to_suite_state_alloc(self): + # _register USEs ninstances from the host module and passes it + # to the suite_state allocator (idempotent first-call alloc). + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'call test_simple_suite_state_alloc(ninstances, errmsg, errflg)', + text, + ) + + def test_suite_init_passes_ninstances_to_group_state_alloc(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'call ccpp_test_simple_physics_state_alloc(ninstances, errmsg, errflg)', + text, + ) + + def test_state_alloc_takes_number_of_instances_arg(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'subroutine ccpp_test_simple_physics_state_alloc(number_of_instances, errmsg, errflg)', + text, + ) + + def test_state_alloc_uses_number_of_instances(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + self.assertIn('allocate(ccpp_group_state(number_of_instances))', text) + + def test_group_state_alloc_is_idempotent(self): + """The generated group_state_alloc must short-circuit when the + array is already allocated — otherwise the second instance's + _init call crashes on a double-allocate. Mirrors the + ``_suite_state_alloc`` idempotency contract. + """ + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + alloc_body = text.split('state_alloc(number_of_instances')[1] + alloc_body = alloc_body.split('end subroutine')[0] + self.assertIn('if (allocated(ccpp_group_state)) return', alloc_body) + # And the guard must precede the allocate, not follow it. + guard_pos = alloc_body.find('if (allocated(ccpp_group_state))') + alloc_pos = alloc_body.find('allocate(ccpp_group_state(') + self.assertGreater(alloc_pos, guard_pos, + "Idempotency guard must precede the allocate statement") + + def test_state_guard_uses_inst_num(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + self.assertIn('ccpp_group_state(inst_num)', text) + + def test_group_init_has_inst_num_arg(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + init_sub = text.split('subroutine ccpp_test_simple_physics_init')[1] + init_sub = init_sub.split('end subroutine')[0] + self.assertIn('inst_num', init_sub) + + def test_suite_cap_dispatches_inst_num_to_group_init(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + physics_init = text.split('subroutine test_simple_physics_init')[1] + physics_init = physics_init.split('end subroutine')[0] + self.assertIn('inst_num', physics_init) + + +class TestSingleInstanceIntegration(unittest.TestCase): + """host_no_instance.meta + control_no_instance.meta omit the instance + pair → generated static API drops instance_number from public + signatures, suite cap uses literal '1' for state-array indexing, + state arrays are allocated with the literal '1' as bound. + """ + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[ + _sf('host_no_instance.meta'), + _sf('control_no_instance.meta'), + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_static_api_ccpp_init_omits_inst_num(self): + with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + text = fh.read() + # ``suite_name_var`` is the local name declared by + # ``control_no_instance.meta`` for the suite_name control var. + self.assertIn( + 'subroutine ccpp_init(suite_name_var, errflg, errmsg)', + text, + ) + # And NOT the multi-instance shape. + self.assertNotIn('inst_num', text) + + def test_static_api_ccpp_register_omits_inst_num(self): + with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + text = fh.read() + self.assertIn( + 'subroutine ccpp_register(suite_name_var, errflg, errmsg)', + text, + ) + + def test_static_api_ccpp_final_omits_inst_num(self): + with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + text = fh.read() + self.assertIn( + 'subroutine ccpp_final(suite_name_var, errflg, errmsg)', + text, + ) + + def test_suite_init_omits_inst_num(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn('subroutine test_simple_init(errmsg, errflg)', text) + + def test_register_passes_literal_one_to_state_alloc(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'call test_simple_suite_state_alloc(1, errmsg, errflg)', text, + ) + + def test_suite_state_indexing_uses_literal_one(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn('ccpp_suite_state(1)', text) + + def test_group_state_alloc_passes_literal_one(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: + text = fh.read() + self.assertIn( + 'call ccpp_test_simple_physics_state_alloc(1, errmsg, errflg)', + text, + ) + + def test_group_cap_init_omits_inst_num(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: + text = fh.read() + init_sub = text.split('subroutine ccpp_test_simple_physics_init')[1] + init_sub = init_sub.split('end subroutine')[0] + self.assertNotIn('inst_num', init_sub) + + +class TestInstancePairingErrors(unittest.TestCase): + """End-to-end: ``capgen`` rejects hosts that declare exactly one of the + instance_number / number_of_instances pair.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_instance_alone_raises(self): + from metadata.parse_tools import CCPPError + with self.assertRaises(CCPPError) as ctx: + capgen( + host_name='test_host', + host_files=[ + _sf('host_no_instance.meta'), # no number_of_instances + _sf('control_full.meta'), # has instance_number + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + msg = str(ctx.exception) + self.assertIn('instance_number', msg) + self.assertIn('number_of_instances', msg) + self.assertIn('paired', msg.lower()) + + def test_ninstances_alone_raises(self): + from metadata.parse_tools import CCPPError + with self.assertRaises(CCPPError) as ctx: + capgen( + host_name='test_host', + host_files=[ + _sf('host_full.meta'), # has number_of_instances + _sf('control_no_instance.meta'),# no instance_number + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + msg = str(ctx.exception) + self.assertIn('instance_number', msg) + self.assertIn('number_of_instances', msg) + self.assertIn('paired', msg.lower()) + + +# --------------------------------------------------------------------------- +# Helper runners for ported test_prebuild test cases +# --------------------------------------------------------------------------- + +def _run_opt_arg(tmpdir): + capgen( + host_name='host', + host_files=[_sf('host_opt_arg.meta'), _sf('control_opt_arg.meta')], + scheme_files=[_sf('scheme_opt_arg.meta')], + suite_files=[_suite_file('suite_opt_arg.xml')], + output_root=tmpdir, + kind_types={}, + ) + return tmpdir + + +def _run_unit_conv(tmpdir): + capgen( + host_name='host', + host_files=[_sf('host_unit_conv.meta'), _sf('control_unit_conv.meta')], + scheme_files=[ + _sf('scheme_unit_conv_1.meta'), + _sf('scheme_unit_conv_2.meta'), + ], + suite_files=[_suite_file('suite_unit_conv.xml')], + output_root=tmpdir, + kind_types={}, + ) + return tmpdir + + +def _run_chunked_data(tmpdir): + capgen( + host_name='host', + host_files=[ + _sf('host_chunked_data.meta'), + _sf('ddt_chunked_data.meta'), + _sf('control_chunked_data.meta'), + ], + scheme_files=[_sf('scheme_chunked_data.meta')], + suite_files=[_suite_file('suite_chunked_data.xml')], + output_root=tmpdir, + kind_types={}, + ) + return tmpdir + + +# --------------------------------------------------------------------------- +# Test: suite types module (optional variable pointer wrappers) +# --------------------------------------------------------------------------- + +class TestSuiteTypesModule(unittest.TestCase): + """Suite types module is generated when optional args are present.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_opt_arg(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _path(self, name): + return os.path.join(self._tmpdir, name) + + def test_types_module_exists(self): + self.assertTrue(os.path.isfile(self._path('ccpp_opt_arg_types.F90'))) + + def test_types_module_declaration(self): + with open(self._path('ccpp_opt_arg_types.F90')) as fh: + text = fh.read() + self.assertIn('module ccpp_opt_arg_types', text) + self.assertIn('end module ccpp_opt_arg_types', text) + + def test_integer_ptr_type_declared(self): + with open(self._path('ccpp_opt_arg_types.F90')) as fh: + text = fh.read() + self.assertIn('type :: integer_rank1_ptr_type', text) + self.assertIn('integer, pointer :: ptr(:) => null()', text) + + def test_real_kind_phys_ptr_type_declared(self): + with open(self._path('ccpp_opt_arg_types.F90')) as fh: + text = fh.read() + self.assertIn('type :: real_kind_phys_rank1_ptr_type', text) + self.assertIn('real(kind=kind_phys), pointer :: ptr(:) => null()', text) + + def test_types_module_public_declarations(self): + with open(self._path('ccpp_opt_arg_types.F90')) as fh: + text = fh.read() + self.assertIn('public :: integer_rank1_ptr_type', text) + self.assertIn('public :: real_kind_phys_rank1_ptr_type', text) + + def test_types_module_uses_ccpp_kinds(self): + with open(self._path('ccpp_opt_arg_types.F90')) as fh: + text = fh.read() + # The real_kind_phys_rank1_ptr_type uses kind_phys → must USE it. + self.assertIn('use ccpp_kinds, only: kind_phys', text) + + def test_no_types_module_for_simple_suite(self): + with tempfile.TemporaryDirectory() as d: + _run_simple(d) + self.assertFalse(os.path.isfile( + os.path.join(d, 'ccpp_test_simple_types.F90') + )) + + +# --------------------------------------------------------------------------- +# Test: optional variable handling (ported from test_prebuild/test_opt_arg) +# --------------------------------------------------------------------------- + +class TestOptArgIntegration(unittest.TestCase): + """End-to-end test with Case 2 (optional, no transform) and Case 4 + (optional + unit conversion km→m).""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_opt_arg(self._tmpdir) + with open( + os.path.join(self._tmpdir, 'ccpp_opt_arg_opt_arg_group_cap.F90') + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_exists(self): + self.assertTrue( + os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_opt_arg_opt_arg_group_cap.F90') + ) + ) + + def test_uses_types_module(self): + self.assertIn('use ccpp_opt_arg_types', self.text) + + def test_group_cap_uses_ccpp_kinds(self): + # Group cap declares ``real(kind=kind_phys)`` transformation locals + # → must USE kind_phys from ccpp_kinds. + self.assertIn('use ccpp_kinds, only: kind_phys', self.text) + + def test_case2_ptr_type_declared(self): + self.assertIn('type(integer_rank1_ptr_type) :: opt_var_p', self.text) + + def test_case4_ptr_and_temp_declared(self): + self.assertIn('type(real_kind_phys_rank1_ptr_type) :: opt_var_2_p', self.text) + self.assertIn('opt_var_2_l', self.text) + + def test_case4_temp_has_target_attr(self): + # The temp is the RHS of ``ptr%ptr => temp``, so it must be a TARGET. + # Match the declaration line: ``real(kind=kind_phys), dimension(nx), target :: opt_var_2_l`` + self.assertRegex( + self.text, + r'real\(kind=kind_phys\)[^\n]*,\s*target[^\n]*::\s*opt_var_2_l\b', + ) + + def test_case2_pre_call_active_guard(self): + self.assertIn('if (flag_for_opt_arg) then', self.text) + + def test_case2_ptr_assignment(self): + self.assertIn('opt_var_p%ptr => opt_arg', self.text) + + def test_case2_nullify_on_else(self): + self.assertIn('nullify(opt_var_p%ptr)', self.text) + + def test_case4_forward_unit_conversion(self): + # km → m: multiply by 1.0E+3 + self.assertIn('opt_var_2_l = 1.0E+3_kind_phys*opt_arg_2', self.text) + + def test_case4_ptr_to_temp(self): + self.assertIn('opt_var_2_p%ptr => opt_var_2_l', self.text) + + def test_case4_backward_unit_conversion(self): + # m → km: multiply by 1.0E-3 + self.assertIn('opt_arg_2', self.text) + self.assertIn('1.0E-3_kind_phys*opt_var_2_l', self.text) + + def test_optional_arg_passed_as_ptr(self): + self.assertIn('opt_var=opt_var_p%ptr', self.text) + self.assertIn('opt_var_2=opt_var_2_p%ptr', self.text) + + def test_active_condition_from_host(self): + # Active condition inherited from host metadata, not scheme metadata. + self.assertIn('flag_for_opt_arg', self.text) + + def test_types_in_datatable(self): + root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + suite_files = root.find('capgen_files').find('suite_files') + names = [os.path.basename(f.text) for f in suite_files.findall('file')] + self.assertIn('ccpp_opt_arg_types.F90', names) + + +# --------------------------------------------------------------------------- +# Test: unit conversion (ported from test_prebuild/test_unit_conv, simplified) +# --------------------------------------------------------------------------- + +class TestSubcycleStdnameLoopBound(unittest.TestCase): + """End-to-end: a subcycle with ``loop=""`` must resolve to + the host's local Fortran name in the generated ``do`` loop, plus + emit the needed USE / dummy-arg threading.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[ + _sf('host_full.meta'), + _sf('control_full.meta'), + _sf('host_subcycle_stdname.meta'), + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_subcycle_stdname.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open(os.path.join( + self._tmpdir, + 'ccpp_subcycle_stdname_suite_physics_cap.F90', + )) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_do_loop_uses_local_name(self): + """The do-loop bound is the host's local Fortran name (n_sub), + not the CCPP standard name (num_subcycles_for_test).""" + self.assertIn('do ccpp_loop_counter = 1, n_sub', self.text) + # And NOT the raw std name. + self.assertNotIn('do ccpp_loop_counter = 1, num_subcycles_for_test', + self.text) + + def test_use_statement_imports_loop_local(self): + """The host module exporting n_sub is USE'd so the symbol is + in scope inside the generated subroutine.""" + # n_sub comes from host_phys_subcycle_helper module. + self.assertIn('use host_phys_subcycle_helper', self.text) + self.assertIn('n_sub', self.text) + + +class TestHostTableDependenciesInDatatable(unittest.TestCase): + """A host table's ``dependencies =`` declarations must surface in + datatable.xml's section. The generator originally + only walked scheme_tables for deps, silently dropping host-table + declarations (real CCPP-physics hosts like SCM's GFS_typedefs.meta + declare many host-side file dependencies).""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[ + _sf('host_with_dependencies.meta'), + _sf('control_full.meta'), + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + import xml.etree.ElementTree as ET + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + self._deps = [ + d.text for d in tree.getroot() + .find('dependencies').findall('dependency') + ] + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_host_dependencies_listed(self): + """Every dep declared on the host table appears in + with the dependencies_path prefix resolved.""" + names = [os.path.basename(p) for p in self._deps] + self.assertIn('some_mp_params.F90', names) + self.assertIn('some_rad_param.f', names) + self.assertIn('some_chem.F90', names) + + def test_dependencies_path_applied_to_host_deps(self): + """The host's dependencies_path is resolved against each entry.""" + joined = '\n'.join(self._deps) + self.assertIn('/tmp/fake_phys/mp/some_mp_params.F90', joined) + self.assertIn('/tmp/fake_phys/radiation/some_rad_param.f', joined) + self.assertIn('/tmp/fake_phys/chemistry/some_chem.F90', joined) + + +class TestSuiteInitFinalEmission(unittest.TestCase): + """End-to-end: an SDF with ```` and ```` at the suite + level produces calls to the named scheme's init/final phases inside + ``_init`` and ``_final``, with USE statements for the + scheme module and the standard error-flag check after the call.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[ + _sf('scheme_multipart.meta'), + _sf('scheme_suite_init_final.meta'), + ], + suite_files=[_suite_file('suite_with_init_final.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open(os.path.join( + self._tmpdir, 'ccpp_with_init_final_suite_cap.F90', + )) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_suite_init_calls_init_scheme(self): + sub = self.text.split('subroutine with_init_final_suite_init')[1] + sub = sub.split('end subroutine')[0] + self.assertIn('call suite_init_final_scheme_init', sub) + + def test_suite_final_calls_final_scheme(self): + sub = self.text.split('subroutine with_init_final_suite_final')[1] + sub = sub.split('end subroutine')[0] + self.assertIn('call suite_init_final_scheme_final', sub) + + def test_init_call_uses_scheme_module(self): + """The scheme module is USE'd inside ``_init`` so the + init subroutine is in scope.""" + sub = self.text.split('subroutine with_init_final_suite_init')[1] + sub = sub.split('end subroutine')[0] + self.assertIn( + 'use suite_init_final_scheme, only:', sub, + ) + self.assertIn('suite_init_final_scheme_init', sub) + + def test_final_call_uses_scheme_module(self): + sub = self.text.split('subroutine with_init_final_suite_final')[1] + sub = sub.split('end subroutine')[0] + self.assertIn( + 'use suite_init_final_scheme, only:', sub, + ) + self.assertIn('suite_init_final_scheme_final', sub) + + def test_init_call_precedes_state_transition(self): + """The init scheme is called BEFORE the FRAMEWORK_INITIALIZED + state transition — failures during the suite-init scheme stop + the state transition from firing.""" + sub = self.text.split('subroutine with_init_final_suite_init')[1] + sub = sub.split('end subroutine')[0] + call_pos = sub.index('call suite_init_final_scheme_init') + state_pos = sub.index('CCPP_SUITE_FRAMEWORK_INITIALIZED') + # There may be multiple state references (early-return guard); + # the final assignment is what matters. + state_set = sub.rindex( + 'ccpp_suite_state({}) = CCPP_SUITE_FRAMEWORK_INITIALIZED'.format( + 'inst_num' if 'inst_num' in sub else '1' + ) + ) + self.assertLess(call_pos, state_set) + + def test_final_call_precedes_unregister_transition(self): + sub = self.text.split('subroutine with_init_final_suite_final')[1] + sub = sub.split('end subroutine')[0] + call_pos = sub.index('call suite_init_final_scheme_final') + state_set = sub.index( + 'ccpp_suite_state({}) = CCPP_SUITE_UNREGISTERED'.format( + 'inst_num' if 'inst_num' in sub else '1' + ) + ) + self.assertLess(call_pos, state_set) + + +class TestNestedSubcycleEmission(unittest.TestCase): + """End-to-end: a nested ```` in the SDF must produce + nested ``do`` loops in the generated cap. Without this, schemes + inside the inner loops run fewer times than the SDF specified, + silently producing wrong numerical answers.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_nested_subcycle.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open(os.path.join( + self._tmpdir, + 'ccpp_nested_subcycle_suite_physics_cap.F90', + )) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_outer_loop_present(self): + """The outer loop uses ``ccpp_loop_counter`` (preserves the + existing single-level convention).""" + self.assertIn('do ccpp_loop_counter = 1, 3', self.text) + + def test_inner_loop_present(self): + """The inner loop uses ``ccpp_loop_counter_2`` so the two loop + vars are distinct in the same scope (Fortran requires it).""" + self.assertIn('do ccpp_loop_counter_2 = 1, 2', self.text) + + def test_two_end_do_statements(self): + """Each nesting level closes with an ``end do``.""" + # Find the run-phase body to scope the check. + run = self.text.split('subroutine ccpp_nested_subcycle_suite_physics_run')[1] + run = run.split('end subroutine')[0] + self.assertEqual(run.count('end do'), 2) + + def test_inner_loop_inside_outer(self): + """The inner ``do`` line appears AFTER the outer ``do`` line + and BEFORE the first ``end do``.""" + run = self.text.split('subroutine ccpp_nested_subcycle_suite_physics_run')[1] + run = run.split('end subroutine')[0] + outer = run.index('do ccpp_loop_counter = 1, 3') + inner = run.index('do ccpp_loop_counter_2 = 1, 2') + first_end = run.index('end do') + self.assertLess(outer, inner) + self.assertLess(inner, first_end) + + def test_scheme_call_inside_inner_loop(self): + """The scheme call is nested inside the inner loop — not in + between the loops.""" + run = self.text.split('subroutine ccpp_nested_subcycle_suite_physics_run')[1] + run = run.split('end subroutine')[0] + inner = run.index('do ccpp_loop_counter_2 = 1, 2') + call = run.index('call temp_calc_adjust_run') + # First end do is the inner loop's close. + first_end = run.index('end do') + self.assertLess(inner, call) + self.assertLess(call, first_end) + + def test_two_counter_declarations(self): + """Each loop variable has its own integer declaration.""" + self.assertIn('integer :: ccpp_loop_counter\n', self.text) + self.assertIn('integer :: ccpp_loop_counter_2\n', self.text) + + +class TestSubcycleStdnameLoopBoundDdt(unittest.TestCase): + """End-to-end: a subcycle ``loop=""`` that resolves to a + DDT-component must emit the access path (``%``) as the + loop bound and USE the DDT instance's parent module by the *root* + of that access path — not the bare component name. + """ + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[ + _sf('host_full.meta'), + _sf('control_full.meta'), + _sf('host_subcycle_stdname_ddt.meta'), + _sf('ddt_subcycle_stdname.meta'), + ], + scheme_files=[_sf('scheme_multipart.meta')], + suite_files=[_suite_file('suite_subcycle_stdname_ddt.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open(os.path.join( + self._tmpdir, + 'ccpp_subcycle_stdname_ddt_suite_physics_cap.F90', + )) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_do_loop_uses_access_path(self): + """The do-loop bound is the access path of the DDT component, + not the bare component name.""" + self.assertIn( + 'do ccpp_loop_counter = 1, phys_state%n_sub', self.text, + ) + # And NOT the bare component or the standard name. + self.assertNotIn('do ccpp_loop_counter = 1, n_sub\n', self.text) + self.assertNotIn( + 'do ccpp_loop_counter = 1, num_subcycles_for_test', self.text, + ) + + def test_use_imports_ddt_instance_root(self): + """The USE statement targets the DDT *instance* (phys_state), + not the bare component (n_sub).""" + # Some line starting with `use test_host_with_ddt_mod, only: ...` + # must list phys_state — n_sub is not a free module symbol. + self.assertIn('phys_state', self.text) + # The bare component must not be on the ``only:`` clause. + import re + m = re.search( + r'use\s+test_host_with_ddt_mod\s*,\s*only:\s*([^\n]+)', + self.text, + ) + self.assertIsNotNone(m) + only_clause = m.group(1) + self.assertIn('phys_state', only_clause) + # Word-boundary check: ``n_sub`` may legitimately appear inside + # ``phys_state%n_sub`` further down in the file, but it must not + # be a free symbol on this USE line. + symbols = [s.strip() for s in only_clause.split(',')] + self.assertNotIn('n_sub', symbols) + + +class TestModuleNameOverrideIntegration(unittest.TestCase): + """End-to-end: a scheme whose ``[ccpp-table-properties]`` declares an + explicit ``module_name`` must cause the generated cap to emit + ``use `` rather than the table's ``name``.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[_sf('scheme_module_name_override.meta')], + suite_files=[_suite_file('suite_module_name_override.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open(os.path.join( + self._tmpdir, + 'ccpp_module_name_override_suite_physics_cap.F90', + )) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_uses_module_name_not_scheme_name(self): + """The USE statement targets the Fortran module name.""" + self.assertIn('use mod_alt_name, only:', self.text) + + def test_group_cap_does_not_use_scheme_name_directly(self): + """The scheme-name token (``scheme_alt_name``) must not appear as a + module on a ``use`` line — it would resolve to a non-existent + module file at compile time.""" + # Word-boundary check: ``scheme_alt_name_run`` (the subroutine) is + # legitimately mentioned in the ``only:`` clause. + import re + bad = re.search(r'use\s+scheme_alt_name\s*,', self.text) + self.assertIsNone(bad, + "Cap should not emit 'use scheme_alt_name, ...' when the " + "metadata declares module_name = mod_alt_name") + + def test_only_clause_carries_phase_subroutine(self): + """The ``only:`` clause exposes ``_`` symbols + from the renamed module — Fortran subroutine names stay tied to + the scheme name, not the module name.""" + self.assertIn('scheme_alt_name_run', self.text) + + +class TestVerticalFlipIntegration(unittest.TestCase): + """End-to-end: host declares air_temperature with default + top_at_one = False; scheme declares top_at_one = True. The + generated group cap must emit a temp + flipped subscript on the + host-side access expression and pass the temp to the scheme. + """ + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[_sf('scheme_top_at_one.meta')], + suite_files=[_suite_file('suite_top_at_one.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + with open( + os.path.join( + self._tmpdir, + 'ccpp_top_at_one_suite_physics_cap.F90', + ) + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_exists(self): + self.assertTrue( + os.path.isfile( + os.path.join( + self._tmpdir, + 'ccpp_top_at_one_suite_physics_cap.F90', + ) + ) + ) + + def test_temp_local_declared(self): + # Transform pipeline kicks in: temp local named ``temp_l`` with + # scheme dimensions (lb:ub on horizontal, nlev on vertical — the + # vertical bound is single-extent shorthand for 1:nlev). + self.assertIn('real(kind=kind_phys), dimension(lb:ub, nlev) :: temp_l', + self.text) + + def test_pre_call_forward_uses_flipped_subscript(self): + # Host metadata declares air_temperature as ``gt0`` with default + # top_at_one = False; scheme wants top_at_one = True → reverse + # stride on the vertical axis of the host-side expression. + self.assertIn('temp_l = gt0(lb:ub, nlev:1:-1)', self.text) + + def test_post_call_backward_writes_into_flipped_lhs(self): + # inout, so backward also fires; writes temp back into host with + # the flipped subscript as LHS. + self.assertIn('gt0(lb:ub, nlev:1:-1) = temp_l', self.text) + + def test_call_site_passes_temp_not_host(self): + # The scheme is called with the temp local, not the host expression. + call_block = self.text.split('call top_at_one_scheme_run')[1] + call_block = call_block.split(')')[0] + self.assertIn('temp=temp_l', call_block) + + def test_comment_mentions_vertical_flip(self): + self.assertIn('vertical flip', self.text) + + +class TestUnitConvIntegration(unittest.TestCase): + """Two schemes in one group: scheme_1 needs m (Case 1/2), + scheme_2 needs km (Case 3/4 — m→km forward, km→m backward).""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_unit_conv(self._tmpdir) + with open( + os.path.join( + self._tmpdir, 'ccpp_unit_conv_unit_conv_group_cap.F90' + ) + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_exists(self): + self.assertTrue( + os.path.isfile( + os.path.join( + self._tmpdir, 'ccpp_unit_conv_unit_conv_group_cap.F90' + ) + ) + ) + + def test_case1_direct_call_scheme_1(self): + # Scheme 1 wants m, host has m — direct reference, no temp. + self.assertIn('call unit_conv_scheme_1_run', self.text) + # data_array should be passed directly (no _l suffix for scheme_1 data_array) + self.assertIn('data_array=data_array(lb:ub)', self.text) + + def test_case3_temp_forward_for_scheme_2(self): + # Scheme 2 wants km: m→km conversion. + self.assertIn('1.0E-3_kind_phys*data_array(lb:ub)', self.text) + + def test_case3_backward_for_scheme_2(self): + # km→m backward conversion. + self.assertIn('1.0E+3_kind_phys*', self.text) + + def test_case2_optional_ptr_for_scheme_1(self): + # Scheme 1 optional data_array_opt: same units, pointer-only. + self.assertIn('data_array_opt_p', self.text) + + def test_case4_optional_plus_transform_for_scheme_2(self): + # Scheme 2 optional data_array_opt: pointer + km transform. + self.assertIn('data_array_opt_2_p', self.text) + + def test_types_module_for_unit_conv(self): + self.assertTrue( + os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_unit_conv_types.F90') + ) + ) + + +# --------------------------------------------------------------------------- +# Test: chunked data (ported from test_prebuild/test_chunked_data) +# --------------------------------------------------------------------------- + +class TestChunkedDataIntegration(unittest.TestCase): + """Suite with DDT-based host variable and ccpp_chunk_number control var.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_chunked_data(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_group_cap_exists(self): + self.assertTrue( + os.path.isfile( + os.path.join( + self._tmpdir, + 'ccpp_chunked_data_chunked_data_group_cap.F90', + ) + ) + ) + + def test_scheme_run_calls_present(self): + with open( + os.path.join( + self._tmpdir, + 'ccpp_chunked_data_chunked_data_group_cap.F90', + ) + ) as fh: + text = fh.read() + self.assertIn('call chunked_data_scheme_run', text) + + def test_ddt_access_in_group_cap(self): + with open( + os.path.join( + self._tmpdir, + 'ccpp_chunked_data_chunked_data_group_cap.F90', + ) + ) as fh: + text = fh.read() + # DDT field access should appear in the call expression. + self.assertIn('chunked_data_instance%array_data', text) + + def test_chunk_number_control_arg(self): + with open( + os.path.join( + self._tmpdir, + 'ccpp_chunked_data_chunked_data_group_cap.F90', + ) + ) as fh: + text = fh.read() + # ccpp_chunk_number → local name nchunk in the run subroutine. + self.assertIn('nchunk', text) + + def test_no_types_module_for_chunked_data(self): + self.assertFalse( + os.path.isfile( + os.path.join(self._tmpdir, 'ccpp_chunked_data_types.F90') + ) + ) + + def test_scheme_module_imported(self): + with open( + os.path.join( + self._tmpdir, + 'ccpp_chunked_data_chunked_data_group_cap.F90', + ) + ) as fh: + text = fh.read() + # Group cap must import the actual scheme module so that + # `call chunked_data_scheme_(...)` resolves. + self.assertIn('use chunked_data_scheme, only:', text) + for phase in ('init', 'timestep_init', 'run', + 'timestep_final', 'final'): + self.assertIn('chunked_data_scheme_{}'.format(phase), text) + + def test_phase_subroutine_order_canonical(self): + with open( + os.path.join( + self._tmpdir, + 'ccpp_chunked_data_chunked_data_group_cap.F90', + ) + ) as fh: + text = fh.read() + # Phase subroutines and public declarations must appear in canonical + # order: init, timestep_init, run, timestep_final, final, then + # state_alloc, state_dealloc. + canonical = [ + '_init', '_timestep_init', '_run', '_timestep_final', '_final', + '_state_alloc', '_state_dealloc', + ] + positions = [ + text.index('public :: ccpp_chunked_data_chunked_data_group{}'.format(s)) + for s in canonical + ] + self.assertEqual(positions, sorted(positions)) + + def test_datatable_has_chunked_data_suite(self): + root = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')).getroot() + suites = root.find('api').find('suites') + names = [s.get('name') for s in suites.findall('suite')] + self.assertIn('chunked_data', names) + + +# --------------------------------------------------------------------------- +# Test: interstitial suite-owned data — allocation and multi-instance +# --------------------------------------------------------------------------- + +def _run_interstitial(tmpdir): + from ccpp_capgen_ng import capgen + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[ + _sf('scheme_interstitial_producer.meta'), + _sf('scheme_interstitial_consumer.meta'), + ], + suite_files=[_suite_file('suite_interstitial.xml')], + output_root=tmpdir, + kind_types={}, + ) + + +class TestInterstitialSuiteData(unittest.TestCase): + """Suite-owned interstitial variable: deferred-shape DDT, allocatable + instance array, suite_state_alloc wired into suite init.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_interstitial(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _data(self): + return open(os.path.join(self._tmpdir, 'ccpp_interstitial_data.F90')).read() + + def _suite_cap(self): + return open(os.path.join(self._tmpdir, 'ccpp_interstitial_cap.F90')).read() + + def _group_cap(self): + return open(os.path.join(self._tmpdir, + 'ccpp_interstitial_diag_group_cap.F90')).read() + + def test_suite_data_field_uses_deferred_shape(self): + """Allocatable field must use (:,:) not dimension(std_name,...).""" + text = self._data() + self.assertIn('(:,:)', text) + self.assertNotIn('dimension(horizontal_dimension', text) + + def test_suite_data_has_allocatable_instance_array(self): + text = self._data() + self.assertIn('type(ccpp_interstitial_data_t), allocatable', text) + self.assertIn('ccpp_suite_data(:)', text) + + def test_suite_data_alloc_subroutine_exists(self): + self.assertIn('ccpp_interstitial_suite_data_alloc', self._data()) + + def test_suite_data_init_fields_uses_host_dims(self): + # init_fields now owns the inner allocations; it needs the host dims. + text = self._data() + init_fields = text.split('subroutine ccpp_interstitial_suite_data_init_fields')[1].split('end subroutine')[0] + self.assertIn('use host_phys', init_fields) + self.assertIn('ncols', init_fields) + self.assertIn('nlev', init_fields) + + def test_suite_data_alloc_allocates_outer_array(self): + self.assertIn('allocate(ccpp_suite_data(number_of_instances))', self._data()) + + def test_suite_data_init_fields_allocates_field_per_instance(self): + # Inner allocations moved out of suite_data_alloc into init_fields so + # suite-owned dims (e.g. set during _register) can be picked up after + # the register phase has run. + text = self._data() + init_fields = text.split('subroutine ccpp_interstitial_suite_data_init_fields')[1].split('end subroutine')[0] + self.assertIn('allocate(ccpp_suite_data(i)%diag_out(ncols, nlev))', init_fields) + + def test_suite_data_dealloc_subroutine_exists(self): + self.assertIn('ccpp_interstitial_suite_data_dealloc', self._data()) + + def test_suite_data_final_fields_subroutine_exists(self): + self.assertIn('ccpp_interstitial_suite_data_final_fields', self._data()) + + def test_suite_cap_has_suite_state_alloc(self): + self.assertIn('interstitial_suite_state_alloc', self._suite_cap()) + + def test_suite_cap_register_calls_suite_state_alloc(self): + # State + DDT-array allocation has moved from _init into + # _register so register can be the first lifecycle entry point. + text = self._suite_cap() + register_body = text.split('subroutine interstitial_register')[1].split('end subroutine')[0] + self.assertIn('interstitial_suite_state_alloc', register_body) + + def test_suite_state_alloc_calls_suite_data_alloc(self): + text = self._suite_cap() + alloc_body = text.split('subroutine interstitial_suite_state_alloc')[1].split('end subroutine')[0] + self.assertIn('ccpp_interstitial_suite_data_alloc', alloc_body) + + def test_suite_init_calls_init_fields(self): + # _init triggers per-instance inner allocations. + text = self._suite_cap() + init_body = text.split('subroutine interstitial_init')[1].split('end subroutine')[0] + self.assertIn('ccpp_interstitial_suite_data_init_fields', init_body) + + def test_suite_state_alloc_allocates_state_array(self): + self.assertIn('allocate(ccpp_suite_state(number_of_instances))', self._suite_cap()) + + def test_suite_cap_final_calls_suite_state_dealloc(self): + text = self._suite_cap() + final_body = text.split('subroutine interstitial_final')[1].split('end subroutine')[0] + self.assertIn('interstitial_suite_state_dealloc', final_body) + + def test_group_cap_uses_suite_data_module(self): + self.assertIn('use ccpp_interstitial_data', self._group_cap()) + + def test_group_cap_accesses_suite_data_with_instance(self): + """Suite var access must index ccpp_suite_data by instance.""" + text = self._group_cap() + self.assertIn('ccpp_suite_data(', text) + + def test_group_run_has_inst_num_dummy_arg(self): + """Gap 3: instance_number must be a dummy arg when suite vars are referenced.""" + text = self._group_cap() + run_sub = text.split('subroutine ccpp_interstitial_diag_group_run')[1] + run_sub = run_sub.split('end subroutine')[0] + self.assertIn('inst_num', run_sub) + self.assertIn('integer, intent(in)', run_sub) + + def test_suite_cap_passes_inst_num_to_group_run(self): + """Suite cap physics_run dispatch must pass inst_num to group run.""" + text = self._suite_cap() + physics_run = text.split('subroutine interstitial_physics_run')[1] + physics_run = physics_run.split('end subroutine')[0] + self.assertIn('inst_num', physics_run) + self.assertIn('call ccpp_interstitial_diag_group_run', physics_run) + + def test_static_api_physics_run_has_inst_num(self): + """Static API ccpp_physics_run must include inst_num when groups need it.""" + with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + text = fh.read() + physics_run = text.split('subroutine ccpp_physics_run')[1] + physics_run = physics_run.split('end subroutine')[0] + self.assertIn('inst_num', physics_run) + + def test_suite_meta_file_exists(self): + """Polish 2: ccpp_.meta must be generated.""" + self.assertTrue( + os.path.isfile(os.path.join(self._tmpdir, 'ccpp_interstitial.meta')) + ) + + def test_suite_meta_contains_suite_var(self): + """Suite meta file must list suite-owned variable.""" + with open(os.path.join(self._tmpdir, 'ccpp_interstitial.meta')) as fh: + text = fh.read() + self.assertIn('diagnostic_interstitial_field', text) + self.assertIn('type = suite', text) + self.assertIn('standard_name = diagnostic_interstitial_field', text) + + +# --------------------------------------------------------------------------- +# Test: Gap 1 — active expression variables in USE statements +# --------------------------------------------------------------------------- + +class TestActiveVarInUseStatements(unittest.TestCase): + """Gap 1: Variables referenced in active= expressions must appear in USE.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + from test_integration import _run_unit_conv + _run_unit_conv(self._tmpdir) + with open( + os.path.join(self._tmpdir, 'ccpp_unit_conv_unit_conv_group_cap.F90') + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_active_flag_in_use_statement(self): + """flag_for_opt_array is referenced in active= but was not a direct arg.""" + self.assertIn('flag_for_opt_array', self.text.split('contains')[0]) + + def test_active_flag_is_use_d_not_declared(self): + """The active flag must appear in a use statement, not a dummy declaration.""" + use_section = self.text.split('implicit none')[0] + self.assertIn('flag_for_opt_array', use_section) + + +# --------------------------------------------------------------------------- +# Test: Gap 2 — temp variable declarations use local names not standard names +# --------------------------------------------------------------------------- + +class TestTempDeclLocalNames(unittest.TestCase): + """Gap 2: Temp locals for unit conversion must be declared with local names.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + from test_integration import _run_unit_conv + _run_unit_conv(self._tmpdir) + with open( + os.path.join(self._tmpdir, 'ccpp_unit_conv_unit_conv_group_cap.F90') + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_temp_uses_chunk_bounds(self): + """Temp array for a horizontal_dimension scheme arg must be sized + with the chunk loop bounds (host's local names for + horizontal_loop_begin / horizontal_loop_end), NOT the full-extent + name. Using ``dimension(ncols)`` over-sizes the temp and produces + a Fortran shape mismatch on the unit-conversion assignment like + ``data_array_l = factor * data_array(lb:ub)``. + + The unit_conv fixture's control table uses ``lb`` / ``ub`` as the + local names for horizontal_loop_begin / horizontal_loop_end. + """ + self.assertIn('dimension(lb:ub)', self.text) + # And NOT the full-extent name. + self.assertNotIn('dimension(ncols)', self.text) + + def test_temp_does_not_use_standard_name(self): + """Standard name must not appear in a dimension() declaration.""" + self.assertNotIn('dimension(horizontal_dimension)', self.text) + self.assertNotIn('dimension(horizontal_loop_extent)', self.text) + self.assertNotIn('dimension(horizontal_loop_begin', self.text) + + +class TestTempDeclLocalNamesOptArg(unittest.TestCase): + """Gap 2: Temp decl uses nx (local) not size_of_std_arg (standard name).""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + from test_integration import _run_opt_arg + _run_opt_arg(self._tmpdir) + with open( + os.path.join(self._tmpdir, 'ccpp_opt_arg_opt_arg_group_cap.F90') + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_temp_uses_local_dim_name(self): + self.assertIn('dimension(nx)', self.text) + + def test_temp_does_not_use_standard_name(self): + self.assertNotIn('dimension(size_of_std_arg)', self.text) + + +# --------------------------------------------------------------------------- +# Test: Polish 1 — timestep_init/timestep_final phase state guards +# --------------------------------------------------------------------------- + +class TestTimestepStateGuards(unittest.TestCase): + """Polish 1: timestep_init must guard on IN_TIMESTEP; timestep_final resets.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + from test_integration import _run_opt_arg + _run_opt_arg(self._tmpdir) + with open( + os.path.join(self._tmpdir, 'ccpp_opt_arg_opt_arg_group_cap.F90') + ) as fh: + self.text = fh.read() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_timestep_init_entry_guard(self): + """timestep_init subroutine must have IN_TIMESTEP entry guard.""" + ts_init = self.text.split( + 'subroutine ccpp_opt_arg_opt_arg_group_timestep_init' + )[1].split('end subroutine')[0] + self.assertIn('CCPP_GROUP_IN_TIMESTEP', ts_init) + self.assertIn('return', ts_init) + + def test_timestep_init_sets_in_timestep(self): + """timestep_init must set state to IN_TIMESTEP after scheme calls.""" + ts_init = self.text.split( + 'subroutine ccpp_opt_arg_opt_arg_group_timestep_init' + )[1].split('end subroutine')[0] + self.assertIn('ccpp_group_state', ts_init) + self.assertIn('CCPP_GROUP_IN_TIMESTEP', ts_init) + + def test_timestep_final_resets_to_initialized(self): + """timestep_final must reset state to INITIALIZED.""" + ts_final = self.text.split( + 'subroutine ccpp_opt_arg_opt_arg_group_timestep_final' + )[1].split('end subroutine')[0] + self.assertIn('CCPP_GROUP_INITIALIZED', ts_final) + self.assertIn('ccpp_group_state', ts_final) + + +# --------------------------------------------------------------------------- +# Test: Polish 2 — ccpp_.meta output file +# --------------------------------------------------------------------------- + +class TestSuiteMetaOutput(unittest.TestCase): + """Polish 2: ccpp_.meta must be written for every suite.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_meta_file_created(self): + self.assertTrue( + os.path.isfile(os.path.join(self._tmpdir, 'ccpp_test_simple.meta')) + ) + + def test_meta_header_comment(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple.meta')) as fh: + text = fh.read() + self.assertTrue(text.startswith('!')) + self.assertIn('ccpp_test_simple', text.split('\n')[0]) + + def test_meta_table_properties(self): + with open(os.path.join(self._tmpdir, 'ccpp_test_simple.meta')) as fh: + text = fh.read() + self.assertIn('[ccpp-table-properties]', text) + self.assertIn('name = ccpp_test_simple_data', text) + self.assertIn('type = suite', text) + + def test_meta_empty_suite_has_no_var_entries(self): + """simple suite has no suite-owned vars so no [ var ] blocks.""" + with open(os.path.join(self._tmpdir, 'ccpp_test_simple.meta')) as fh: + text = fh.read() + self.assertNotIn('standard_name =', text) + + def test_meta_with_suite_vars_lists_them(self): + """Interstitial suite's meta must list the interstitial variable.""" + with tempfile.TemporaryDirectory() as d: + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[ + _sf('scheme_interstitial_producer.meta'), + _sf('scheme_interstitial_consumer.meta'), + ], + suite_files=[_suite_file('suite_interstitial.xml')], + output_root=d, + kind_types={}, + ) + with open(os.path.join(d, 'ccpp_interstitial.meta')) as fh: + text = fh.read() + self.assertIn('standard_name = diagnostic_interstitial_field', text) + self.assertIn('units = K', text) + self.assertIn('type = real', text) + self.assertIn('kind = kind_phys', text) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_kinds_writer.py b/unit-tests/test_kinds_writer.py new file mode 100644 index 00000000..d15baf5e --- /dev/null +++ b/unit-tests/test_kinds_writer.py @@ -0,0 +1,156 @@ +"""Unit tests for generator.kinds_writer.""" + +import doctest +import os +import tempfile +import unittest + +from metadata.parse_tools import CCPPError +from generator.kinds_writer import _generate_ccpp_kinds, write_ccpp_kinds + +_ISO = 'iso_fortran_env' + + +class TestGenerateCcppKinds(unittest.TestCase): + + def test_single_kind(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + self.assertIn('module ccpp_kinds', lines) + self.assertIn('end module ccpp_kinds', lines) + self.assertTrue(any('kind_phys' in l and 'REAL64' in l for l in lines)) + self.assertTrue(any('parameter' in l and 'public' in l for l in lines)) + + def test_iso_use_single(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + iso_line = next(l for l in lines if 'iso_fortran_env' in l) + self.assertIn('REAL64', iso_line) + self.assertNotIn('&', iso_line) + + def test_iso_use_shared_spec_deduped(self): + """Two kinds sharing the same spec → spec listed once.""" + lines = _generate_ccpp_kinds({ + 'kind_a': (_ISO, 'REAL64'), + 'kind_b': (_ISO, 'REAL64'), + }) + iso_lines = [l for l in lines if 'iso_fortran_env' in l] + iso_text = ' '.join(iso_lines) + self.assertEqual(iso_text.count('REAL64'), 1) + + def test_iso_use_multiple_distinct_specs(self): + """Two distinct specs → both appear in the same use line.""" + lines = _generate_ccpp_kinds({ + 'kind_phys': (_ISO, 'REAL64'), + 'kind_dyn': (_ISO, 'REAL32'), + }) + iso_line = next(l for l in lines if 'iso_fortran_env' in l) + self.assertIn('REAL32', iso_line) + self.assertIn('REAL64', iso_line) + + def test_host_module(self): + """Host-supplied module emits use of that module with renamed param.""" + lines = _generate_ccpp_kinds({ + 'kind_phys': ('my_host_kinds', 'kind_r8'), + }) + text = '\n'.join(lines) + self.assertIn('use my_host_kinds, only: kind_r8', text) + self.assertIn('kind_phys = kind_r8', text) + self.assertNotIn('iso_fortran_env', text) + + def test_mixed_modules(self): + """Mixed modules → one use line per module, sorted alphabetically.""" + lines = _generate_ccpp_kinds({ + 'kind_phys': ('my_host_kinds', 'kind_r8'), + 'kind_iso': (_ISO, 'REAL64'), + }) + use_lines = [l for l in lines if l.lstrip().startswith('use ')] + self.assertEqual(len(use_lines), 2) + # Sorted alphabetically: iso_fortran_env first, then my_host_kinds. + self.assertIn('iso_fortran_env', use_lines[0]) + self.assertIn('my_host_kinds', use_lines[1]) + + def test_multiple_kinds_sorted(self): + lines = _generate_ccpp_kinds({ + 'kind_z': (_ISO, 'REAL32'), + 'kind_a': (_ISO, 'REAL64'), + }) + param_lines = [l for l in lines if 'parameter' in l and 'public' in l] + self.assertEqual(len(param_lines), 2) + idx_a = next(i for i, l in enumerate(lines) if 'kind_a' in l and 'parameter' in l) + idx_z = next(i for i, l in enumerate(lines) if 'kind_z' in l and 'parameter' in l) + self.assertLess(idx_a, idx_z) + + def test_aligned_equals(self): + """Declarations should be column-aligned on '='.""" + lines = _generate_ccpp_kinds({ + 'kind_phys': (_ISO, 'REAL64'), + 'k': (_ISO, 'REAL32'), + }) + param_lines = [l for l in lines if 'parameter' in l and 'public' in l] + eq_positions = [l.index('=') for l in param_lines] + self.assertEqual(len(set(eq_positions)), 1, "All '=' should be at the same column") + + def test_implicit_none_private(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + self.assertIn(' implicit none', lines) + self.assertIn(' private', lines) + + def test_empty_raises(self): + with self.assertRaises(CCPPError): + _generate_ccpp_kinds({}) + + def test_header_comment(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + self.assertTrue(lines[0].startswith('!')) + self.assertIn('ccpp_kinds', lines[0]) + + def test_no_trailing_newlines_in_lines(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + for line in lines: + self.assertNotIn('\n', line) + + def test_integer_parameter(self): + lines = _generate_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}) + param_line = next(l for l in lines if 'kind_phys' in l and 'parameter' in l) + self.assertIn('integer', param_line) + self.assertIn('parameter', param_line) + self.assertIn('public', param_line) + + +class TestWriteCcppKinds(unittest.TestCase): + + def test_writes_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}, tmpdir) + self.assertTrue(os.path.isfile(path)) + self.assertEqual(os.path.basename(path), 'ccpp_kinds.F90') + + def test_file_content(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}, tmpdir) + with open(path) as fh: + content = fh.read() + self.assertIn('module ccpp_kinds', content) + self.assertIn('kind_phys', content) + self.assertIn('REAL64', content) + self.assertTrue(content.endswith('\n')) + + def test_creates_output_dir(self): + with tempfile.TemporaryDirectory() as tmpdir: + subdir = os.path.join(tmpdir, 'new_subdir') + write_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}, subdir) + self.assertTrue(os.path.isdir(subdir)) + + def test_returns_absolute_path(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_ccpp_kinds({'kind_phys': (_ISO, 'REAL64')}, tmpdir) + self.assertTrue(os.path.isabs(path)) + + +def load_tests(loader, tests, ignore): + import generator.kinds_writer as kw + tests.addTests(doctest.DocTestSuite(kw)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_legacy_compat.py b/unit-tests/test_legacy_compat.py new file mode 100644 index 00000000..3b668fd2 --- /dev/null +++ b/unit-tests/test_legacy_compat.py @@ -0,0 +1,280 @@ +"""Tests for the transient legacy-mode shim. + +This whole file is part of the legacy-compat migration shim and should +be deleted alongside ``metadata/legacy_compat.py`` when the migration +is complete. Search ``legacy-compat`` to find every touchpoint. +""" + +from __future__ import annotations + +import doctest +import io +import logging +import os +import sys +import unittest + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen-ng') +if _CAPGEN_DIR not in sys.path: + sys.path.insert(0, _CAPGEN_DIR) + +from metadata import legacy_compat # noqa: E402 +from metadata.metadata_table import ( # noqa: E402 + MetaVar, _parse_dimensions, _parse_lines, +) +from metadata.parse_tools import CCPPError, ParseContext # noqa: E402 + + +def _ctx(): + return ParseContext(linenum=1, filename='legacy_compat_test.meta') + + +class _LegacyModeFixture(unittest.TestCase): + """Mixin that flips legacy mode on for the duration of a test and + guarantees it goes back off afterwards (the flag is process state). + """ + + def setUp(self): + # Sanity: never start a test with the flag set by an earlier + # test that crashed before cleanup. + legacy_compat.disable() + + def tearDown(self): + legacy_compat.disable() + + +class TestTranslateOff(_LegacyModeFixture): + """When legacy mode is disabled, ``translate`` is a strict identity.""" + + def test_identity_on_legacy_name_when_disabled(self): + self.assertFalse(legacy_compat.is_enabled()) + self.assertEqual( + legacy_compat.translate('horizontal_loop_extent'), + 'horizontal_loop_extent', + ) + + def test_identity_on_unknown_name(self): + self.assertEqual( + legacy_compat.translate('air_temperature'), 'air_temperature', + ) + + +class TestEnableDisable(_LegacyModeFixture): + + def test_enable_flips_flag_and_writes_banner(self): + sink = io.StringIO() + legacy_compat.enable(_stream=sink) + self.assertTrue(legacy_compat.is_enabled()) + out = sink.getvalue() + # Bold banner: starred border, the deprecated name, and the + # canonical replacement all appear. + self.assertIn('LEGACY-MODE ENABLED', out) + self.assertIn('horizontal_loop_extent', out) + self.assertIn('horizontal_dimension', out) + self.assertIn('TRANSIENT', out) + self.assertIn('REMOVED', out) + self.assertGreaterEqual(out.count('*' * 10), 2) + + def test_enable_is_idempotent(self): + sink1 = io.StringIO() + legacy_compat.enable(_stream=sink1) + first = sink1.getvalue() + sink2 = io.StringIO() + legacy_compat.enable(_stream=sink2) # second call — no banner + self.assertEqual(sink2.getvalue(), '') + self.assertIn('LEGACY-MODE', first) + + def test_disable_resets(self): + legacy_compat.enable(_stream=io.StringIO()) + self.assertTrue(legacy_compat.is_enabled()) + legacy_compat.disable() + self.assertFalse(legacy_compat.is_enabled()) + + def test_logger_receives_warning(self): + logger = logging.getLogger('legacy_compat_test_logger') + records = [] + + class _Capture(logging.Handler): + def emit(self, record): + records.append(record) + + handler = _Capture(level=logging.WARNING) + logger.addHandler(handler) + try: + legacy_compat.enable(logger=logger, _stream=io.StringIO()) + self.assertEqual(len(records), 1) + self.assertEqual(records[0].levelno, logging.WARNING) + self.assertIn('horizontal_loop_extent', records[0].getMessage()) + finally: + logger.removeHandler(handler) + + +class TestTranslateOn(_LegacyModeFixture): + """When legacy mode is enabled, the documented map applies.""" + + def setUp(self): + super().setUp() + legacy_compat.enable(_stream=io.StringIO()) + + def test_horizontal_loop_extent_rewritten(self): + self.assertEqual( + legacy_compat.translate('horizontal_loop_extent'), + 'horizontal_dimension', + ) + + def test_unknown_name_passes_through(self): + self.assertEqual( + legacy_compat.translate('air_temperature'), 'air_temperature', + ) + + def test_uppercase_legacy_passes_through_translate(self): + # ``translate`` itself is case-sensitive — case folding is the + # caller's responsibility (it happens upstream in + # ``check_cf_standard_name``). This pins that contract so a + # future refactor doesn't accidentally widen it. + self.assertEqual( + legacy_compat.translate('Horizontal_Loop_Extent'), + 'Horizontal_Loop_Extent', + ) + + +######################################################################## +# Integration through metadata_table hook points +######################################################################## + +class TestMetaVarStandardNameHook(_LegacyModeFixture): + """``MetaVar.set_attr('standard_name', ...)`` runs the value through + ``check_cf_standard_name`` (which lowercases) AND through + ``legacy_compat.translate``. When legacy mode is enabled, a scheme + declaring ``standard_name = horizontal_loop_extent`` ends up with + ``standard_name == 'horizontal_dimension'``.""" + + def _make_var(self, std_name_value): + ctx = _ctx() + v = MetaVar('foo', ctx) + v.set_attr('standard_name', std_name_value, ctx) + return v + + def test_disabled_keeps_legacy_name(self): + # Default mode: the parser accepts the legacy name (it's a + # valid CF identifier) but does NOT rewrite it. Downstream + # consumers reject it just like they would in a non-legacy + # build. + v = self._make_var('horizontal_loop_extent') + self.assertEqual(v.standard_name, 'horizontal_loop_extent') + + def test_enabled_rewrites_lowercase_legacy_name(self): + legacy_compat.enable(_stream=io.StringIO()) + v = self._make_var('horizontal_loop_extent') + self.assertEqual(v.standard_name, 'horizontal_dimension') + + def test_enabled_rewrites_mixedcase_legacy_name(self): + # check_cf_standard_name lowercases first; translate runs + # against the lowercase form. Mixed-case legacy spellings + # therefore still get rewritten. + legacy_compat.enable(_stream=io.StringIO()) + v = self._make_var('Horizontal_Loop_Extent') + self.assertEqual(v.standard_name, 'horizontal_dimension') + + def test_enabled_leaves_non_legacy_unchanged(self): + legacy_compat.enable(_stream=io.StringIO()) + v = self._make_var('air_temperature') + self.assertEqual(v.standard_name, 'air_temperature') + + +class TestDimensionHook(_LegacyModeFixture): + """``_parse_dimensions`` runs each non-integer token through + ``legacy_compat.translate`` after lowercasing.""" + + def test_disabled_keeps_legacy_dim_token(self): + dims = _parse_dimensions( + '(horizontal_loop_extent, vertical_layer_dimension)', _ctx(), + ) + self.assertEqual(dims, + ['horizontal_loop_extent', 'vertical_layer_dimension']) + + def test_enabled_rewrites_legacy_dim_token(self): + legacy_compat.enable(_stream=io.StringIO()) + dims = _parse_dimensions( + '(horizontal_loop_extent, vertical_layer_dimension)', _ctx(), + ) + self.assertEqual(dims, + ['horizontal_dimension', 'vertical_layer_dimension']) + + def test_enabled_rewrites_inside_range_form(self): + legacy_compat.enable(_stream=io.StringIO()) + dims = _parse_dimensions( + '(ccpp_constant_one:horizontal_loop_extent)', _ctx(), + ) + self.assertEqual(dims, ['ccpp_constant_one:horizontal_dimension']) + + def test_enabled_passes_integer_literals_through(self): + legacy_compat.enable(_stream=io.StringIO()) + dims = _parse_dimensions('(1:8)', _ctx()) + self.assertEqual(dims, ['1:8']) + + def test_enabled_mixed_case_legacy_in_dim(self): + legacy_compat.enable(_stream=io.StringIO()) + dims = _parse_dimensions('(Horizontal_Loop_Extent)', _ctx()) + self.assertEqual(dims, ['horizontal_dimension']) + + +class TestEndToEndMetadataParse(_LegacyModeFixture): + """Full parse of a small scheme metadata snippet via ``_parse_lines`` + confirms that the legacy name flows through both hook points + (standard_name on a scalar arg + dim token on an array arg).""" + + _META = ( + '[ccpp-table-properties]\n' + ' name = legacy_test_scheme\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = legacy_test_scheme_run\n' + ' type = scheme\n' + '[ ncols ]\n' + ' standard_name = horizontal_loop_extent\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = in\n' + '[ t ]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_loop_extent, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = inout\n' + ) + + def _parse_and_get_section(self): + tables = _parse_lines(self._META.splitlines(keepends=True), 't.meta') + return tables[0].sections()[0] + + def test_legacy_off_preserves_legacy_name(self): + section = self._parse_and_get_section() + std_names = [v.standard_name for v in section.variables] + self.assertIn('horizontal_loop_extent', std_names) + dim_tokens = section.variables[1].dimensions + self.assertEqual(dim_tokens[0], 'horizontal_loop_extent') + + def test_legacy_on_rewrites_both_sites(self): + legacy_compat.enable(_stream=io.StringIO()) + section = self._parse_and_get_section() + std_names = [v.standard_name for v in section.variables] + self.assertIn('horizontal_dimension', std_names) + self.assertNotIn('horizontal_loop_extent', std_names) + dim_tokens = section.variables[1].dimensions + self.assertEqual(dim_tokens[0], 'horizontal_dimension') + + +######################################################################## +# Doctest loader for legacy_compat module +######################################################################## + +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(legacy_compat)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_metadata_table.py b/unit-tests/test_metadata_table.py new file mode 100644 index 00000000..b4aa8000 --- /dev/null +++ b/unit-tests/test_metadata_table.py @@ -0,0 +1,1997 @@ +#!/usr/bin/env python3 + +"""Unit tests for :mod:`metadata.metadata_table`. + +Run with:: + + python -m pytest capgen-ng/tests/test_metadata_table.py -v + +or:: + + python -m unittest capgen-ng.tests.test_metadata_table + +All test methods follow the ``test_`` naming convention and are +documented inline to explain both what is being tested and *why* it matters +for the redesigned generator. +""" + +import os +import sys +import textwrap +import tempfile +import unittest + +# ---- locate the package root ----------------------------------------------- +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_PKG_ROOT = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen-ng') +if _PKG_ROOT not in sys.path: + sys.path.insert(0, _PKG_ROOT) + +# ---- imports from the package ----------------------------------------------- +from metadata.parse_tools import CCPPError, ParseSyntaxError +from metadata.metadata_table import ( + MetaVar, + MetadataSection, + MetadataTable, + parse_metadata_file, + _parse_lines, + VALID_TABLE_TYPES, + VALID_SCHEME_PHASES, + VALID_INTENTS, + SCHEME_TABLE_TYPE, + SINGLETON_TABLE_TYPES, + _is_blank, + _parse_bool, + _parse_dimensions, + _check_var_type, + _parse_config_line, +) +from metadata.parse_tools.parse_source import ParseContext + +# ---- sample file directory -------------------------------------------------- +_SAMPLE_DIR = os.path.join(_TESTS_DIR, 'sample_files') + + +######################################################################## +# Helper utilities +######################################################################## + +def _ctx(lineno=0): + """Return a minimal :class:`ParseContext` for testing.""" + return ParseContext(linenum=lineno, filename='test_file.meta') + + +def _parse_text(text: str): + """Parse a metadata string and return the list of MetadataTable objects.""" + lines = textwrap.dedent(text).splitlines(keepends=True) + return _parse_lines(lines, '') + + +######################################################################## +# Helper function tests +######################################################################## + +class TestIsBlank(unittest.TestCase): + """Tests for :func:`_is_blank`.""" + + def test_empty_string(self): + self.assertTrue(_is_blank('')) + + def test_whitespace_only(self): + self.assertTrue(_is_blank(' \t ')) + + def test_hash_comment(self): + self.assertTrue(_is_blank('# this is a comment')) + + def test_semicolon_comment(self): + self.assertTrue(_is_blank('; comment')) + + def test_hash_with_leading_spaces(self): + self.assertTrue(_is_blank(' # indented comment')) + + def test_real_content(self): + self.assertFalse(_is_blank('name = foo')) + + def test_bracket_header(self): + self.assertFalse(_is_blank('[ccpp-table-properties]')) + + +class TestParseBool(unittest.TestCase): + """Tests for :func:`_parse_bool`.""" + + def test_true_variants(self): + ctx = _ctx() + for val in ('True', 'true', 'TRUE', '.true.', '.TRUE.', 't', '1'): + with self.subTest(val=val): + self.assertTrue(_parse_bool(val, ctx)) + + def test_false_variants(self): + ctx = _ctx() + for val in ('False', 'false', 'FALSE', '.false.', '.FALSE.', 'f', '0'): + with self.subTest(val=val): + self.assertFalse(_parse_bool(val, ctx)) + + def test_invalid(self): + ctx = _ctx() + with self.assertRaises(CCPPError): + _parse_bool('yes', ctx) + + +class TestParseDimensions(unittest.TestCase): + """Tests for :func:`_parse_dimensions`.""" + + def test_scalar(self): + self.assertEqual(_parse_dimensions('()', _ctx()), []) + + def test_one_dim(self): + self.assertEqual( + _parse_dimensions('(horizontal_dimension)', _ctx()), + ['horizontal_dimension'] + ) + + def test_two_dims(self): + result = _parse_dimensions( + '(horizontal_dimension, vertical_layer_dimension)', _ctx() + ) + self.assertEqual(result, ['horizontal_dimension', 'vertical_layer_dimension']) + + def test_whitespace_inside(self): + result = _parse_dimensions(' ( dim1 , dim2 ) ', _ctx()) + self.assertEqual(result, ['dim1', 'dim2']) + + def test_missing_parens(self): + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_dimensions('dim1, dim2', _ctx()) + + def test_empty_entry(self): + """An empty entry like ``(,dim2)`` must be rejected.""" + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_dimensions('(, dim2)', _ctx()) + + def test_mixed_case_normalised_to_lower(self): + """CCPP standard names are case-insensitive; the parser must + normalise dimension tokens to lower-case so downstream + host_dict lookups (which store std names lower-cased per + :func:`check_cf_standard_name`) succeed.""" + result = _parse_dimensions( + '(number_of_aerosol_tracers_MG)', _ctx(), + ) + self.assertEqual(result, ['number_of_aerosol_tracers_mg']) + + def test_mixed_case_in_range_normalised(self): + """Range form ``lower:upper`` lowercases both halves.""" + result = _parse_dimensions( + '(ccpp_constant_one:Vertical_LAYER_dimension)', _ctx(), + ) + self.assertEqual( + result, ['ccpp_constant_one:vertical_layer_dimension'], + ) + + def test_integer_literal_passes_through(self): + """Integer-literal dim entries (used in DDT field shapes) are + unaffected by the lower-case normalisation.""" + result = _parse_dimensions('(8)', _ctx()) + self.assertEqual(result, ['8']) + + +class TestCheckVarType(unittest.TestCase): + """Tests for :func:`_check_var_type`.""" + + def test_intrinsic_types(self): + for t in ('real', 'integer', 'logical', 'character', 'complex'): + with self.subTest(t=t): + self.assertEqual(_check_var_type(t, _ctx()), t) + + def test_ddt_identifier(self): + self.assertEqual(_check_var_type('gfs_statein_type', _ctx()), + 'gfs_statein_type') + + def test_type_parens_form(self): + result = _check_var_type('type(my_ddt)', _ctx()) + self.assertEqual(result, 'type(my_ddt)') + + def test_external_type(self): + result = _check_var_type('external:mpi_f08:mpi_comm', _ctx()) + self.assertEqual(result, 'external:mpi_f08:mpi_comm') + + def test_invalid_type(self): + with self.assertRaises((CCPPError, ParseSyntaxError)): + _check_var_type('123invalid', _ctx()) + + +class TestParseConfigLine(unittest.TestCase): + """Tests for :func:`_parse_config_line`.""" + + def test_simple_pair(self): + result = _parse_config_line(' name = foo ', _ctx()) + self.assertEqual(result, [('name', 'foo')]) + + def test_pipe_separator(self): + result = _parse_config_line('units = 1 | dimensions = ()', _ctx()) + self.assertEqual(result, [('units', '1'), ('dimensions', '()')]) + + def test_blank_line(self): + self.assertEqual(_parse_config_line('', _ctx()), []) + + def test_comment_line(self): + self.assertEqual(_parse_config_line('# comment', _ctx()), []) + + def test_bad_line(self): + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_config_line('no_equals_sign', _ctx()) + + +######################################################################## +# MetaVar tests +######################################################################## + +class TestMetaVar(unittest.TestCase): + """Tests for :class:`MetaVar`.""" + + def _make_var(self, local_name='my_var', **attrs): + """Build a MetaVar with sensible defaults plus any overrides.""" + ctx = _ctx() + var = MetaVar(local_name, ctx) + defaults = { + 'standard_name': 'my_standard_name', + 'units': '1', + 'dimensions': '()', + 'type': 'integer', + } + defaults.update(attrs) + for k, v in defaults.items(): + var.set_attr(k, v, ctx) + return var + + def test_creation(self): + var = self._make_var() + self.assertEqual(var.local_name, 'my_var') + self.assertEqual(var.standard_name, 'my_standard_name') + + def test_standard_name_lowercased(self): + """Standard names must be CF names (lowercased by the checker).""" + ctx = _ctx() + var = MetaVar('v', ctx) + var.set_attr('standard_name', 'Horizontal_Loop_Extent', ctx) + self.assertEqual(var.standard_name, 'horizontal_loop_extent') + + def test_dimensions_scalar(self): + var = self._make_var(dimensions='()') + self.assertEqual(var.dimensions, []) + + def test_dimensions_array(self): + var = self._make_var(dimensions='(horizontal_dimension, vertical_layer_dimension)') + self.assertEqual(var.dimensions, + ['horizontal_dimension', 'vertical_layer_dimension']) + + def test_intent_valid(self): + ctx = _ctx() + for intent in ('in', 'out', 'inout'): + with self.subTest(intent=intent): + var = self._make_var() + var.set_attr('intent', intent, ctx) + self.assertEqual(var.intent, intent) + + def test_intent_invalid(self): + ctx = _ctx() + var = self._make_var() + with self.assertRaises(CCPPError): + var.set_attr('intent', 'banana', ctx) + + def test_protected_bool(self): + var = self._make_var(protected='True') + self.assertTrue(var.protected) + + def test_optional_bool(self): + var = self._make_var(optional='False') + self.assertFalse(var.optional) + + def test_local_name_accepts_long_subscript_index(self): + """Sliced local names like ``dqdt(:,:,index_of_)`` + carry CCPP standard names as subscript tokens; those routinely + exceed the 63-char Fortran identifier limit. Only the base + identifier (``dqdt``) needs to fit the limit — the subscript is + a standard-name reference resolved separately. + """ + long_std = ('index_of_cloud_liquid_water_mixing_ratio_' + 'in_tracer_concentration_array') # 67 chars + self.assertGreater(len(long_std), 63) + long_local = 'dqdt(:,:,{})'.format(long_std) + # Must NOT raise. + var = self._make_var(local_name=long_local) + self.assertEqual(var.local_name, long_local) + + def test_local_name_base_still_length_checked(self): + """The base identifier (everything before the first ``(``) must + still fit the 63-char limit — only subscript tokens are exempt. + """ + from metadata.parse_tools import ParseSyntaxError + long_base = 'a' * 64 # 64 > FORTRAN_MAX_IDENT_LEN + with self.assertRaises(ParseSyntaxError): + MetaVar(long_base, _ctx()) + # Same long base inside a slice — still rejected. + with self.assertRaises(ParseSyntaxError): + MetaVar('{}(:)'.format(long_base), _ctx()) + + def test_local_name_rejects_malformed_reference(self): + """The form check still applies — only the per-token length + limit was relaxed. A malformed reference is still an error.""" + from metadata.parse_tools import ParseSyntaxError + with self.assertRaises(ParseSyntaxError): + MetaVar('not a valid id', _ctx()) + + def test_top_at_one_default_false(self): + var = self._make_var() + self.assertFalse(var.top_at_one) + + def test_top_at_one_true(self): + for value in ('True', '.true.', 'true'): + with self.subTest(value=value): + var = self._make_var(top_at_one=value) + self.assertTrue(var.top_at_one) + + def test_top_at_one_false(self): + for value in ('False', '.false.', 'false'): + with self.subTest(value=value): + var = self._make_var(top_at_one=value) + self.assertFalse(var.top_at_one) + + def test_diagnostic_name_explicit(self): + var = self._make_var(diagnostic_name='temperature') + self.assertEqual(var.diagnostic_name, 'temperature') + self.assertEqual(var.diagnostic_name_fixed, '') + + def test_diagnostic_name_template_accepted(self): + var = self._make_var(diagnostic_name='foo_${scheme_name}') + self.assertEqual(var.diagnostic_name, 'foo_${scheme_name}') + + def test_diagnostic_name_fixed_explicit(self): + var = self._make_var(diagnostic_name_fixed='Q') + self.assertEqual(var.diagnostic_name_fixed, 'Q') + # When _fixed is set, diagnostic_name returns '' (no local_name fallback). + self.assertEqual(var.diagnostic_name, '') + + def test_diagnostic_name_defaults_to_local_name(self): + """Neither attribute set: diagnostic_name defaults to local_name.""" + var = self._make_var(local_name='my_local_var') + self.assertEqual(var.diagnostic_name, 'my_local_var') + self.assertEqual(var.diagnostic_name_fixed, '') + + def test_diagnostic_name_invalid(self): + ctx = _ctx() + var = self._make_var() + with self.assertRaises(CCPPError): + var.set_attr('diagnostic_name', 'pref_${scheme}_suff', ctx) + + def test_diagnostic_name_fixed_invalid(self): + ctx = _ctx() + var = self._make_var() + with self.assertRaises(CCPPError): + var.set_attr('diagnostic_name_fixed', '2bad', ctx) + + def test_diagnostic_name_mutually_exclusive(self): + """Setting both diagnostic_name and diagnostic_name_fixed is an error.""" + ctx = _ctx() + var = self._make_var(diagnostic_name='temperature') + with self.assertRaises(CCPPError): + var.set_attr('diagnostic_name_fixed', 'Q', ctx) + + def test_diagnostic_name_fixed_then_name_mutually_exclusive(self): + """Order independence: fixed first, then name, also rejected.""" + ctx = _ctx() + var = self._make_var(diagnostic_name_fixed='Q') + with self.assertRaises(CCPPError): + var.set_attr('diagnostic_name', 'temperature', ctx) + + def test_duplicate_attr(self): + """Setting the same attribute twice is an error.""" + ctx = _ctx() + var = MetaVar('v', ctx) + var.set_attr('standard_name', 'foo', ctx) + with self.assertRaises(CCPPError): + var.set_attr('standard_name', 'bar', ctx) + + def test_unknown_attr(self): + ctx = _ctx() + var = MetaVar('v', ctx) + with self.assertRaises(CCPPError): + var.set_attr('banana', 'value', ctx) + + def test_validate_requires_intent_for_scheme(self): + """Scheme variables need intent; missing it must raise CCPPError.""" + ctx = _ctx() + var = self._make_var() # no intent set + with self.assertRaises(CCPPError): + var.validate(require_intent=True, context=ctx) + + def test_validate_no_intent_for_host(self): + """Host variables do not need intent; validate must pass without it.""" + ctx = _ctx() + var = self._make_var() + var.validate(require_intent=False, context=ctx) # should not raise + + def test_validate_missing_units_defaults_to_none(self): + """Omitting ``units`` is allowed; it defaults to ``'none'``.""" + ctx = _ctx() + var = MetaVar('v', ctx) + var.set_attr('standard_name', 'foo', ctx) + var.set_attr('dimensions', '()', ctx) + var.set_attr('type', 'integer', ctx) + var.validate(require_intent=False, context=ctx) # should not raise + self.assertEqual(var.units, 'none') + + def test_external_ddt_helpers(self): + var = self._make_var(type='external:mpi_f08:mpi_comm') + self.assertTrue(var.is_external_ddt()) + self.assertEqual(var.external_ddt_module(), 'mpi_f08') + self.assertEqual(var.external_ddt_typename(), 'mpi_comm') + + def test_non_external_ddt_helpers(self): + var = self._make_var(type='real') + self.assertFalse(var.is_external_ddt()) + self.assertIsNone(var.external_ddt_module()) + self.assertIsNone(var.external_ddt_typename()) + + def test_invalid_local_name(self): + """Local name must be a valid Fortran identifier.""" + with self.assertRaises((CCPPError, ParseSyntaxError)): + MetaVar('123bad', _ctx()) + + def test_local_name_too_long(self): + """Local name must not exceed 63 characters (Fortran limit).""" + long_name = 'a' * 64 + with self.assertRaises((CCPPError, ParseSyntaxError)): + MetaVar(long_name, _ctx()) + + +######################################################################## +# MetadataSection tests +######################################################################## + +class TestMetadataSection(unittest.TestCase): + """Tests for :class:`MetadataSection`.""" + + def _make_scheme_section(self, phase='run', scheme_name='my_scheme'): + ctx = _ctx() + return MetadataSection( + section_name='{}__{}'.format(scheme_name, phase).replace('__', '_'), + section_type='scheme', + table_name=scheme_name, + context=ctx, + ) + + def test_scheme_phase_extraction(self): + for phase in VALID_SCHEME_PHASES: + with self.subTest(phase=phase): + sec = self._make_scheme_section(phase=phase) + self.assertEqual(sec.phase, phase) + + def test_finalize_rejected(self): + """'finalize' was renamed to 'final'; the old name must be rejected.""" + ctx = _ctx() + with self.assertRaises(CCPPError) as cm: + MetadataSection('my_scheme_finalize', 'scheme', 'my_scheme', ctx) + self.assertIn('final', str(cm.exception)) + + def test_host_section_has_no_phase(self): + ctx = _ctx() + sec = MetadataSection('physics_data', 'host', 'physics_data', ctx) + self.assertIsNone(sec.phase) + + def test_duplicate_standard_name(self): + """Adding two variables with the same standard name must raise.""" + ctx = _ctx() + sec = MetadataSection('my_scheme_run', 'scheme', 'my_scheme', ctx) + var1 = MetaVar('a_var', ctx) + for attr, val in [('standard_name', 'foo'), ('units', '1'), + ('dimensions', '()'), ('type', 'integer'), + ('intent', 'in')]: + var1.set_attr(attr, val, ctx) + var1.validate(require_intent=True, context=ctx) + sec.add_variable(var1) + + var2 = MetaVar('b_var', ctx) + for attr, val in [('standard_name', 'foo'), ('units', '1'), + ('dimensions', '()'), ('type', 'integer'), + ('intent', 'in')]: + var2.set_attr(attr, val, ctx) + var2.validate(require_intent=True, context=ctx) + with self.assertRaises(CCPPError): + sec.add_variable(var2) + + def test_invalid_table_type(self): + ctx = _ctx() + with self.assertRaises(CCPPError): + MetadataSection('t', 'banana', 't', ctx) + + def test_scheme_name_mismatch(self): + """Section name not starting with scheme name must raise.""" + ctx = _ctx() + with self.assertRaises(CCPPError): + MetadataSection('other_scheme_run', 'scheme', 'my_scheme', ctx) + + +######################################################################## +# MetadataTable tests +######################################################################## + +class TestMetadataTable(unittest.TestCase): + """Tests for :class:`MetadataTable`.""" + + def test_valid_types(self): + for ttype in VALID_TABLE_TYPES: + with self.subTest(ttype=ttype): + ctx = _ctx() + tbl = MetadataTable('tbl', ttype, 'f.meta', ctx) + self.assertEqual(tbl.table_type, ttype) + + def test_invalid_type(self): + ctx = _ctx() + with self.assertRaises(CCPPError): + MetadataTable('t', 'module', 'f.meta', ctx) + + def test_is_scheme(self): + ctx = _ctx() + self.assertTrue(MetadataTable('s', 'scheme', 'f.meta', ctx).is_scheme) + self.assertFalse(MetadataTable('h', 'host', 'f.meta', ctx).is_scheme) + + def test_singleton_allows_one_section(self): + """Singleton table types (host, control, ddt, suite) allow only one section.""" + for ttype in SINGLETON_TABLE_TYPES: + with self.subTest(ttype=ttype): + ctx = _ctx() + tbl = MetadataTable('t', ttype, 'f.meta', ctx) + sec1 = MetadataSection('t', ttype, 't', ctx) + tbl.add_section(sec1) + sec2 = MetadataSection('t', ttype, 't', ctx) + with self.assertRaises(CCPPError): + tbl.add_section(sec2) + + def test_scheme_allows_multiple_sections(self): + """Scheme tables allow one section per phase.""" + ctx = _ctx() + tbl = MetadataTable('s', 'scheme', 'f.meta', ctx) + for phase in ('init', 'run', 'final'): + sec = MetadataSection('s_{}'.format(phase), 'scheme', 's', ctx) + tbl.add_section(sec) + self.assertEqual(len(tbl.sections()), 3) + + def test_section_type_mismatch(self): + """Section type must match table type.""" + ctx = _ctx() + tbl = MetadataTable('my_host', 'host', 'f.meta', ctx) + # The section name must be valid for scheme (scheme_name_phase); + # use a different scheme name so MetadataSection construction succeeds, + # and the CCPPError is raised only at add_section() due to type mismatch. + sec = MetadataSection('some_scheme_run', 'scheme', 'some_scheme', ctx) + with self.assertRaises(CCPPError): + tbl.add_section(sec) + + def test_section_for_phase(self): + ctx = _ctx() + tbl = MetadataTable('s', 'scheme', 'f.meta', ctx) + run_sec = MetadataSection('s_run', 'scheme', 's', ctx) + tbl.add_section(run_sec) + self.assertIs(tbl.section_for_phase('run'), run_sec) + self.assertIsNone(tbl.section_for_phase('init')) + + +######################################################################## +# parse_metadata_file / _parse_lines tests +######################################################################## + +class TestParseLines(unittest.TestCase): + """Tests for the actual ini-file parser via :func:`_parse_lines`.""" + + # ---- valid cases ------------------------------------------------------- + + def test_host_table(self): + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + tbl = tables[0] + self.assertEqual(tbl.table_name, 'my_host') + self.assertEqual(tbl.table_type, 'host') + self.assertEqual(len(tbl.sections()), 1) + sec = tbl.sections()[0] + self.assertEqual(len(sec.variables), 1) + var = sec.variables[0] + self.assertEqual(var.local_name, 'im') + self.assertEqual(var.standard_name, 'horizontal_dimension') + self.assertEqual(var.dimensions, []) + self.assertIsNone(var.intent) + + def test_scheme_three_phases(self): + text = """ + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_init + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ im ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer + intent = in + + [ccpp-arg-table] + name = my_scheme_final + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + tbl = tables[0] + self.assertTrue(tbl.is_scheme) + self.assertEqual(len(tbl.sections()), 3) + phases = {sec.phase for sec in tbl.sections()} + self.assertEqual(phases, {'init', 'run', 'final'}) + + def test_two_tables_in_one_file(self): + """A .meta file may contain one DDT table followed by a scheme table.""" + text = """ + [ccpp-table-properties] + name = my_ddt + type = ddt + + [ccpp-arg-table] + name = my_ddt + type = ddt + [ field1 ] + standard_name = some_field + units = 1 + dimensions = () + type = real + + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ im ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer + intent = in + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 2) + self.assertEqual(tables[0].table_type, 'ddt') + self.assertEqual(tables[1].table_type, 'scheme') + + def test_pipe_separated_attributes(self): + """Multiple attributes on one line with ``|`` separator.""" + text = """ + [ccpp-table-properties] + name = s + type = scheme + + [ccpp-arg-table] + name = s_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none | dimensions = () | type = character | kind = len=512 + intent = out + """ + tables = _parse_text(text) + var = tables[0].sections()[0].variables[0] + self.assertEqual(var.units, 'none') + self.assertEqual(var.dimensions, []) + self.assertEqual(var.type, 'character') + self.assertEqual(var.kind, 'len=512') + self.assertEqual(var.intent, 'out') + + def test_control_table(self): + text = """ + [ccpp-table-properties] + name = ctrl + type = control + + [ccpp-arg-table] + name = ctrl + type = control + [ tnum ] + standard_name = thread_number + units = 1 + dimensions = () + type = integer + """ + tables = _parse_text(text) + self.assertEqual(tables[0].table_type, 'control') + + def test_ddt_instance_in_host_table(self): + """Host table declaring a DDT instance variable (array of DDT).""" + text = """ + [ccpp-table-properties] + name = CCPP_data + type = host + + [ccpp-arg-table] + name = CCPP_data + type = host + [ gfs_statein ] + standard_name = gfs_statein + units = none + dimensions = (number_of_instances) + type = gfs_statein_type + """ + tables = _parse_text(text) + var = tables[0].sections()[0].variables[0] + self.assertEqual(var.type, 'gfs_statein_type') + self.assertEqual(var.dimensions, ['number_of_instances']) + + def test_comments_and_blank_lines_ignored(self): + """Comment lines (``#``) and blank lines must be skipped silently.""" + text = """ + # This is a comment + + [ccpp-table-properties] + name = h + type = host + + ######################################## + [ccpp-arg-table] + # section comment + name = h + type = host + ; semicolon comment + [ x ] + standard_name = some_var + units = 1 + dimensions = () + type = integer + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + self.assertEqual(len(tables[0].sections()[0].variables), 1) + + # ---- error cases ------------------------------------------------------- + + def test_module_type_rejected(self): + """``type = module`` must produce a CCPPError mentioning 'host'.""" + text = """ + [ccpp-table-properties] + name = physics_mod + type = module + """ + with self.assertRaises(CCPPError) as cm: + _parse_text(text) + self.assertIn('host', str(cm.exception).lower()) + + def test_banana_type_rejected(self): + """An unknown table type must raise CCPPError.""" + text = """ + [ccpp-table-properties] + name = bad + type = banana + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_finalize_phase_rejected(self): + """``_finalize`` phase name must raise CCPPError mentioning 'final'.""" + text = """ + [ccpp-table-properties] + name = s + type = scheme + + [ccpp-arg-table] + name = s_finalize + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + intent = out + """ + with self.assertRaises(CCPPError) as cm: + _parse_text(text) + self.assertIn('final', str(cm.exception)) + + def test_duplicate_standard_name_in_section(self): + """Two variables with the same standard name in one section must raise.""" + text = """ + [ccpp-table-properties] + name = s + type = scheme + + [ccpp-arg-table] + name = s_run + type = scheme + [ a ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer + intent = in + [ b ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer + intent = in + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_missing_intent_for_scheme_variable(self): + """Scheme variables without ``intent`` must raise at validation time.""" + text = """ + [ccpp-table-properties] + name = s + type = scheme + + [ccpp-arg-table] + name = s_run + type = scheme + [ im ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_invalid_intent_value(self): + text = """ + [ccpp-table-properties] + name = s + type = scheme + + [ccpp-arg-table] + name = s_run + type = scheme + [ im ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer + intent = banana + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_bad_dimensions(self): + """Dimensions not enclosed in parentheses must raise.""" + text = """ + [ccpp-table-properties] + name = h + type = host + + [ccpp-arg-table] + name = h + type = host + [ x ] + standard_name = foo + units = 1 + dimensions = no_parens + type = integer + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_missing_required_attribute(self): + """Missing ``type`` must raise at variable validation.""" + text = """ + [ccpp-table-properties] + name = h + type = host + + [ccpp-arg-table] + name = h + type = host + [ x ] + standard_name = foo + units = 1 + dimensions = () + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_missing_units_defaults_to_none(self): + """Omitting ``units`` is accepted; the variable's units default to ``'none'``.""" + text = """ + [ccpp-table-properties] + name = h + type = host + + [ccpp-arg-table] + name = h + type = host + [ x ] + standard_name = foo + dimensions = () + type = integer + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + section = tables[0].sections()[0] + var = section.variables[0] + self.assertEqual(var.units, 'none') + + def test_section_type_mismatch_raises(self): + """Section type different from table type must raise.""" + text = """ + [ccpp-table-properties] + name = h + type = host + + [ccpp-arg-table] + name = h + type = scheme + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + def test_singleton_table_second_section_raises(self): + """A ``host`` table with two sections must raise.""" + text = """ + [ccpp-table-properties] + name = h + type = host + + [ccpp-arg-table] + name = h + type = host + + [ccpp-arg-table] + name = h + type = host + """ + with self.assertRaises(CCPPError): + _parse_text(text) + + +######################################################################## +# Attribute ownership enforcement tests +######################################################################## + +class TestAttributeOwnership(unittest.TestCase): + """Parse-time rejection of active/optional in wrong table types.""" + + def test_active_in_scheme_raises(self): + """'active' attribute in scheme metadata must raise ParseSyntaxError.""" + text = """ + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + active = .true. + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_optional_in_host_raises(self): + """'optional' attribute in host metadata must raise ParseSyntaxError.""" + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + optional = true + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_optional_in_control_raises(self): + """'optional' attribute in control metadata must raise ParseSyntaxError.""" + text = """ + [ccpp-table-properties] + name = my_ctrl + type = control + + [ccpp-arg-table] + name = my_ctrl + type = control + [ x ] + standard_name = foo + units = 1 + dimensions = () + type = integer + optional = true + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_optional_in_ddt_raises(self): + """'optional' attribute in ddt metadata must raise ParseSyntaxError.""" + text = """ + [ccpp-table-properties] + name = my_ddt_type + type = ddt + + [ccpp-arg-table] + name = my_ddt_type + type = ddt + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + optional = false + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_active_in_host_allowed(self): + """'active' attribute in host metadata is valid.""" + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ flag ] + standard_name = my_flag + units = flag + dimensions = () + type = logical + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + active = my_flag + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + + def test_optional_in_scheme_allowed(self): + """'optional' attribute in scheme metadata is valid.""" + text = """ + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + intent = in + optional = true + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + + def test_constituent_in_host_raises(self): + """'constituent' attribute in host metadata must raise.""" + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + constituent = true + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_advected_in_host_raises(self): + """'advected' attribute in host metadata must raise.""" + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + advected = .true. + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_molar_mass_in_ddt_raises(self): + """'molar_mass' attribute in ddt metadata must raise.""" + text = """ + [ccpp-table-properties] + name = my_ddt_type + type = ddt + + [ccpp-arg-table] + name = my_ddt_type + type = ddt + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + molar_mass = 18.0 + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + +######################################################################## +# Constituent attribute tests (scheme metadata only) +######################################################################## + +class TestConstituentAttributes(unittest.TestCase): + """Parsing and is_constituent rollup for scheme-only constituent hints.""" + + def _scheme_var(self, *, extra_attrs: str = '') -> MetaVar: + text = """ + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ x ] + standard_name = foo + units = kg kg-1 + dimensions = () + type = real + intent = inout + {extra} + """.format(extra=extra_attrs) + tables = _parse_text(text) + return tables[0].sections()[0].variables[0] + + def test_constituent_true_accepted(self): + var = self._scheme_var(extra_attrs='constituent = True') + self.assertTrue(var.constituent) + self.assertTrue(var.is_constituent) + + def test_advected_dot_true_accepted(self): + var = self._scheme_var(extra_attrs='advected = .true.') + self.assertTrue(var.advected) + self.assertTrue(var.is_constituent) + + def test_molar_mass_accepted(self): + var = self._scheme_var(extra_attrs='molar_mass = 18.0') + self.assertEqual(var.molar_mass, 18.0) + self.assertTrue(var.is_constituent) + + def test_defaults_make_non_constituent(self): + var = self._scheme_var() + self.assertFalse(var.constituent) + self.assertFalse(var.advected) + self.assertEqual(var.molar_mass, 0.0) + self.assertFalse(var.is_constituent) + + def test_negative_molar_mass_rejected(self): + with self.assertRaises((CCPPError, ParseSyntaxError)): + self._scheme_var(extra_attrs='molar_mass = -1.0') + + +######################################################################## +# File-based tests (use sample_files/) +######################################################################## + +class TestParseMetadataFiles(unittest.TestCase): + """Integration tests using real ``.meta`` files in ``sample_files/``.""" + + def _sample(self, name): + return os.path.join(_SAMPLE_DIR, name) + + # ---- valid files ------------------------------------------------------- + + def test_host_simple_file(self): + tables = parse_metadata_file(self._sample('host_simple.meta')) + self.assertEqual(len(tables), 1) + tbl = tables[0] + self.assertEqual(tbl.table_name, 'physics_data') + self.assertEqual(tbl.table_type, 'host') + snames = {v.standard_name for v in tbl.variables()} + self.assertIn('horizontal_dimension', snames) + self.assertIn('vertical_layer_dimension', snames) + # loop bounds and error vars are control vars (control_simple.meta), not host vars + self.assertNotIn('horizontal_loop_begin', snames) + self.assertNotIn('horizontal_loop_end', snames) + + def test_control_simple_file(self): + tables = parse_metadata_file(self._sample('control_simple.meta')) + self.assertEqual(len(tables), 1) + self.assertEqual(tables[0].table_type, 'control') + + def test_ddt_simple_file(self): + tables = parse_metadata_file(self._sample('ddt_simple.meta')) + self.assertEqual(len(tables), 1) + tbl = tables[0] + self.assertEqual(tbl.table_type, 'ddt') + snames = [v.standard_name for v in tbl.variables()] + self.assertIn('geopotential_at_interface', snames) + self.assertIn('geopotential', snames) + # DDT field variables have no intent + for var in tbl.variables(): + self.assertIsNone(var.intent) + + def test_scheme_multipart_file(self): + """Scheme with init, run, final phases from sample file.""" + tables = parse_metadata_file(self._sample('scheme_multipart.meta')) + self.assertEqual(len(tables), 1) + tbl = tables[0] + self.assertTrue(tbl.is_scheme) + phases = {sec.phase for sec in tbl.sections()} + self.assertEqual(phases, {'init', 'run', 'final'}) + run_sec = tbl.section_for_phase('run') + self.assertIsNotNone(run_sec) + run_stdnames = {v.standard_name for v in run_sec.variables} + self.assertIn('air_temperature', run_stdnames) + + def test_host_with_ddt_instance_file(self): + """Host table declaring a DDT instance variable.""" + tables = parse_metadata_file(self._sample('host_with_ddt_instance.meta')) + self.assertEqual(len(tables), 1) + var = tables[0].sections()[0].variables[0] + self.assertEqual(var.local_name, 'gfs_statein') + self.assertEqual(var.type, 'gfs_statein_type') + self.assertEqual(var.dimensions, ['number_of_instances']) + + # ---- error files ------------------------------------------------------- + + def test_bad_module_type_file(self): + """``type = module`` must raise with a message mentioning 'host'.""" + with self.assertRaises(CCPPError) as cm: + parse_metadata_file(self._sample('bad_module_type.meta')) + self.assertIn('host', str(cm.exception).lower()) + + def test_bad_finalize_phase_file(self): + """``_finalize`` phase must raise with a message mentioning 'final'.""" + with self.assertRaises(CCPPError) as cm: + parse_metadata_file(self._sample('bad_finalize_phase.meta')) + self.assertIn('final', str(cm.exception)) + + def test_bad_invalid_type_file(self): + """Completely invalid table type raises CCPPError.""" + with self.assertRaises(CCPPError): + parse_metadata_file(self._sample('bad_invalid_type.meta')) + + def test_bad_duplicate_stdname_file(self): + """Duplicate standard name in one section raises CCPPError.""" + with self.assertRaises(CCPPError): + parse_metadata_file(self._sample('bad_duplicate_stdname.meta')) + + def test_nonexistent_file(self): + """Parsing a non-existent file raises CCPPError.""" + with self.assertRaises(CCPPError): + parse_metadata_file('/nonexistent/path/file.meta') + + +######################################################################## +# Variables() de-duplication across scheme phases +######################################################################## + +class TestTableVariables(unittest.TestCase): + """Tests for :meth:`MetadataTable.variables` cross-phase de-duplication.""" + + def test_dedup_across_phases(self): + """``ccpp_error_message`` appears in init and run; variables() returns it once.""" + text = """ + [ccpp-table-properties] + name = sch + type = scheme + + [ccpp-arg-table] + name = sch_init + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + intent = out + + [ccpp-arg-table] + name = sch_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + intent = out + [ im ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer + intent = in + """ + tables = _parse_text(text) + all_vars = tables[0].variables() + snames = [v.standard_name for v in all_vars] + # ccpp_error_message appears in both phases but should be returned once + self.assertEqual(snames.count('ccpp_error_message'), 1) + self.assertIn('horizontal_loop_extent', snames) + + +######################################################################## +# CLI helper tests +######################################################################## + +class TestCLIHelpers(unittest.TestCase): + """Tests for CLI utility functions in ccpp_capgen_ng.""" + + def setUp(self): + import importlib + import importlib.util + script = os.path.join(_PKG_ROOT, 'ccpp_capgen_ng.py') + spec = importlib.util.spec_from_file_location('ccpp_capgen_ng', script) + self.mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(self.mod) + + def test_split_file_list(self): + result = self.mod._split_file_list('a.meta, b.meta, c.meta') + self.assertEqual(result, ['a.meta', 'b.meta', 'c.meta']) + + def test_split_file_list_empty(self): + self.assertEqual(self.mod._split_file_list(''), []) + + def test_parse_kind_types_valid_iso_default(self): + result = self.mod._parse_kind_types(['kind_phys=REAL64', 'kind_dyn=REAL32']) + self.assertEqual(result, { + 'kind_phys': ('iso_fortran_env', 'REAL64'), + 'kind_dyn': ('iso_fortran_env', 'REAL32'), + }) + + def test_parse_kind_types_explicit_module(self): + result = self.mod._parse_kind_types(['kind_phys=my_kinds:kind_r8']) + self.assertEqual(result, {'kind_phys': ('my_kinds', 'kind_r8')}) + + def test_parse_kind_types_mixed(self): + result = self.mod._parse_kind_types([ + 'kind_iso=REAL64', + 'kind_host=my_kinds:kind_r4', + ]) + self.assertEqual(result, { + 'kind_iso': ('iso_fortran_env', 'REAL64'), + 'kind_host': ('my_kinds', 'kind_r4'), + }) + + def test_parse_kind_types_malformed(self): + with self.assertRaises(CCPPError): + self.mod._parse_kind_types(['bad_entry']) + + def test_parse_kind_types_duplicate(self): + with self.assertRaises(CCPPError): + self.mod._parse_kind_types(['kind_phys=REAL64', 'kind_phys=REAL32']) + + def test_parse_kind_types_non_iso_without_module_rejected(self): + """Bare non-ISO spec must error -- the default module only applies to ISO names.""" + with self.assertRaises(CCPPError) as cm: + self.mod._parse_kind_types(['kind_phys=kind_r8']) + self.assertIn('ISO_FORTRAN_ENV', str(cm.exception)) + + def test_parse_kind_types_too_many_colons_rejected(self): + with self.assertRaises(CCPPError): + self.mod._parse_kind_types(['kind_phys=mod:sub:spec']) + + def test_parse_kind_types_empty_module_rejected(self): + with self.assertRaises(CCPPError): + self.mod._parse_kind_types(['kind_phys=:REAL64']) + + # ---- _collect_metadata_kind_specs -------------------------------------- + + def _make_table_with_specs(self, file_path, name, specs): + ctx = ParseContext(0, file_path) + t = MetadataTable(name, 'scheme', file_path, ctx) + t.kind_specs = list(specs) + return t + + def test_collect_metadata_kind_specs_empty(self): + self.assertEqual(self.mod._collect_metadata_kind_specs([]), {}) + + def test_collect_metadata_kind_specs_single_table(self): + t = self._make_table_with_specs( + '/p/a.meta', 'a', + [('kind_temp', 'temp_kinds', 'temp_r8')], + ) + self.assertEqual( + self.mod._collect_metadata_kind_specs([t]), + {'kind_temp': ('temp_kinds', 'temp_r8')}, + ) + + def test_collect_metadata_kind_specs_identical_duplicates_collapsed(self): + spec = ('kind_temp', 'temp_kinds', 'temp_r8') + a = self._make_table_with_specs('/p/a.meta', 'a', [spec]) + b = self._make_table_with_specs('/p/b.meta', 'b', [spec]) + self.assertEqual( + self.mod._collect_metadata_kind_specs([a, b]), + {'kind_temp': ('temp_kinds', 'temp_r8')}, + ) + + def test_collect_metadata_kind_specs_conflict_raises(self): + a = self._make_table_with_specs( + '/p/a.meta', 'a', + [('kind_temp', 'temp_kinds', 'temp_r8')], + ) + b = self._make_table_with_specs( + '/p/b.meta', 'b', + [('kind_temp', 'other_kinds', 'r8')], + ) + with self.assertRaises(CCPPError) as cm: + self.mod._collect_metadata_kind_specs([a, b]) + msg = str(cm.exception) + self.assertIn("kind 'kind_temp'", msg) + self.assertIn('/p/a.meta', msg) + self.assertIn('/p/b.meta', msg) + + def test_collect_metadata_kind_specs_multiple_distinct_kinds(self): + a = self._make_table_with_specs( + '/p/a.meta', 'a', + [('kind_temp', 'temp_kinds', 'temp_r8'), + ('kind_aux', 'aux_kinds', 'aux_r4')], + ) + self.assertEqual( + self.mod._collect_metadata_kind_specs([a]), + { + 'kind_temp': ('temp_kinds', 'temp_r8'), + 'kind_aux': ('aux_kinds', 'aux_r4'), + }, + ) + + # ---- _merge_cli_and_metadata_kinds ------------------------------------- + + def test_merge_cli_and_metadata_no_overlap(self): + cli = {'kind_phys': ('iso_fortran_env', 'REAL64')} + meta = {'kind_temp': ('temp_kinds', 'temp_r8')} + self.assertEqual( + self.mod._merge_cli_and_metadata_kinds(cli, meta), + { + 'kind_phys': ('iso_fortran_env', 'REAL64'), + 'kind_temp': ('temp_kinds', 'temp_r8'), + }, + ) + + def test_merge_cli_and_metadata_identical_collapsed(self): + cli = {'kind_temp': ('temp_kinds', 'temp_r8')} + meta = {'kind_temp': ('temp_kinds', 'temp_r8')} + self.assertEqual( + self.mod._merge_cli_and_metadata_kinds(cli, meta), + {'kind_temp': ('temp_kinds', 'temp_r8')}, + ) + + def test_merge_cli_and_metadata_conflict_raises(self): + cli = {'kind_temp': ('cli_kinds', 'r8')} + meta = {'kind_temp': ('meta_kinds', 'r8')} + with self.assertRaises(CCPPError) as cm: + self.mod._merge_cli_and_metadata_kinds(cli, meta) + self.assertIn("kind 'kind_temp'", str(cm.exception).lower() + .replace('Kind', 'kind')) + self.assertIn('cli_kinds', str(cm.exception)) + self.assertIn('meta_kinds', str(cm.exception)) + + def test_merge_then_default_kind_phys_injected_when_neither_provides(self): + """Default kind_phys is injected after the merge when neither side declares it.""" + import logging + log = logging.getLogger('ccpp_capgen_ng_test') + merged = self.mod._merge_cli_and_metadata_kinds({}, {}) + merged = self.mod._ensure_kind_phys_default(merged, log) + self.assertEqual( + merged, {'kind_phys': ('iso_fortran_env', 'REAL64')} + ) + + def test_metadata_kind_phys_suppresses_default(self): + """Metadata declaring kind_phys keeps the default from being injected.""" + import logging + log = logging.getLogger('ccpp_capgen_ng_test') + meta = {'kind_phys': ('host_kinds', 'kind_r8')} + merged = self.mod._merge_cli_and_metadata_kinds({}, meta) + merged = self.mod._ensure_kind_phys_default(merged, log) + self.assertEqual(merged, {'kind_phys': ('host_kinds', 'kind_r8')}) + + +######################################################################## +# Tests: apply_table_props (source_path, dependencies, dependencies_path) +######################################################################## + +class TestApplyTableProps(unittest.TestCase): + """Tests for MetadataTable.apply_table_props and parsing of table-level props.""" + + def _make_table(self, file_path='/project/src/foo.meta'): + ctx = ParseContext(0, file_path) + t = MetadataTable('foo', 'scheme', file_path, ctx) + return t + + def test_defaults_when_no_props(self): + t = self._make_table() + t.apply_table_props({}) + self.assertEqual(t.dependencies, []) + # source_path defaults to the meta file's directory + self.assertEqual(t.source_path, os.path.dirname(os.path.abspath(t.file_path))) + + def test_source_path_resolved(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'source_path': 'fortran'}) + self.assertEqual(t.source_path, '/project/src/fortran') + + def test_source_path_with_dotdot(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'source_path': '../other'}) + self.assertEqual(t.source_path, '/project/other') + + def test_single_dependency_no_dep_path(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': 'util.F90'}) + self.assertEqual(t.dependencies, ['/project/src/util.F90']) + + def test_multiple_dependencies(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': 'a.F90, b.F90'}) + self.assertIn('/project/src/a.F90', t.dependencies) + self.assertIn('/project/src/b.F90', t.dependencies) + self.assertEqual(len(t.dependencies), 2) + + def test_dependencies_path_as_base(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': 'util.F90', 'dependencies_path': 'lib'}) + self.assertEqual(t.dependencies, ['/project/src/lib/util.F90']) + + def test_dependencies_none_string(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': 'none'}) + self.assertEqual(t.dependencies, []) + + def test_dependency_relative_dotdot(self): + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': '../../shared/helper.F90'}) + self.assertEqual(t.dependencies, ['/shared/helper.F90']) + + def test_dependencies_list_form_accumulates(self): + """When ``dependencies`` appears more than once in a single + table header, the parser passes a list to apply_table_props; + each entry can itself be a comma-separated list of paths.""" + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': [ + 'a.F90', + 'b.F90, c.F90', + 'sub/d.F90', + ]}) + self.assertEqual(t.dependencies, [ + '/project/src/a.F90', + '/project/src/b.F90', + '/project/src/c.F90', + '/project/src/sub/d.F90', + ]) + + def test_dependencies_list_form_honors_dep_path(self): + """``dependencies_path`` applies to every entry in a list, + whether the entry is a single path or a comma-separated + bundle.""" + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({ + 'dependencies_path': '../../', + 'dependencies': [ + 'tools/mpiutil.F90', + 'Radiation/RRTMG/radlw_main.F90,Radiation/RRTMG/radsw_main.F90', + ], + }) + self.assertEqual(t.dependencies, [ + '/tools/mpiutil.F90', + '/Radiation/RRTMG/radlw_main.F90', + '/Radiation/RRTMG/radsw_main.F90', + ]) + + def test_dependencies_list_form_with_none_skip(self): + """A ``none`` entry inside a list is silently skipped (matches + the single-string ``none`` shorthand).""" + t = self._make_table('/project/src/foo.meta') + t.apply_table_props({'dependencies': [ + 'a.F90', + 'none', + 'b.F90', + ]}) + self.assertEqual(t.dependencies, [ + '/project/src/a.F90', + '/project/src/b.F90', + ]) + + def test_unrecognised_props_silently_ignored(self): + t = self._make_table() + t.apply_table_props({'unknown_key': 'value'}) + self.assertEqual(t.dependencies, []) + self.assertEqual(t.kind_specs, []) + + def test_kind_spec_explicit_form(self): + t = self._make_table() + t.apply_table_props({'kind_spec': 'temp_kinds:kind_temp=>temp_r8'}) + self.assertEqual( + t.kind_specs, [('kind_temp', 'temp_kinds', 'temp_r8')] + ) + + def test_kind_spec_shorthand_form(self): + t = self._make_table() + t.apply_table_props({'kind_spec': 'host_kinds:kind_r8'}) + self.assertEqual( + t.kind_specs, [('kind_r8', 'host_kinds', 'kind_r8')] + ) + + def test_kind_spec_list_accumulates(self): + t = self._make_table() + t.apply_table_props({'kind_spec': [ + 'temp_kinds:kind_temp=>temp_r8', + 'host_kinds:kind_r4', + ]}) + self.assertEqual(t.kind_specs, [ + ('kind_temp', 'temp_kinds', 'temp_r8'), + ('kind_r4', 'host_kinds', 'kind_r4'), + ]) + + def test_kind_spec_malformed_raises(self): + t = self._make_table() + with self.assertRaises(CCPPError): + t.apply_table_props({'kind_spec': 'real8'}) + + def test_kind_spec_extra_arrow_segment_rejected(self): + t = self._make_table() + with self.assertRaises(CCPPError): + t.apply_table_props( + {'kind_spec': 'mod:a=>b=>c'} + ) + + def test_module_name_default_empty(self): + t = self._make_table() + t.apply_table_props({}) + self.assertEqual(t.module_name, '') + + def test_module_name_explicit(self): + """``module_name`` override for cases where the Fortran module + name differs from the metadata table name (e.g. ``effr_pre`` table + whose Fortran module is ``mod_effr_pre``).""" + t = self._make_table() + t.apply_table_props({'module_name': 'mod_effr_pre'}) + self.assertEqual(t.module_name, 'mod_effr_pre') + + def test_module_name_whitespace_stripped(self): + t = self._make_table() + t.apply_table_props({'module_name': ' mod_foo '}) + self.assertEqual(t.module_name, 'mod_foo') + + def test_module_name_empty_string_keeps_default(self): + t = self._make_table() + t.apply_table_props({'module_name': ' '}) + self.assertEqual(t.module_name, '') + + +class TestTablePropsParseIntegration(unittest.TestCase): + """Verify that source_path/dependencies are parsed from actual meta text.""" + + def _parse(self, src, fname='/project/src/my_scheme.meta'): + return _parse_lines(src.splitlines(keepends=True), fname) + + def test_source_path_parsed(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + source_path = fortran + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src) + self.assertEqual(len(tbls), 1) + self.assertEqual(tbls[0].source_path, '/project/src/fortran') + + def test_dependencies_parsed(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + dependencies = util.F90, helper.F90 + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src) + self.assertIn('/project/src/util.F90', tbls[0].dependencies) + self.assertIn('/project/src/helper.F90', tbls[0].dependencies) + + def test_dependencies_path_applied(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + dependencies = qux.F90 + dependencies_path = adjust + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src) + self.assertEqual(tbls[0].dependencies, ['/project/src/adjust/qux.F90']) + + def test_dependencies_repeated_in_header(self): + """``dependencies`` may appear multiple times in a single table + header — each line contributes its (comma-separated) paths. + Real-world example from CCPP physics: dependencies_path = ../../ + followed by ~7 dependencies lines that each list a comma- + separated bundle in a different subtree. + """ + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = GFS_rrtmg_setup + type = scheme + dependencies_path = ../../ + dependencies = tools/mpiutil.F90 + dependencies = hooks/machine.F + dependencies = Radiation/radiation_aerosols.f + dependencies = Radiation/radiation_astronomy.f, Radiation/radiation_clouds.f, Radiation/radiation_gases.f + dependencies = Radiation/RRTMG/radlw_main.F90,Radiation/RRTMG/radlw_param.f,Radiation/RRTMG/radsw_main.F90,Radiation/RRTMG/radsw_param.f + [ccpp-arg-table] + name = GFS_rrtmg_setup_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src, fname='/project/physics/src/GFS_rrtmg_setup.meta') + self.assertEqual(len(tbls), 1) + # dependencies_path = ../../ → base is /project/ + self.assertEqual(tbls[0].dependencies, [ + '/project/tools/mpiutil.F90', + '/project/hooks/machine.F', + '/project/Radiation/radiation_aerosols.f', + '/project/Radiation/radiation_astronomy.f', + '/project/Radiation/radiation_clouds.f', + '/project/Radiation/radiation_gases.f', + '/project/Radiation/RRTMG/radlw_main.F90', + '/project/Radiation/RRTMG/radlw_param.f', + '/project/Radiation/RRTMG/radsw_main.F90', + '/project/Radiation/RRTMG/radsw_param.f', + ]) + + def test_dependencies_path_still_rejects_duplicates(self): + """Even with ``dependencies`` now accumulating, the + ``dependencies_path`` attribute itself remains single-valued — + repeating it is a metadata error.""" + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + dependencies_path = ../ + dependencies_path = ../../ + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + with self.assertRaises(CCPPError): + self._parse(src) + + def test_source_path_still_rejects_duplicates(self): + """``source_path`` is single-valued too.""" + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + source_path = fortran + source_path = src + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + with self.assertRaises(CCPPError): + self._parse(src) + + def test_kind_spec_single_line_parsed(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + kind_spec = temp_kinds:kind_temp=>temp_r8 + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src) + self.assertEqual( + tbls[0].kind_specs, + [('kind_temp', 'temp_kinds', 'temp_r8')], + ) + + def test_kind_spec_multiple_lines_accumulate(self): + """Repeat ``kind_spec`` lines accumulate without a duplicate-key error.""" + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + kind_spec = temp_kinds:kind_temp=>temp_r8 + kind_spec = host_kinds:kind_r4 + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + tbls = self._parse(src) + self.assertEqual(tbls[0].kind_specs, [ + ('kind_temp', 'temp_kinds', 'temp_r8'), + ('kind_r4', 'host_kinds', 'kind_r4'), + ]) + + def test_kind_spec_malformed_value_raises(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + kind_spec = not_a_kind_spec + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + """) + with self.assertRaises(CCPPError): + self._parse(src) + + def test_props_at_eof_no_arg_table(self): + """source_path parsed even if there are no [ccpp-arg-table] sections.""" + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + source_path = src + """) + tbls = self._parse(src) + self.assertEqual(len(tbls), 1) + self.assertEqual(tbls[0].source_path, '/project/src/src') + + def test_multiple_tables_independent_deps(self): + src = textwrap.dedent("""\ + [ccpp-table-properties] + name = scheme_a + type = scheme + dependencies = a.F90 + [ccpp-table-properties] + name = scheme_b + type = scheme + dependencies = b.F90 + """) + tbls = self._parse(src) + self.assertEqual(len(tbls), 2) + self.assertEqual(tbls[0].dependencies, ['/project/src/a.F90']) + self.assertEqual(tbls[1].dependencies, ['/project/src/b.F90']) + + +######################################################################## +# Doctest loader +######################################################################## + +def load_tests(loader, tests, ignore): + """Auto-discover and run all doctests in the metadata subpackage.""" + import doctest + import metadata.metadata_table as _mt + import metadata.parse_tools.parse_source as _ps + import metadata.parse_tools.parse_log as _pl + tests.addTests(doctest.DocTestSuite(_mt)) + tests.addTests(doctest.DocTestSuite(_ps)) + tests.addTests(doctest.DocTestSuite(_pl)) + return tests + + +######################################################################## + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_static_api.py b/unit-tests/test_static_api.py new file mode 100644 index 00000000..8ccabe72 --- /dev/null +++ b/unit-tests/test_static_api.py @@ -0,0 +1,1084 @@ +"""Unit tests for generator.static_api.""" + +import doctest +import os +import tempfile +import unittest +from unittest.mock import MagicMock + +from metadata.parse_tools import CCPPError +from generator.suite_resolver import resolve_suite +from generator.static_api import ( + _all_ctrl_args_for_phase, + _arg_top_level_name, + _build_local_to_std_top_level_map, + _collect_host_io, + _emit_var_set_loop, + _generate_static_api, + _suite_io_subroutine, + _suite_list_subroutine, + _suite_part_list_subroutine, + _suite_schemes_subroutine, + write_static_api, +) +from test_suite_resolver import ( + _load_full_host_dict, + _load_scheme_store, + _parse_suite, +) + + +def _resolve(): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + return resolve_suite(suite, store, hd) + + +def _generate(): + sr = _resolve() + return _generate_static_api(['test_simple'], [sr]) + + +class TestAllCtrlArgsForPhase(unittest.TestCase): + + def test_only_error_ctrl_args_in_test_case(self): + sr = _resolve() + # temp_calc_adjust uses errmsg/errflg which are now control vars. + args = _all_ctrl_args_for_phase([sr], 'run') + std_names = {a.standard_name for a in args} + self.assertEqual(std_names, {'ccpp_error_message', 'ccpp_error_code'}) + + def test_mismatched_lengths_raises(self): + from metadata.parse_tools import CCPPError + with self.assertRaises(CCPPError): + _generate_static_api(['a', 'b'], [_resolve()]) + + +class TestGenerateStaticApiModule(unittest.TestCase): + """Static API: ccpp_register/init/final are mandatory entry points and + are always emitted with the minimal lifecycle signature.""" + + def setUp(self): + self.lines = _generate() + self.text = '\n'.join(self.lines) + + def test_module_header_comment(self): + self.assertTrue(self.lines[0].startswith('!')) + self.assertIn('ccpp_static_api', self.lines[0]) + + def test_module_declaration(self): + self.assertIn('module ccpp_static_api', self.text) + self.assertIn('end module ccpp_static_api', self.text) + + def test_does_not_use_constituent_mod(self): + # Constituent merging is now opt-in via type=host (Task #6 follow-up). + self.assertNotIn('use ccpp_constituent_prop_mod', self.text) + self.assertNotIn('ccpp_model_constituents_t', self.text) + + def test_uses_suite_cap(self): + self.assertIn('use ccpp_test_simple_cap', self.text) + # Register is now mandatory and always imported. + self.assertIn('test_simple_register', self.text) + self.assertIn('test_simple_init', self.text) + self.assertIn('test_simple_final', self.text) + + def test_implicit_none_private(self): + self.assertIn('implicit none', self.text) + self.assertIn('private', self.text) + + def test_ccpp_register_public(self): + # ccpp_register is mandatory. + self.assertIn('public :: ccpp_register', self.text) + + def test_other_public_entry_points(self): + for ep in ('ccpp_init', 'ccpp_final', + 'ccpp_physics_init', 'ccpp_physics_timestep_init', + 'ccpp_physics_run', 'ccpp_physics_timestep_final', + 'ccpp_physics_final'): + self.assertIn('public :: {}'.format(ep), self.text) + + def test_contains_block(self): + self.assertIn('contains', self.lines) + + def test_no_constituent_reexport_when_absent(self): + # The test_simple fixture has no constituents — host_constituents + # module isn't emitted, so static_api must not USE or re-export it. + self.assertNotIn('use ccpp_host_constituents', self.text) + self.assertNotIn('ccpp_register_constituents', self.text) + self.assertNotIn('ccpp_initialize_constituents', self.text) + + +class TestStaticApiConstituentReexport(unittest.TestCase): + """When any suite uses constituent state, static_api USEs + ccpp_host_constituents and re-publics every host-facing routine plus + the constituent object so hosts can ``use ccpp_static_api, only: ...`` + for everything they need from CCPP.""" + + def setUp(self): + from test_suite_resolver import ( + _load_constituent_host_dict, + _load_constituent_consumer_store, + _parse_suite, + ) + hd = _load_constituent_host_dict() + store = _load_constituent_consumer_store() + suite = _parse_suite('suite_consume_constituent.xml') + sr = resolve_suite(suite, store, hd) + self.text = '\n'.join( + _generate_static_api(['consume_consts'], [sr], host_dict=hd, + scheme_store=store), + ) + + def _expected_symbols(self): + return [ + 'ccpp_model_constituents_obj', + 'ccpp_register_constituents', + 'ccpp_initialize_constituents', + 'ccpp_is_scheme_constituent', + 'ccpp_number_constituents', + 'ccpp_gather_constituents', + 'ccpp_update_constituents', + 'ccpp_const_get_index', + 'ccpp_constituents_array', + 'ccpp_advected_constituents_array', + 'ccpp_model_const_properties', + 'ccpp_deallocate_dynamic_constituents', + ] + + def test_uses_host_constituents_module(self): + self.assertIn('use ccpp_host_constituents, only:', self.text) + + def test_all_symbols_imported(self): + for sym in self._expected_symbols(): + self.assertIn(sym, self.text, + 'symbol not imported: {}'.format(sym)) + + def test_all_symbols_re_public(self): + for sym in self._expected_symbols(): + self.assertIn('public :: {}'.format(sym), self.text) + + +class TestCcppRegisterMandatory(unittest.TestCase): + """ccpp_register is always emitted with the minimal lifecycle signature + and dispatches to every suite's _register routine.""" + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_subroutine_present(self): + self.assertIn('subroutine ccpp_register(', self.text) + self.assertIn('end subroutine ccpp_register', self.text) + + def test_signature_minimal_no_host_dict(self): + # With no host_dict, fallback local names are used. + self.assertIn( + 'subroutine ccpp_register(suite_name, errflg, errmsg)', self.text, + ) + + def test_no_constituents_arg(self): + # Constituents handling is opt-in; not in the signature any longer. + sig_block = self.text.split('subroutine ccpp_register')[1].split( + 'end subroutine ccpp_register' + )[0] + self.assertNotIn('constituents', sig_block) + + def test_dispatches_to_suite_register(self): + self.assertIn("case('test_simple')", self.text) + self.assertIn('call test_simple_register(errmsg, errflg)', self.text) + + def test_default_case_error(self): + self.assertIn("'ccpp_register: unknown suite:", self.text) + + +class TestCcppInitFinalSubroutines(unittest.TestCase): + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_init_subroutine(self): + self.assertIn( + 'subroutine ccpp_init(suite_name, errflg, errmsg)', self.text, + ) + self.assertIn('call test_simple_init(errmsg, errflg)', self.text) + self.assertIn("'ccpp_init: unknown suite:", self.text) + + def test_final_subroutine(self): + self.assertIn( + 'subroutine ccpp_final(suite_name, errflg, errmsg)', self.text, + ) + self.assertIn('call test_simple_final(errmsg, errflg)', self.text) + self.assertIn("'ccpp_final: unknown suite:", self.text) + + +class TestCcppPhysicsSubroutines(unittest.TestCase): + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_all_physics_subroutines_present(self): + for phase in ('init', 'timestep_init', 'run', 'timestep_final', 'final'): + self.assertIn('subroutine ccpp_physics_{}'.format(phase), self.text) + + def test_run_dispatches_to_suite_cap(self): + # No host_dict passed → no ctrl entries → no-arg call to suite cap. + self.assertIn('call test_simple_physics_run()', self.text) + + def test_init_dispatches_to_suite_cap(self): + self.assertIn('call test_simple_physics_init()', self.text) + + def test_final_dispatches_to_suite_cap(self): + self.assertIn('call test_simple_physics_final()', self.text) + + def test_physics_no_default_error_case(self): + # Physics dispatch has no error case for unknown suite — just skips. + run_block_start = self.text.index('subroutine ccpp_physics_run') + run_block_end = self.text.index('end subroutine ccpp_physics_run') + run_block = self.text[run_block_start:run_block_end] + self.assertNotIn('case default', run_block) + + def test_select_case_on_suite_name(self): + self.assertIn('select case(trim(suite_name))', self.text) + + +class TestMultipleSuites(unittest.TestCase): + """Static API with two suites uses select case for both.""" + + def setUp(self): + sr = _resolve() + from copy import deepcopy + sr2 = deepcopy(sr) + sr2.suite_name = 'suite_b' + lines = _generate_static_api(['test_simple', 'suite_b'], [sr, sr2]) + self.text = '\n'.join(lines) + + def test_both_suites_in_register(self): + # ccpp_register dispatches to all suites. + self.assertIn("case('test_simple')", self.text) + self.assertIn("case('suite_b')", self.text) + + def test_both_suite_caps_used(self): + self.assertIn('use ccpp_test_simple_cap', self.text) + self.assertIn('use ccpp_suite_b_cap', self.text) + + +class TestWriteStaticApi(unittest.TestCase): + + def test_writes_file(self): + sr = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + path = write_static_api(['test_simple'], [sr], tmpdir) + self.assertTrue(os.path.isfile(path)) + self.assertEqual(os.path.basename(path), 'ccpp_static_api.F90') + + def test_file_content(self): + sr = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + path = write_static_api(['test_simple'], [sr], tmpdir) + with open(path) as fh: + content = fh.read() + self.assertIn('module ccpp_static_api', content) + # ccpp_register is now mandatory and always emitted. + self.assertIn('subroutine ccpp_register', content) + self.assertTrue(content.endswith('\n')) + + def test_creates_output_dir(self): + sr = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + subdir = os.path.join(tmpdir, 'api') + write_static_api(['test_simple'], [sr], subdir) + self.assertTrue(os.path.isdir(subdir)) + + def test_returns_absolute_path(self): + sr = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + path = write_static_api(['test_simple'], [sr], tmpdir) + self.assertTrue(os.path.isabs(path)) + + +class TestCcppInitMultiInstance(unittest.TestCase): + """ccpp_init includes instance_number when the host provides it; the new + minimal signature drops number_of_instances entirely.""" + + def setUp(self): + hd = _load_full_host_dict() + sr = _resolve() + lines = _generate_static_api(['test_simple'], [sr], hd) + self.text = '\n'.join(lines) + + def test_init_signature_has_instance_number(self): + # host_full.meta declares inst_num as instance_number; ninstances is + # NOT in the lifecycle signature any more. + self.assertIn( + 'subroutine ccpp_init(suite_name, errflg, errmsg, inst_num)', + self.text, + ) + + def test_init_signature_no_ninstances(self): + init_block = self.text.split('subroutine ccpp_init')[1].split( + 'end subroutine ccpp_init' + )[0] + self.assertNotIn('ninstances', init_block) + + def test_init_passes_inst_to_suite(self): + self.assertIn( + 'call test_simple_init(inst_num, errmsg, errflg)', self.text, + ) + + def test_register_signature_has_instance_number(self): + self.assertIn( + 'subroutine ccpp_register(suite_name, errflg, errmsg, inst_num)', + self.text, + ) + + def test_final_signature_has_instance_number(self): + self.assertIn( + 'subroutine ccpp_final(suite_name, errflg, errmsg, inst_num)', + self.text, + ) + + +class TestCcppInitSingleInstance(unittest.TestCase): + """ccpp_init drops inst_num when the host doesn't declare instance_number.""" + + def setUp(self): + hd = {k: v for k, v in _load_full_host_dict().items() + if k not in ('number_of_instances', 'instance_number')} + sr = _resolve() + lines = _generate_static_api(['test_simple'], [sr], hd) + self.text = '\n'.join(lines) + + def test_init_signature_no_instance_args(self): + self.assertIn( + 'subroutine ccpp_init(suite_name, errflg, errmsg)', self.text, + ) + init_block = self.text.split('subroutine ccpp_init')[1].split( + 'end subroutine ccpp_init' + )[0] + self.assertNotIn('ninstances', init_block) + self.assertNotIn('inst_num', init_block) + + def test_init_passes_no_extra_args_to_suite(self): + self.assertIn('call test_simple_init(errmsg, errflg)', self.text) + + +######################################################################## +# Suite-introspection: helpers +######################################################################## + +class TestEmitVarSetLoop(unittest.TestCase): + + def test_basic(self): + out = _emit_var_set_loop('x', ['a', 'b'], ' ') + self.assertEqual(out, [' allocate(x(2))', " x(1) = 'a'", " x(2) = 'b'"]) + + def test_empty(self): + out = _emit_var_set_loop('x', [], ' ') + self.assertEqual(out, [' allocate(x(0))']) + + def test_no_allocate(self): + out = _emit_var_set_loop('x', ['a'], ' ', allocate=False) + self.assertEqual(out, [" x(1) = 'a'"]) + + +class TestBuildLocalToStdTopLevelMap(unittest.TestCase): + """Reverse map covers top-level host_dict entries only (no DDT-leaf rows).""" + + def test_includes_top_level_entries(self): + # host_full has only plain leaves (no DDTs) → all entries top-level. + hd = _load_full_host_dict() + m = _build_local_to_std_top_level_map(hd) + self.assertEqual(m['gt0'], 'air_temperature') + self.assertEqual(m['ncols'], 'horizontal_dimension') + + def test_excludes_ddt_leaves(self): + from metadata.metadata_table import parse_metadata_file + from metadata.variable_resolver import build_flat_host_dict + hd = build_flat_host_dict( + parse_metadata_file(os.path.join( + os.path.dirname(__file__), 'sample_files', + 'host_with_ddt_instance.meta')), + [], + parse_metadata_file(os.path.join( + os.path.dirname(__file__), 'sample_files', + 'ddt_simple.meta')), + ) + m = _build_local_to_std_top_level_map(hd) + # The DDT instance itself appears (top-level). + self.assertEqual(m['gfs_statein'], 'gfs_statein') + # DDT-leaf local names ('phii', 'phil') are excluded. + self.assertNotIn('phii', m) + self.assertNotIn('phil', m) + + def test_none_returns_empty(self): + self.assertEqual(_build_local_to_std_top_level_map(None), {}) + + +class TestArgTopLevelName(unittest.TestCase): + """Collapse a flat DDT-leaf back to its top-level DDT instance name.""" + + def _make_arg(self, std_name, access_path): + """Mock ResolvedArg with just the fields _arg_top_level_name reads.""" + host_entry = MagicMock() + host_entry.access_path = access_path + arg = MagicMock() + arg.standard_name = std_name + arg.host_entry = host_entry + return arg + + def test_plain_leaf_unchanged(self): + arg = self._make_arg('air_temperature', 'gt0') + self.assertEqual(_arg_top_level_name(arg, {}), 'air_temperature') + + def test_ddt_leaf_collapsed(self): + arg = self._make_arg( + 'geopotential_at_interface', + 'gfs_statein(instance_number)%phii', + ) + m = {'gfs_statein': 'gfs_statein'} + self.assertEqual(_arg_top_level_name(arg, m), 'gfs_statein') + + def test_nested_ddt_collapses_to_outermost(self): + arg = self._make_arg('inner_field', 'outer(2)%inner%fld') + m = {'outer': 'outer_std_name'} + self.assertEqual(_arg_top_level_name(arg, m), 'outer_std_name') + + def test_unmapped_root_falls_back_to_arg_std_name(self): + # If the root local_name isn't in the map (inconsistent metadata), + # fall back to the arg's own standard_name. + arg = self._make_arg('foo', 'unknown(1)%bar') + self.assertEqual(_arg_top_level_name(arg, {}), 'foo') + + def test_no_host_entry(self): + arg = MagicMock() + arg.standard_name = 'baz' + arg.host_entry = None + self.assertEqual(_arg_top_level_name(arg, {'x': 'y'}), 'baz') + + +class TestCollectHostIo(unittest.TestCase): + """_collect_host_io: intent partitioning, control-var exclusion, sort.""" + + def setUp(self): + self.sr = _resolve() + self.hd = _load_full_host_dict() + + def test_includes_control_vars(self): + # temp_calc_adjust declares errflg/errmsg with intent=out — they + # appear in outputs. Matches original capgen's introspection. + inputs, outputs = _collect_host_io(self.sr, self.hd) + self.assertIn('ccpp_error_code', outputs) + self.assertIn('ccpp_error_message', outputs) + # …and not in inputs (intent=out only, not inout). + self.assertNotIn('ccpp_error_code', inputs) + self.assertNotIn('ccpp_error_message', inputs) + + def test_includes_host_args(self): + inputs, outputs = _collect_host_io(self.sr, self.hd) + # air_temperature is intent=inout in run phase → both lists. + self.assertIn('air_temperature', inputs) + self.assertIn('air_temperature', outputs) + + def test_sorted_outputs(self): + inputs, outputs = _collect_host_io(self.sr, self.hd) + self.assertEqual(inputs, sorted(inputs)) + self.assertEqual(outputs, sorted(outputs)) + + def test_collapse_ddts_no_ddts_unchanged(self): + # host_full has no DDTs → collapse is a no-op. + flat_in, flat_out = _collect_host_io(self.sr, self.hd, collapse_ddts=False) + coll_in, coll_out = _collect_host_io(self.sr, self.hd, collapse_ddts=True) + self.assertEqual(flat_in, coll_in) + self.assertEqual(flat_out, coll_out) + + def test_no_host_dict_collapse_falls_back(self): + # collapse_ddts=True without host_dict must not raise. + # With no DDTs the result is identical to the non-collapsed view. + no_hd_in, _ = _collect_host_io(self.sr, None, collapse_ddts=True) + flat_in, _ = _collect_host_io(self.sr, self.hd, collapse_ddts=False) + self.assertEqual(no_hd_in, flat_in) + + +class TestCollectHostIoIncludesNonHostSources(unittest.TestCase): + """_collect_host_io includes constituent args + register-phase + ccpp_constituent_properties_t args + control vars in the introspection + lists (matches original capgen). Only suite-owned vars are excluded.""" + + def setUp(self): + from test_suite_resolver import ( + _load_constituent_host_dict, + _load_constituent_consumer_store, + _load_constituent_scheme_store, + _parse_suite, + ) + # Mix: a register-phase producer scheme + a consumer scheme. + # Build a SuiteResolution for each, pass both to _collect_host_io. + self.hd = _load_constituent_host_dict() + consumer_store = _load_constituent_consumer_store() + register_store = _load_constituent_scheme_store() + consumer_suite = _parse_suite('suite_consume_constituent.xml') + register_suite = _parse_suite('suite_register_constituents.xml') + self.consumer_sr = resolve_suite(consumer_suite, consumer_store, self.hd) + self.register_sr = resolve_suite(register_suite, register_store, self.hd) + + def test_consumer_base_constituent_in_inputs(self): + inputs, _ = _collect_host_io(self.consumer_sr, self.hd) + # cldliq is intent=in advected=true → in inputs. + self.assertIn('cloud_liquid_water_mixing_ratio', inputs) + + def test_consumer_tendency_in_outputs(self): + _, outputs = _collect_host_io(self.consumer_sr, self.hd) + # tend_cldliq is intent=out constituent=true → in outputs. + self.assertIn( + 'tendency_of_cloud_liquid_water_mixing_ratio', outputs, + ) + + def test_register_phase_properties_t_in_outputs(self): + # The register-phase scheme declares dyn_const as intent=out + # ccpp_constituent_properties_t — appears in the output list. + _, outputs = _collect_host_io(self.register_sr, self.hd) + self.assertIn('dynamic_constituents_for_register_test', outputs) + + def test_control_vars_in_outputs(self): + # The register scheme also declares errmsg/errflg with intent=out. + _, outputs = _collect_host_io(self.register_sr, self.hd) + self.assertIn('ccpp_error_code', outputs) + self.assertIn('ccpp_error_message', outputs) + + +class TestCollectHostIoIncludesFrameworkDims(unittest.TestCase): + """``number_of_ccpp_constituents`` (and any other framework-constituent + dim that appears only as a dim token in scheme metadata) is included + in the introspection inputs list — matches original capgen. Host-side + dims (horizontal_dimension, vertical_layer_dimension) are NOT + included; they're stable host structure.""" + + def setUp(self): + from test_suite_resolver import ( + _load_full_host_dict, _load_scheme_store, _parse_suite, + ) + # Use the consume_constituent fixture — the scheme references + # number_of_ccpp_constituents as a dim of ccpp_constituents. + # But that fixture only declares 2D constituent vars, not the + # 3D ccpp_constituents directly. Build a minimal fixture + # specifically for this test. + from metadata.metadata_table import _parse_lines + from metadata.variable_resolver import SchemeStore + scheme_text = ( + '[ccpp-table-properties]\n' + ' name = uses_const_array\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = uses_const_array_run\n' + ' type = scheme\n' + '[ const ]\n' + ' standard_name = ccpp_constituents\n' + ' units = none\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension, ' + 'number_of_ccpp_constituents)\n' + ' type = real | kind = kind_phys\n' + ' intent = inout\n' + ) + tables = _parse_lines(scheme_text.splitlines(keepends=True), 't.meta') + store = SchemeStore.build_from(tables) + + # Use the host-only dict (no ccpp_model_constituents_t DDT + # instance) so the host-wins rule doesn't fire and the scheme's + # ccpp_constituents arg routes through capgen-ng's + # auto-provisioning path — that's the code path that surfaces + # number_of_ccpp_constituents as an input via used_const_dim_std_names. + self.hd = _load_full_host_dict() + # Parse a one-scheme suite XML inline. + import tempfile, os, logging + from generator.suite_xml import parse_suite_xml + suite_xml = ( + '\n' + '\n' + ' uses_const_array\n' + '\n' + ) + with tempfile.TemporaryDirectory() as tmp: + xml_path = os.path.join(tmp, 'suite.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), + skip_validation=True) + self.sr = resolve_suite(suite, store, self.hd) + + def test_number_of_ccpp_constituents_in_inputs(self): + inputs, _ = _collect_host_io(self.sr, self.hd) + self.assertIn('number_of_ccpp_constituents', inputs) + + def test_horizontal_dim_not_in_inputs(self): + inputs, _ = _collect_host_io(self.sr, self.hd) + # Sanity: the host-side dims are NOT included even though they + # appear as scheme arg dimensions. + self.assertNotIn('horizontal_dimension', inputs) + self.assertNotIn('vertical_layer_dimension', inputs) + + +class TestCollectHostIoIncludesSubcycleLoopBound(unittest.TestCase): + """A subcycle ``loop=""`` bound is supplied by the host + (it controls the per-cap do-loop count) and must therefore appear + in the introspection inputs list — otherwise a host comparing its + declared variables against ``ccpp_physics_suite_variables`` will + silently miss the dependency.""" + + def setUp(self): + from test_suite_resolver import ( + _load_full_host_dict, _load_scheme_store, _parse_suite, + ) + from metadata.metadata_table import _parse_lines + from metadata.variable_resolver import build_flat_host_dict + from generator.suite_resolver import resolve_suite + + # Pair the standard host_full + scheme_multipart with a small + # extension that declares the subcycle-count std name. + helper_src = ''' +[ccpp-table-properties] + name = subcycle_helper + type = host +[ccpp-arg-table] + name = subcycle_helper + type = host +[ n_sub ] + standard_name = num_subcycles_for_my_scheme + units = count + dimensions = () + type = integer +''' + helper_tbls = _parse_lines( + helper_src.splitlines(keepends=True), 'h.meta', + ) + + from test_suite_resolver import _SAMPLES_DIR + from metadata.metadata_table import parse_metadata_file + host_tbls = parse_metadata_file( + os.path.join(_SAMPLES_DIR, 'host_full.meta') + ) + ctrl_tbls = parse_metadata_file( + os.path.join(_SAMPLES_DIR, 'control_full.meta') + ) + host_only = [t for t in host_tbls if t.table_type == 'host'] + ctrl_only = [t for t in ctrl_tbls if t.table_type == 'control'] + ddt_only = [t for t in host_tbls if t.table_type == 'ddt'] + self.hd = build_flat_host_dict(host_only + helper_tbls, ctrl_only, ddt_only) + + scheme_store = _load_scheme_store() + + suite_xml = ( + '\n' + '\n' + ' \n' + ' \n' + ' temp_calc_adjust\n' + ' \n' + ' \n' + '\n' + ) + import tempfile, logging + from generator.suite_xml import parse_suite_xml + with tempfile.TemporaryDirectory() as tmp: + xml_path = os.path.join(tmp, 's.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), + skip_validation=True) + self.sr = resolve_suite(suite, scheme_store, self.hd) + + def test_subcycle_std_name_in_inputs(self): + inputs, _outputs = _collect_host_io(self.sr, self.hd) + self.assertIn('num_subcycles_for_my_scheme', inputs) + + def test_integer_literal_subcycle_does_not_pollute(self): + """A literal-integer subcycle bound (``loop="3"``) has no + ``loop_std_name`` set and therefore contributes nothing to + the inputs list.""" + from test_suite_resolver import ( + _load_scheme_store, _load_full_host_dict, + ) + from generator.suite_resolver import resolve_suite + import tempfile, logging + from generator.suite_xml import parse_suite_xml + + hd = _load_full_host_dict() + store = _load_scheme_store() + suite_xml = ( + '\n' + '\n' + ' \n' + ' \n' + ' temp_calc_adjust\n' + ' \n' + ' \n' + '\n' + ) + with tempfile.TemporaryDirectory() as tmp: + xml_path = os.path.join(tmp, 's.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), + skip_validation=True) + sr = resolve_suite(suite, store, hd) + inputs, _ = _collect_host_io(sr, hd) + # No spurious integer / std-name additions from the literal loop. + self.assertNotIn('3', inputs) + + +class TestCollectHostIoIncludesActiveExpr(unittest.TestCase): + """A flag referenced via ``active=()`` on a host variable + must appear in the introspection inputs list — even if no scheme + declares it as a direct argument. The host needs to know the + suite consumes the flag to decide whether the optional variables + are present.""" + + def setUp(self): + from test_suite_resolver import _load_scheme_store + from metadata.metadata_table import _parse_lines + from metadata.variable_resolver import build_flat_host_dict + from generator.suite_resolver import resolve_suite + import tempfile, logging + from generator.suite_xml import parse_suite_xml + + # Host has a flag (``flag_for_passive_check``) referenced only as + # an active=() expression on another host var. No scheme takes + # the flag as a direct argument — without the active-expr walk + # in _collect_host_io it would silently disappear. + host_src = ''' +[ccpp-table-properties] + name = active_helper + type = host +[ccpp-arg-table] + name = active_helper + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer +[ im ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + units = count + dimensions = () + type = integer +[ flag_passive ] + standard_name = flag_for_passive_check + units = flag + dimensions = () + type = logical +[ dt ] + standard_name = time_step_for_physics + units = s + dimensions = () + type = real + kind = kind_phys +[ gt0 ] + standard_name = air_temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + active = (flag_for_passive_check) +''' + from test_suite_resolver import _SAMPLES_DIR + from metadata.metadata_table import parse_metadata_file + ctrl_tbls = parse_metadata_file( + os.path.join(_SAMPLES_DIR, 'control_full.meta') + ) + host_tbls = _parse_lines(host_src.splitlines(keepends=True), 'h.meta') + ctrl_only = [t for t in ctrl_tbls if t.table_type == 'control'] + self.hd = build_flat_host_dict(host_tbls, ctrl_only, []) + + store = _load_scheme_store() + suite_xml = ( + '\n' + '\n' + ' temp_calc_adjust\n' + '\n' + ) + with tempfile.TemporaryDirectory() as tmp: + xml_path = os.path.join(tmp, 's.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), + skip_validation=True) + self.sr = resolve_suite(suite, store, self.hd) + + def test_active_flag_in_inputs(self): + inputs, _outputs = _collect_host_io(self.sr, self.hd) + self.assertIn('flag_for_passive_check', inputs) + + def test_active_flag_not_in_outputs(self): + """The flag is a pure input — it must not leak into outputs.""" + _, outputs = _collect_host_io(self.sr, self.hd) + self.assertNotIn('flag_for_passive_check', outputs) + + +######################################################################## +# Suite-introspection: ccpp_physics_suite_list +######################################################################## + +class TestSuiteListSubroutine(unittest.TestCase): + + def test_single_suite(self): + text = '\n'.join(_suite_list_subroutine(['test_simple'])) + self.assertIn('subroutine ccpp_physics_suite_list(suites)', text) + self.assertIn( + 'character(len=*), allocatable, intent(out) :: suites(:)', text, + ) + self.assertIn('allocate(suites(1))', text) + self.assertIn("suites(1) = 'test_simple'", text) + self.assertIn('end subroutine ccpp_physics_suite_list', text) + + def test_multi_suite(self): + text = '\n'.join(_suite_list_subroutine(['a', 'b', 'c'])) + self.assertIn('allocate(suites(3))', text) + self.assertIn("suites(1) = 'a'", text) + self.assertIn("suites(2) = 'b'", text) + self.assertIn("suites(3) = 'c'", text) + + +######################################################################## +# Suite-introspection: ccpp_physics_suite_part_list +######################################################################## + +class TestSuitePartListSubroutine(unittest.TestCase): + + def setUp(self): + self.sr = _resolve() + self.text = '\n'.join(_suite_part_list_subroutine(['test_simple'], [self.sr])) + + def test_signature(self): + self.assertIn('subroutine ccpp_physics_suite_part_list(', self.text) + self.assertIn( + 'character(len=*), intent(in) :: suite_name', self.text, + ) + self.assertIn( + 'character(len=*), allocatable, intent(out) :: part_list(:)', self.text, + ) + self.assertIn('integer, intent(out) :: errflg', self.text) + + def test_dispatch_and_groups(self): + self.assertIn("case ('test_simple')", self.text) + # test_simple has one group named 'physics'. + group_names = [g.group_name for g in self.sr.groups] + for i, gname in enumerate(group_names): + self.assertIn("part_list({}) = '{}'".format(i + 1, gname), self.text) + self.assertIn('allocate(part_list({}))'.format(len(group_names)), self.text) + + def test_default_error_case(self): + self.assertIn('case default', self.text) + self.assertIn('errflg = 1', self.text) + self.assertIn( + "errmsg = 'ccpp_physics_suite_part_list: unknown suite: '", + self.text, + ) + + def test_initialises_outputs(self): + # errmsg/errflg cleared at top of routine. + self.assertIn("errmsg = ''", self.text) + self.assertIn('errflg = 0', self.text) + + def test_errmsg_is_assumed_length(self): + """The errmsg dummy must be ``character(len=*)`` so the host can + pass any character length without copy-in/copy-out (e.g. the + host might use ``character(len=256)`` while the framework's + scheme metadata uses ``len=512``).""" + self.assertIn('character(len=*), intent(out) :: errmsg', + self.text) + self.assertNotIn('character(len=512)', self.text) + self.assertNotIn('character(len=256)', self.text) + + +######################################################################## +# Suite-introspection: ccpp_physics_suite_schemes +######################################################################## + +class TestSuiteSchemesSubroutine(unittest.TestCase): + + def setUp(self): + self.sr = _resolve() + self.text = '\n'.join( + _suite_schemes_subroutine(['test_simple'], [self.sr]) + ) + + def test_signature(self): + self.assertIn('subroutine ccpp_physics_suite_schemes(', self.text) + self.assertIn( + 'character(len=*), allocatable, intent(out) :: scheme_list(:)', + self.text, + ) + + def test_lists_temp_calc_adjust(self): + # test_simple's only scheme (across all phases) is temp_calc_adjust. + self.assertIn("scheme_list(1) = 'temp_calc_adjust'", self.text) + self.assertIn('allocate(scheme_list(1))', self.text) + + def test_dedup_across_phases(self): + # temp_calc_adjust appears in init, run, and final phases — must + # appear exactly once in the emitted list. + n = self.text.count("scheme_list(1) = 'temp_calc_adjust'") + self.assertEqual(n, 1) + self.assertNotIn("scheme_list(2) = 'temp_calc_adjust'", self.text) + + def test_default_error_case(self): + self.assertIn( + "errmsg = 'ccpp_physics_suite_schemes: unknown suite: '", + self.text, + ) + + def test_errmsg_is_assumed_length(self): + self.assertIn('character(len=*), intent(out) :: errmsg', + self.text) + self.assertNotIn('character(len=512)', self.text) + self.assertNotIn('character(len=256)', self.text) + + +######################################################################## +# Suite-introspection: ccpp_physics_suite_variables +######################################################################## + +class TestSuiteVariablesSubroutine(unittest.TestCase): + + def setUp(self): + self.sr = _resolve() + self.hd = _load_full_host_dict() + self.text = '\n'.join(_suite_io_subroutine( + ['test_simple'], [self.sr], self.hd, collapse_ddts=False, + )) + + def test_subroutine_name(self): + self.assertIn('subroutine ccpp_physics_suite_variables(', self.text) + self.assertIn('end subroutine ccpp_physics_suite_variables', self.text) + + def test_signature_includes_optional_filters(self): + self.assertIn( + 'logical, optional, intent(in) :: input_vars', + self.text, + ) + self.assertIn( + 'logical, optional, intent(in) :: output_vars', + self.text, + ) + + def test_no_struct_elements_arg(self): + # struct_elements is intentionally dropped (was a no-op in capgen-ng). + self.assertNotIn('struct_elements', self.text) + + def test_errmsg_is_assumed_length(self): + """``suite_variables`` (and ``suite_host_data`` — they share the + same emitter) must declare errmsg as ``character(len=*)`` so the + host can pass any character length.""" + self.assertIn('character(len=*), intent(out) :: errmsg', + self.text) + self.assertNotIn('character(len=512)', self.text) + self.assertNotIn('character(len=256)', self.text) + + def test_three_branch_dispatch(self): + self.assertIn('if (input_vars_use .and. output_vars_use) then', self.text) + self.assertIn('else if (input_vars_use) then', self.text) + self.assertIn('else if (output_vars_use) then', self.text) + # Empty fall-through branch. + self.assertIn('allocate(variable_list(0))', self.text) + + def test_includes_control_vars(self): + # ccpp_error_code and ccpp_error_message DO appear in the emitted + # variable list literals — they're scheme args (intent=out) and + # are part of the host-facing surface (matches original capgen). + self.assertIn("'ccpp_error_code'", self.text) + self.assertIn("'ccpp_error_message'", self.text) + + def test_includes_host_data_vars(self): + # air_temperature is in the host metadata and is intent=inout in run. + self.assertIn("'air_temperature'", self.text) + + def test_default_present_check(self): + self.assertIn('if (present(input_vars)) then', self.text) + self.assertIn('if (present(output_vars)) then', self.text) + self.assertIn('input_vars_use = .true.', self.text) + self.assertIn('output_vars_use = .true.', self.text) + + def test_default_error_case(self): + self.assertIn( + "errmsg = 'ccpp_physics_suite_variables: unknown suite: '", + self.text, + ) + + +######################################################################## +# Suite-introspection: ccpp_physics_suite_host_data +######################################################################## + +class TestSuiteHostDataSubroutine(unittest.TestCase): + """Same shape as _variables; differs only in DDT collapsing.""" + + def setUp(self): + self.sr = _resolve() + self.hd = _load_full_host_dict() + self.text = '\n'.join(_suite_io_subroutine( + ['test_simple'], [self.sr], self.hd, collapse_ddts=True, + )) + + def test_subroutine_name(self): + self.assertIn('subroutine ccpp_physics_suite_host_data(', self.text) + self.assertIn('end subroutine ccpp_physics_suite_host_data', self.text) + + def test_default_error_case(self): + self.assertIn( + "errmsg = 'ccpp_physics_suite_host_data: unknown suite: '", + self.text, + ) + + def test_no_ddts_matches_variables(self): + # host_full has no DDTs → the routine bodies (modulo the subroutine + # name and error message) should contain the same variable + # literals as ..._variables. + var_text = '\n'.join(_suite_io_subroutine( + ['test_simple'], [self.sr], self.hd, collapse_ddts=False, + )) + # A spot-check: any host-data variable in one is in the other. + self.assertIn("'air_temperature'", self.text) + self.assertIn("'air_temperature'", var_text) + + +######################################################################## +# Suite-introspection: full module wiring +######################################################################## + +class TestIntrospectionRoutinesInModule(unittest.TestCase): + """Verify all five introspection routines are emitted and made public.""" + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_all_routines_present(self): + for sub in ( + 'ccpp_physics_suite_list', + 'ccpp_physics_suite_part_list', + 'ccpp_physics_suite_schemes', + 'ccpp_physics_suite_variables', + 'ccpp_physics_suite_host_data', + ): + self.assertIn('subroutine {}'.format(sub), self.text) + self.assertIn('end subroutine {}'.format(sub), self.text) + self.assertIn('public :: {}'.format(sub), self.text) + + +def load_tests(loader, tests, ignore): + import generator.static_api as sa + tests.addTests(doctest.DocTestSuite(sa)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_suite_cap.py b/unit-tests/test_suite_cap.py new file mode 100644 index 00000000..972d7b75 --- /dev/null +++ b/unit-tests/test_suite_cap.py @@ -0,0 +1,358 @@ +"""Unit tests for generator.suite_cap.""" + +import doctest +import os +import tempfile +import unittest +from unittest.mock import MagicMock + +from metadata.metadata_table import parse_metadata_file +from metadata.variable_resolver import build_flat_host_dict, SchemeStore +from generator.suite_resolver import resolve_suite +from generator.suite_cap import ( + _all_suite_scheme_names, + _schemes_with_register, + _suite_ctrl_args_for_phase, + _generate_suite_cap, + write_suite_cap, +) +from test_suite_resolver import ( + _load_full_host_dict, + _load_scheme_store, + _parse_suite, +) + + +def _resolve(): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + return resolve_suite(suite, store, hd), store + + +def _generate(): + sr, store = _resolve() + return _generate_suite_cap('test_simple', sr, store) + + +class TestAllSuiteSchemeNames(unittest.TestCase): + + def test_single_scheme(self): + sr, _ = _resolve() + names = _all_suite_scheme_names(sr) + self.assertIn('temp_calc_adjust', names) + + def test_no_duplicates(self): + sr, _ = _resolve() + names = _all_suite_scheme_names(sr) + self.assertEqual(len(names), len(set(names))) + + +class TestSchemesWithRegister(unittest.TestCase): + + def test_none_have_register(self): + sr, store = _resolve() + names = _all_suite_scheme_names(sr) + reg = _schemes_with_register(names, store) + # temp_calc_adjust has no register phase. + self.assertEqual(reg, []) + + def test_scheme_with_register(self): + store = MagicMock() + store.phases_for.side_effect = ( + lambda n: ['register', 'run'] if n == 'my_scheme' else ['run'] + ) + result = _schemes_with_register(['my_scheme', 'other_scheme'], store) + self.assertEqual(result, ['my_scheme']) + + +class TestSuiteCtrlArgsForPhase(unittest.TestCase): + + def test_only_error_ctrl_args_in_test_case(self): + sr, _ = _resolve() + # temp_calc_adjust uses errmsg/errflg which are now control vars. + args = _suite_ctrl_args_for_phase(sr, 'run') + std_names = {a.standard_name for a in args} + self.assertEqual(std_names, {'ccpp_error_message', 'ccpp_error_code'}) + + def test_unknown_phase_returns_empty(self): + sr, _ = _resolve() + args = _suite_ctrl_args_for_phase(sr, 'register') + self.assertEqual(args, []) + + +class TestGenerateSuiteCapModule(unittest.TestCase): + """Suite cap with no register-providing schemes: constituent USE and + _register are NOT emitted (conditional emission).""" + + def setUp(self): + self.lines = _generate() + self.text = '\n'.join(self.lines) + + def test_module_header_comment(self): + self.assertTrue(self.lines[0].startswith('!')) + self.assertIn('test_simple', self.lines[0]) + + def test_module_declaration(self): + self.assertIn('module ccpp_test_simple_cap', self.text) + self.assertIn('end module ccpp_test_simple_cap', self.text) + + def test_does_not_use_constituent_mod(self): + # No register-providing schemes → no constituent module dependency. + self.assertNotIn('use ccpp_constituent_prop_mod', self.text) + self.assertNotIn('ccpp_model_constituents_t', self.text) + + def test_uses_group_cap_mod(self): + self.assertIn('use ccpp_test_simple_physics_cap', self.text) + + def test_implicit_none_private(self): + self.assertIn('implicit none', self.text) + self.assertIn('private', self.text) + + def test_public_register_always_emitted(self): + # _register is mandatory in the new design — always public. + self.assertIn('public :: test_simple_register', self.text) + + def test_public_init_final(self): + self.assertIn('public :: test_simple_init', self.text) + self.assertIn('public :: test_simple_final', self.text) + + def test_public_all_physics_phases(self): + for phase in ('init', 'timestep_init', 'run', 'timestep_final', 'final'): + self.assertIn( + 'public :: test_simple_physics_{}'.format(phase), self.text + ) + + def test_contains_block(self): + self.assertIn('contains', self.lines) + + +class TestRegisterSubroutineAlwaysEmitted(unittest.TestCase): + """``_register`` is mandatory and emitted unconditionally. When no + schemes have a register phase, the body is just the state-alloc + guard + + state-transition skeleton; no scheme calls.""" + + def setUp(self): + lines = _generate() + self.text = '\n'.join(lines) + + def test_register_subroutine_present(self): + self.assertIn('subroutine test_simple_register', self.text) + self.assertIn('end subroutine test_simple_register', self.text) + + def test_no_constituents_arg(self): + # Constituents are now opt-in via type=host; not in the cap at all + # when no register-phase scheme declares ccpp_constituent_properties_t. + self.assertNotIn('constituents', self.text) + self.assertNotIn('ccpp_constituent_prop_mod', self.text) + + def test_no_scheme_register_calls(self): + # temp_calc_adjust has no register phase → no scheme_register call. + self.assertNotIn('call temp_calc_adjust_register', self.text) + + def test_state_alloc_called(self): + # Register always allocates state (idempotent) on first call. + self.assertIn('call test_simple_suite_state_alloc', self.text) + + def test_idempotent_guard(self): + # Per-instance idempotent skip if already at REGISTERED or beyond. + self.assertIn('>= CCPP_SUITE_REGISTERED', self.text) + + def test_state_transition(self): + self.assertIn('= CCPP_SUITE_REGISTERED', self.text) + + +class TestInitFinalSubroutines(unittest.TestCase): + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_init_subroutine(self): + self.assertIn('subroutine test_simple_init(errmsg, errflg)', self.text) + self.assertIn('end subroutine test_simple_init', self.text) + + def test_final_subroutine(self): + self.assertIn('subroutine test_simple_final(errmsg, errflg)', self.text) + self.assertIn('end subroutine test_simple_final', self.text) + + def test_init_calls_group_state_alloc(self): + # No host_dict passed → single-instance → literal 1 for ninstances. + self.assertIn( + 'call ccpp_test_simple_physics_state_alloc(1, errmsg, errflg)', + self.text, + ) + + def test_register_calls_suite_state_alloc(self): + # Suite state allocation happens in _register, not _init. + # No host_dict → single-instance → literal 1 for ninstances. + self.assertIn( + 'call test_simple_suite_state_alloc(1, errmsg, errflg)', + self.text, + ) + + def test_final_calls_state_dealloc(self): + self.assertIn('call ccpp_test_simple_physics_state_dealloc(errmsg, errflg)', self.text) + + +class TestPhysicsDispatch(unittest.TestCase): + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_run_dispatch_present(self): + # No control vars in test setup → no-arg signature. + self.assertIn('subroutine test_simple_physics_run()', self.text) + self.assertIn('end subroutine test_simple_physics_run', self.text) + + def test_run_dispatches_to_group_cap(self): + # No group_name control var → unconditional call, no select case. + self.assertIn('call ccpp_test_simple_physics_run()', self.text) + + def test_init_dispatches_to_group_cap(self): + self.assertIn('call ccpp_test_simple_physics_init()', self.text) + + def test_final_dispatches_to_group_cap(self): + self.assertIn('call ccpp_test_simple_physics_final()', self.text) + + def test_timestep_init_dispatches_to_group_cap(self): + # Group phase subroutines are always emitted so the state machine + # transitions through every phase, even when no scheme has a routine + # for that phase — so the dispatch must always call into the group cap. + self.assertIn('subroutine test_simple_physics_timestep_init()', self.text) + self.assertIn('call ccpp_test_simple_physics_timestep_init()', self.text) + + def test_timestep_final_dispatches_to_group_cap(self): + self.assertIn('subroutine test_simple_physics_timestep_final()', self.text) + self.assertIn('call ccpp_test_simple_physics_timestep_final()', self.text) + + def test_no_select_case_without_group_name_ctrl(self): + # No group_name control var in test setup → no select case dispatch. + self.assertNotIn('select case(trim(group_name))', self.text) + + +class TestWriteSuiteCap(unittest.TestCase): + + def test_writes_file(self): + sr, store = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_cap('test_simple', sr, store, tmpdir) + self.assertTrue(os.path.isfile(path)) + self.assertEqual(os.path.basename(path), 'ccpp_test_simple_cap.F90') + + def test_file_content(self): + sr, store = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_cap('test_simple', sr, store, tmpdir) + with open(path) as fh: + content = fh.read() + self.assertIn('module ccpp_test_simple_cap', content) + # Register subroutine is always emitted now. + self.assertIn('subroutine test_simple_register', content) + self.assertTrue(content.endswith('\n')) + + def test_creates_output_dir(self): + sr, store = _resolve() + with tempfile.TemporaryDirectory() as tmpdir: + subdir = os.path.join(tmpdir, 'caps') + write_suite_cap('test_simple', sr, store, subdir) + self.assertTrue(os.path.isdir(subdir)) + + +class TestFinalSubroutineStateMachine(unittest.TestCase): + """``_final`` transitions per-instance state to UNREGISTERED and + triggers a last-to-leave dealloc when every instance has finalized.""" + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_state_check_register_message(self): + # The not-allocated guard now refers to ccpp_register, not ccpp_init. + self.assertIn( + 'ccpp_register has not been called', self.text, + ) + + def test_idempotent_unregistered_skip(self): + self.assertIn('== CCPP_SUITE_UNREGISTERED', self.text) + + def test_state_transition_to_unregistered(self): + self.assertIn('= CCPP_SUITE_UNREGISTERED', self.text) + + def test_last_to_leave_dealloc(self): + self.assertIn( + 'all(ccpp_suite_state == CCPP_SUITE_UNREGISTERED)', self.text, + ) + + +class TestSuiteCapNoConstituentDeclarations(unittest.TestCase): + """Under option A the suite cap no longer owns constituent state. + + All declarations (constituent obj, pointers, index_of_) live in + the host-wide ``ccpp_host_constituents`` module. The suite cap is + responsible only for packing per-suite dynamic-constituent arrays + into the shared buffer during ``_register``. + """ + + def setUp(self): + from test_suite_resolver import ( + _load_constituent_host_dict, + _load_constituent_consumer_store, + ) + self.hd = _load_constituent_host_dict() + self.store = _load_constituent_consumer_store() + self.suite = _parse_suite('suite_consume_constituent.xml') + self.sr = resolve_suite(self.suite, self.store, self.hd) + self.text = '\n'.join( + _generate_suite_cap('consume_consts', self.sr, self.store, self.hd) + ) + + def test_no_kind_phys_import(self): + # Suite cap doesn't need kind_phys — constituent arrays live elsewhere. + self.assertNotIn('use ccpp_kinds, only: kind_phys', self.text) + + def test_no_constituent_pointer_type_import(self): + self.assertNotIn('ccpp_constituent_prop_ptr_t', self.text) + + def test_no_module_level_pointers(self): + self.assertNotIn( + 'pointer, public :: ccpp_constituents', self.text, + ) + self.assertNotIn( + 'pointer, public :: ccpp_constituent_tendencies', self.text, + ) + + def test_no_index_of_X_in_suite_cap(self): + self.assertNotIn('index_of_cloud_liquid_water_mixing_ratio', self.text) + + def test_no_const_index_call_in_init(self): + init_body = self.text.split('subroutine consume_consts_init')[1].split( + 'end subroutine consume_consts_init' + )[0] + self.assertNotIn('%const_index(', init_body) + self.assertNotIn('%vars_layer', init_body) + + +class TestSuiteCapNoConstituentEmissionWhenAbsent(unittest.TestCase): + """When the suite does not reference any constituent state, the + suite cap emits no constituent-related code.""" + + def setUp(self): + self.text = '\n'.join(_generate()) + + def test_no_constituent_pointers(self): + self.assertNotIn('=> null()', self.text) + + def test_no_index_of_declarations(self): + self.assertNotIn('index_of_', self.text) + + def test_no_constituent_prop_ptr_type_import(self): + self.assertNotIn('ccpp_constituent_prop_ptr_t', self.text) + + +def load_tests(loader, tests, ignore): + import generator.suite_cap as sc + tests.addTests(doctest.DocTestSuite(sc)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_suite_data.py b/unit-tests/test_suite_data.py new file mode 100644 index 00000000..1e9d9f68 --- /dev/null +++ b/unit-tests/test_suite_data.py @@ -0,0 +1,249 @@ +"""Unit tests for generator.suite_data.""" + +import doctest +import os +import tempfile +import unittest + +from metadata.parse_tools import CCPPError +from generator.suite_data import _generate_suite_data, write_suite_data +from generator.suite_resolver import SuiteVar + + +def _make_sv(std_name, local='myvar', type_='real', kind='kind_phys', + units='K', dims=None, scheme='sch', phase='run'): + return SuiteVar( + standard_name=std_name, + local_name=local, + type_=type_, + kind=kind, + units=units, + dimensions=dims or [], + source_scheme=scheme, + source_phase=phase, + ) + + +class TestGenerateSuiteDataEmpty(unittest.TestCase): + """Suite with no suite-owned variables.""" + + def setUp(self): + self.lines = _generate_suite_data('mysuite', {}) + self.text = '\n'.join(self.lines) + + def test_module_header_comment(self): + self.assertTrue(self.lines[0].startswith('!')) + self.assertIn('mysuite', self.lines[0]) + + def test_module_declaration(self): + self.assertIn('module ccpp_mysuite_data', self.text) + self.assertIn('end module ccpp_mysuite_data', self.text) + + def test_implicit_none_private(self): + self.assertIn('implicit none', self.text) + self.assertIn('private', self.text) + + def test_ddt_type_name(self): + self.assertIn('type, public :: ccpp_mysuite_data_t', self.text) + self.assertIn('end type ccpp_mysuite_data_t', self.text) + + def test_empty_ddt_comment(self): + self.assertIn('no suite-owned variables', self.text) + + def test_module_instance(self): + self.assertIn( + 'type(ccpp_mysuite_data_t), allocatable, target, public :: ccpp_suite_data(:)', + self.text, + ) + + def test_no_trailing_newlines(self): + for line in self.lines: + self.assertNotIn('\n', line) + + +class TestGenerateSuiteDataWithVars(unittest.TestCase): + """Suite with suite-owned variables.""" + + def setUp(self): + suite_vars = { + 'air_temp_adjusted': _make_sv( + 'air_temp_adjusted', 'temp_adj', 'real', 'kind_phys', 'K', + dims=['horizontal_loop_extent', 'vertical_layer_dimension'], + ), + 'humidity': _make_sv( + 'humidity', 'q', 'real', 'kind_phys', 'kg kg-1', + dims=['horizontal_loop_extent'], + ), + } + self.lines = _generate_suite_data('suite_x', suite_vars) + self.text = '\n'.join(self.lines) + + def test_fields_present(self): + self.assertIn('temp_adj', self.text) + self.assertIn('q', self.text) + + def test_allocatable_arrays(self): + # Array fields should be allocatable. + self.assertIn('allocatable', self.text) + + def test_real_kind(self): + self.assertIn('real(kind=kind_phys)', self.text) + + def test_uses_ccpp_kinds(self): + # Suite vars referencing ``kind_phys`` must USE it from ccpp_kinds. + self.assertIn('use ccpp_kinds, only: kind_phys', self.text) + + def test_no_empty_comment(self): + self.assertNotIn('no suite-owned variables', self.text) + + def test_fields_sorted(self): + # Sorted by standard_name: air_temp_adjusted before humidity. + idx_temp = self.text.index('temp_adj') + idx_q = self.text.index(' q') + self.assertLess(idx_temp, idx_q) + + def test_components_have_no_target_attribute(self): + """Fortran does NOT allow ``target`` as a derived-type component + attribute; the TARGET attribute lives on the outer instance array + instead. See :func:`test_instance_array_is_target`.""" + # No component-level ``, target`` substring should appear on a + # field declaration line. We check the strict patterns we'd + # emit if this regressed. + self.assertNotIn('allocatable, target :: temp_adj', self.text) + self.assertNotIn('allocatable, target :: q', self.text) + + def test_instance_array_is_target(self): + """The module-level instance array carries TARGET so every + ``ccpp_suite_data(i)%component(...)`` subobject is a valid + pointer-assignment target (used by the group cap to pointer-assign + optional-arg wrappers and transformation temporaries). + """ + self.assertIn( + 'type(ccpp_suite_x_data_t), allocatable, target, public :: ' + 'ccpp_suite_data(:)', + self.text, + ) + + +class TestGenerateSuiteDataScalar(unittest.TestCase): + """Suite var that is a scalar (no dimensions).""" + + def setUp(self): + sv = _make_sv('flag_var', 'flag', 'logical', '', '1', dims=[]) + self.lines = _generate_suite_data('s', {'flag_var': sv}) + self.text = '\n'.join(self.lines) + + def test_no_allocatable_for_scalar(self): + # Scalar fields should NOT be allocatable; only the outer instance array is. + type_body = self.text.split('type, public ::')[1].split('end type')[0] + self.assertNotIn('allocatable', type_body) + + def test_logical_type(self): + self.assertIn('logical', self.text) + + def test_no_ccpp_kinds_use_when_no_kinded_vars(self): + # No real(kind=...) vars → no ``use ccpp_kinds`` should be emitted. + self.assertNotIn('use ccpp_kinds', self.text) + + +class TestGenerateSuiteDataDDT(unittest.TestCase): + """Suite-owned variable whose type is a DDT defined in a scheme module.""" + + def setUp(self): + self.suite_vars = { + 'volume_mixing_ratio_ddt': _make_sv( + 'volume_mixing_ratio_ddt', 'vmr', 'vmr_type', '', 'none', + dims=[], scheme='make_ddt', + ), + } + self.ddt_module_map = {'vmr_type': 'make_ddt'} + + def test_emits_use_for_ddt_module(self): + lines = _generate_suite_data( + 'ddt_suite', self.suite_vars, + ddt_module_map=self.ddt_module_map, + ) + text = '\n'.join(lines) + self.assertIn('use make_ddt, only: vmr_type', text) + # Components carry the TARGET attribute so group caps can + # pointer-assign into them (see suite_data.py docstring). + self.assertIn('type(vmr_type) :: vmr', text) + + def test_use_appears_before_implicit_none(self): + lines = _generate_suite_data( + 'ddt_suite', self.suite_vars, + ddt_module_map=self.ddt_module_map, + ) + idx_use = next(i for i, l in enumerate(lines) if 'use make_ddt' in l) + idx_impl = next(i for i, l in enumerate(lines) if 'implicit none' in l) + self.assertLess(idx_use, idx_impl) + + def test_handles_type_paren_form(self): + suite_vars = { + 'wrapped_ddt': _make_sv( + 'wrapped_ddt', 'w', 'type(my_type)', '', 'none', dims=[], + ), + } + lines = _generate_suite_data( + 'ddt_suite', suite_vars, + ddt_module_map={'my_type': 'wrap_mod'}, + ) + text = '\n'.join(lines) + self.assertIn('use wrap_mod, only: my_type', text) + + def test_missing_ddt_module_raises(self): + with self.assertRaisesRegex(CCPPError, "vmr_type"): + _generate_suite_data( + 'ddt_suite', self.suite_vars, + ddt_module_map={}, + ) + + def test_no_ddt_no_use(self): + # When suite vars are all intrinsic, no DDT USE lines are emitted. + sv = _make_sv('temp', 't', 'real', 'kind_phys', 'K', dims=[]) + lines = _generate_suite_data( + 'ds', {'temp': sv}, ddt_module_map=None, + ) + text = '\n'.join(lines) + self.assertNotIn('use make_ddt', text) + self.assertIn('use ccpp_kinds, only: kind_phys', text) + + +class TestWriteSuiteData(unittest.TestCase): + + def test_writes_file(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_data('s', {}, tmpdir) + self.assertTrue(os.path.isfile(path)) + + def test_filename(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_data('test_simple', {}, tmpdir) + self.assertEqual(os.path.basename(path), 'ccpp_test_simple_data.F90') + + def test_file_ends_with_newline(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_data('s', {}, tmpdir) + with open(path) as fh: + self.assertTrue(fh.read().endswith('\n')) + + def test_creates_output_dir(self): + with tempfile.TemporaryDirectory() as tmpdir: + subdir = os.path.join(tmpdir, 'newdir') + write_suite_data('s', {}, subdir) + self.assertTrue(os.path.isdir(subdir)) + + def test_returns_absolute_path(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = write_suite_data('s', {}, tmpdir) + self.assertTrue(os.path.isabs(path)) + + +def load_tests(loader, tests, ignore): + import generator.suite_data as sd + tests.addTests(doctest.DocTestSuite(sd)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py new file mode 100644 index 00000000..8490ce0e --- /dev/null +++ b/unit-tests/test_suite_resolver.py @@ -0,0 +1,2880 @@ +"""Unit and integration tests for generator.suite_resolver and generator.group_cap. + +Tests are organized as: +- Unit tests for helper functions (unit conversion, subscript builder, etc.) +- Unit tests for single-argument resolution (Cases 1-4) +- Integration tests: full suite resolution using sample files + suite XML +- Group cap output tests: check generated Fortran source lines +""" + +import doctest +import os +import sys +import tempfile +import unittest + +from metadata.metadata_table import _parse_lines, parse_metadata_file +from metadata.parse_tools import CCPPError, ParseContext +from metadata.variable_resolver import build_flat_host_dict, SchemeStore + +from generator.suite_resolver import ( + _normalize_unit_string, + _unit_to_id, + find_unit_conversion, + _apply_transform_formula, + _build_call_subscript, + _build_merged_subscript, + _substitute_instance_idx, + _translate_active_expr, + _root_symbol, + _local_name_conflict, + _resolve_one_arg, + _dedup_scheme_names, + resolve_suite, + iter_phase_calls, + SuiteVar, + ResolvedArg, + ResolvedCall, + ResolvedGroup, + ResolvedSubcycle, + SuiteResolution, +) +from generator.group_cap import ( + _fortran_type_str, + _dim_decl, + _dim_decl_local, + _use_statements, + _generate_group_cap, + _collect_kinds_used, + _transform_comment, + write_group_cap, +) + +# --------------------------------------------------------------------------- +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SAMPLES_DIR = os.path.join(_TESTS_DIR, 'sample_files') +_SUITE_DIR = os.path.join(_TESTS_DIR, 'sample_suite_files') + + +def _sf(name): + return os.path.join(_SAMPLES_DIR, name) + + +def _suite_file(name): + return os.path.join(_SUITE_DIR, name) + + +def _ctx(): + return ParseContext(0, 'test.meta') + + +def _parse(src, fname='t.meta'): + return _parse_lines(src.splitlines(keepends=True), fname) + + +def _load_full_host_dict(): + """Load the host_full + control_full metadata into a flat dict.""" + host_tbls = parse_metadata_file(_sf('host_full.meta')) + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + return build_flat_host_dict(host_tbls, ctrl_tbls, []) + + +def _load_scheme_store(): + """Load temp_calc_adjust scheme.""" + tables = parse_metadata_file(_sf('scheme_multipart.meta')) + return SchemeStore.build_from(tables) + + +def _parse_suite(name='suite_test_simple.xml'): + """Parse a suite XML file and return the Suite object.""" + from generator.suite_xml import parse_suite_xml + import logging + logger = logging.getLogger('test') + with tempfile.TemporaryDirectory() as tmpdir: + return parse_suite_xml(_suite_file(name), tmpdir, logger, + skip_validation=True) + + +######################################################################## +# Tests: _normalize_unit_string +######################################################################## + +class TestNormalizeUnitString(unittest.TestCase): + + def test_bare_positive_exponent_gets_plus(self): + self.assertEqual(_normalize_unit_string('m2'), 'm+2') + self.assertEqual(_normalize_unit_string('m2 s-2'), 'm+2 s-2') + self.assertEqual(_normalize_unit_string('kg m2'), 'kg m+2') + + def test_existing_plus_unchanged(self): + self.assertEqual(_normalize_unit_string('m+2'), 'm+2') + self.assertEqual(_normalize_unit_string('m+2 s-2'), 'm+2 s-2') + + def test_negative_exponent_unchanged(self): + self.assertEqual(_normalize_unit_string('m s-1'), 'm s-1') + self.assertEqual(_normalize_unit_string('kg kg-1'), 'kg kg-1') + self.assertEqual(_normalize_unit_string('kg m-3'), 'kg m-3') + + def test_no_exponent_unchanged(self): + self.assertEqual(_normalize_unit_string('Pa'), 'Pa') + self.assertEqual(_normalize_unit_string('kg m s-2'), 'kg m s-2') + self.assertEqual(_normalize_unit_string(''), '') + + def test_idempotent(self): + once = _normalize_unit_string('m2 s-2') + twice = _normalize_unit_string(once) + self.assertEqual(once, twice) + + +######################################################################## +# Tests: _unit_to_id +######################################################################## + +class TestUnitToId(unittest.TestCase): + + def test_simple(self): + self.assertEqual(_unit_to_id('Pa'), 'Pa') + self.assertEqual(_unit_to_id('K'), 'K') + + def test_space_to_underscore(self): + self.assertEqual(_unit_to_id('m s-1'), 'm_s_minus_1') + self.assertEqual(_unit_to_id('kg kg-1'), 'kg_kg_minus_1') + + def test_positive_exponent(self): + self.assertEqual(_unit_to_id('m2 s-2'), 'm_plus_2_s_minus_2') + + def test_explicit_plus(self): + # hypothetical 'm+3' + self.assertEqual(_unit_to_id('m+3'), 'm_plus_3') + + def test_no_change(self): + self.assertEqual(_unit_to_id('radian'), 'radian') + self.assertEqual(_unit_to_id('degree'), 'degree') + + +######################################################################## +# Tests: find_unit_conversion +######################################################################## + +class TestFindUnitConversion(unittest.TestCase): + + def test_known_conversion(self): + fn = find_unit_conversion('Pa', 'hPa') + self.assertIsNotNone(fn) + formula = fn() + self.assertIn('{var}', formula) + + def test_same_unit_no_conversion(self): + self.assertIsNone(find_unit_conversion('K', 'K')) + self.assertIsNone(find_unit_conversion('Pa', 'Pa')) + + def test_equivalent_exponent_forms_no_conversion(self): + # ``m2`` and ``m+2`` are equivalent; the resolver must not treat + # them as a unit mismatch. See _normalize_unit_string. + self.assertIsNone(find_unit_conversion('m2 s-2', 'm+2 s-2')) + self.assertIsNone(find_unit_conversion('m+2 s-2', 'm2 s-2')) + self.assertIsNone(find_unit_conversion('m2', 'm+2')) + + def test_unknown_pair(self): + self.assertIsNone(find_unit_conversion('XYZ', 'ABC')) + + def test_reverse_conversion(self): + fwd = find_unit_conversion('Pa', 'hPa') + bwd = find_unit_conversion('hPa', 'Pa') + self.assertIsNotNone(fwd) + self.assertIsNotNone(bwd) + # Forward and backward are different formulae. + self.assertNotEqual(fwd(), bwd()) + + def test_m_s_conversion(self): + fn = find_unit_conversion('m s-1', 'km h-1') + self.assertIsNotNone(fn) + + +######################################################################## +# Tests: _apply_transform_formula +######################################################################## + +class TestApplyTransformFormula(unittest.TestCase): + + def test_with_kind(self): + from metadata.unit_conversion import Pa__to__hPa + result = _apply_transform_formula(Pa__to__hPa, 'pressure', 'kind_phys') + self.assertEqual(result, '1.0E-2_kind_phys*pressure') + + def test_without_kind(self): + from metadata.unit_conversion import Pa__to__hPa + result = _apply_transform_formula(Pa__to__hPa, 'pressure', '') + self.assertEqual(result, '1.0E-2*pressure') + + def test_complex_expr(self): + from metadata.unit_conversion import mm__to__m + result = _apply_transform_formula(mm__to__m, 'arr(lb:ub, 1:nlev)', 'k') + self.assertEqual(result, '1.0E-3_k*arr(lb:ub, 1:nlev)') + + +######################################################################## +# Tests: _build_call_subscript +######################################################################## + +_HOST_DICT_SRC = ''' +[ccpp-table-properties] + name = hm + type = host +[ccpp-arg-table] + name = hm + type = host +[ lb ] + standard_name = horizontal_loop_begin + units = count + dimensions = () + type = integer + protected = True +[ ub ] + standard_name = horizontal_loop_end + units = count + dimensions = () + type = integer + protected = True +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer +[ nlevp1 ] + standard_name = vertical_interface_dimension + units = count + dimensions = () + type = integer +''' + + +class TestBuildCallSubscript(unittest.TestCase): + + def _hd(self): + tbls = _parse(_HOST_DICT_SRC) + return build_flat_host_dict(tbls, [], []) + + def test_scalar(self): + hd = self._hd() + sub, used = _build_call_subscript([], 'run', hd) + self.assertEqual(sub, '') + self.assertEqual(used, set()) + + def test_horizontal_run(self): + hd = self._hd() + sub, used = _build_call_subscript(['horizontal_dimension'], 'run', hd) + self.assertEqual(sub, '(lb:ub)') + self.assertIn('horizontal_loop_begin', used) + self.assertIn('horizontal_loop_end', used) + + def test_horizontal_init(self): + hd = self._hd() + sub, used = _build_call_subscript(['horizontal_dimension'], 'init', hd) + self.assertEqual(sub, '(lb:ub)') + + def test_vertical(self): + hd = self._hd() + sub, used = _build_call_subscript(['vertical_layer_dimension'], 'run', hd) + self.assertEqual(sub, '(1:nlev)') + self.assertIn('vertical_layer_dimension', used) + + def test_vertical_interface(self): + hd = self._hd() + sub, used = _build_call_subscript(['vertical_interface_dimension'], 'run', hd) + self.assertEqual(sub, '(1:nlevp1)') + + def test_2d_array_run(self): + hd = self._hd() + sub, used = _build_call_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], 'run', hd + ) + self.assertEqual(sub, '(lb:ub, 1:nlev)') + + def test_2d_array_init(self): + hd = self._hd() + sub, used = _build_call_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], 'init', hd + ) + self.assertEqual(sub, '(lb:ub, 1:nlev)') + + def test_arbitrary_dim(self): + src = _HOST_DICT_SRC + '''[ nspecies ] + standard_name = number_of_species + units = count + dimensions = () + type = integer +''' + hd = build_flat_host_dict(_parse(src), [], []) + sub, used = _build_call_subscript(['number_of_species'], 'run', hd) + self.assertEqual(sub, '(1:nspecies)') + self.assertIn('number_of_species', used) + + def test_unknown_dim_raises(self): + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['unknown_dimension_xyz'], 'run', hd) + self.assertIn('unknown_dimension_xyz', str(cm.exception)) + + def test_unknown_dim_lists_available_std_names(self): + """When a dim lookup fails, the error message must enumerate + every standard name the resolver can see — sorted, with source + annotation — so the user can spot typos / case mismatches / + missing declarations at a glance.""" + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['unknown_dimension_xyz'], 'run', hd) + msg = str(cm.exception) + self.assertIn('Available standard names', msg) + # The host_dict from _HOST_DICT_SRC contains horizontal_dimension + # at minimum — must be listed with a source tag. + self.assertIn('horizontal_dimension', msg) + self.assertIn('[host:', msg) + + def test_unknown_dim_did_you_mean_for_close_match(self): + """A near-miss spelling (case difference) surfaces a 'did you + mean' section above the full listing.""" + # Host has 'horizontal_dimension'; query with mixed case to + # trigger close-match detection. + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['Horizontal_Dimension'], 'run', hd) + msg = str(cm.exception) + self.assertIn('Did you mean', msg) + self.assertIn('horizontal_dimension', msg) + + def test_instance_dim_without_instance_number_raises(self): + """Host metadata referencing an instance dim must declare + ``instance_number``; otherwise the resolver should error with a + clear, actionable message rather than silently mis-resolving. + """ + # Host dict lacks both instance_number AND number_of_instances. + # Asking the resolver to subscript a (number_of_instances) dim + # must raise CCPPError. + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['number_of_instances'], 'run', hd) + msg = str(cm.exception) + self.assertIn('instance_number', msg) + self.assertIn('number_of_instances', msg) + + def test_missing_horiz_bounds_raises(self): + """Missing horizontal_loop_begin/end in host dict → CCPPError.""" + src = ''' +[ccpp-table-properties] + name = hm2 + type = host +[ccpp-arg-table] + name = hm2 + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +''' + hd = build_flat_host_dict(_parse(src), [], []) + with self.assertRaises(CCPPError): + _build_call_subscript(['horizontal_dimension'], 'run', hd) + + def test_horiz_range_ccpp_constant_one(self): + """ccpp_constant_one:horizontal_dimension resolves to lb:ub.""" + hd = self._hd() + sub, used = _build_call_subscript( + ['ccpp_constant_one:horizontal_dimension'], 'run', hd + ) + self.assertEqual(sub, '(lb:ub)') + self.assertIn('horizontal_loop_begin', used) + self.assertIn('horizontal_loop_end', used) + self.assertIn('horizontal_dimension', used) + + def test_horiz_range_integer_one(self): + """1:horizontal_dimension resolves to lb:ub.""" + hd = self._hd() + sub, used = _build_call_subscript( + ['1:horizontal_dimension'], 'run', hd + ) + self.assertEqual(sub, '(lb:ub)') + self.assertIn('horizontal_loop_begin', used) + self.assertIn('horizontal_loop_end', used) + + def test_horiz_range_bad_lower_raises(self): + """A lower bound other than 1/ccpp_constant_one for horizontal_dimension raises CCPPError.""" + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['2:horizontal_dimension'], 'run', hd) + self.assertIn('horizontal_dimension', str(cm.exception)) + self.assertIn('ccpp_constant_one', str(cm.exception)) + + def test_horiz_range_named_lower_raises(self): + """A named lower bound for horizontal_dimension (not resolving to 1) raises CCPPError.""" + hd = self._hd() + with self.assertRaises(CCPPError) as cm: + _build_call_subscript(['vertical_layer_dimension:horizontal_dimension'], 'run', hd) + self.assertIn('horizontal_dimension', str(cm.exception)) + + def test_vertical_explicit_range(self): + """General lower:upper range resolved via host_dict for vertical dims.""" + src = _HOST_DICT_SRC + '''[ bot ] + standard_name = bottom_vertical_interface_index + units = count + dimensions = () + type = integer +''' + hd = build_flat_host_dict(_parse(src), [], []) + sub, used = _build_call_subscript( + ['bottom_vertical_interface_index:vertical_interface_dimension'], 'run', hd + ) + self.assertEqual(sub, '(bot:nlevp1)') + self.assertIn('bottom_vertical_interface_index', used) + self.assertIn('vertical_interface_dimension', used) + + def test_ccpp_constant_one_as_lower_vertical(self): + """ccpp_constant_one:vertical_layer_dimension is equivalent to vertical_layer_dimension.""" + hd = self._hd() + sub, _ = _build_call_subscript( + ['ccpp_constant_one:vertical_layer_dimension'], 'run', hd + ) + self.assertEqual(sub, '(1:nlev)') + + +######################################################################## +# Tests: _build_merged_subscript (sliced local_name with std-name indices) +######################################################################## + +_HOST_SLICE_SRC = _HOST_DICT_SRC + '''[ index_qv ] + standard_name = index_of_water_vapor_specific_HUMidity + units = index + dimensions = () + type = integer + protected = True +''' + + +class TestBuildMergedSubscript(unittest.TestCase): + """Cover the slicing-with-standard-name-index code path. + + A host local_name like ``q(:,:,index_of_water_vapor_specific_HUMidity)`` + parses into ``local_subscript=[':', ':', 'index_of_water_vapor_specific_HUMidity']``. + The mixed-case token is the CCPP standard name of the index variable; the + cap must emit the host's local name (``index_qv``) and import it. + """ + + def _hd(self): + return build_flat_host_dict(_parse(_HOST_SLICE_SRC), [], []) + + def test_resolves_mixed_case_standard_name(self): + hd = self._hd() + sub, used = _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', 'index_of_water_vapor_specific_HUMidity'], + 'run', hd, + ) + self.assertEqual(sub, '(lb:ub, 1:nlev, index_qv)') + # Lowercased standard name is reported as used so the cap emits a + # `use test_host_mod, only: index_qv` line. + self.assertIn('index_of_water_vapor_specific_humidity', used) + + def test_integer_literal_passthrough(self): + hd = self._hd() + sub, used = _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', '1'], + 'run', hd, + ) + self.assertEqual(sub, '(lb:ub, 1:nlev, 1)') + + def test_unknown_index_raises(self): + hd = self._hd() + with self.assertRaises(CCPPError): + _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', 'no_such_standard_name'], + 'run', hd, + ) + + +######################################################################## +# Tests: _translate_active_expr +######################################################################## + +class TestTranslateActiveExpr(unittest.TestCase): + + def _hd(self): + src = ''' +[ccpp-table-properties] + name = m + type = host +[ccpp-arg-table] + name = m + type = host +[ do_something ] + standard_name = flag_for_something + units = flag + dimensions = () + type = logical +''' + return build_flat_host_dict(_parse(src), [], []) + + def test_empty(self): + hd = self._hd() + self.assertEqual(_translate_active_expr('', hd), '') + + def test_simple_replacement(self): + hd = self._hd() + result = _translate_active_expr('flag_for_something', hd) + self.assertEqual(result, 'do_something') + + def test_in_expression(self): + hd = self._hd() + result = _translate_active_expr('.not. flag_for_something', hd) + self.assertEqual(result, '.not. do_something') + + def test_unknown_preserved(self): + hd = self._hd() + result = _translate_active_expr('unknown_stdname .eqv. .true.', hd) + self.assertEqual(result, 'unknown_stdname .eqv. .true.') + + def test_ddt_component_flag_uses_full_access_path(self): + """A flag declared as a DDT-component must translate to the full + access path, not the bare component name — otherwise the + generated cap references an undefined symbol. + """ + ddt_src = ''' +[ccpp-table-properties] + name = inst_type + type = ddt +[ccpp-arg-table] + name = inst_type + type = ddt +[opt_array_flag] + standard_name = flag_for_opt_array + units = 1 + dimensions = () + type = logical +''' + host_src = ''' +[ccpp-table-properties] + name = data_mod + type = host +[ccpp-arg-table] + name = data_mod + type = host +[ncols] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ninstances] + standard_name = number_of_instances + units = count + dimensions = () + type = integer +[instance_data] + standard_name = instance_data + units = ddt + dimensions = (number_of_instances) + type = inst_type +''' + hd = build_flat_host_dict(_parse(host_src), [], _parse(ddt_src)) + result = _translate_active_expr('(flag_for_opt_array)', hd) + # No instance_number declared in this fixture → falls back to (1). + self.assertEqual(result, '(instance_data(1)%opt_array_flag)') + + +######################################################################## +# Tests: _substitute_instance_idx +######################################################################## + +class TestSubstituteInstanceIdx(unittest.TestCase): + + def _hd_with_inst(self, inst_local: str = 'inst'): + src = ''' +[ccpp-table-properties] + name = ctrl + type = control +[ccpp-arg-table] + name = ctrl + type = control +[ {inst} ] + standard_name = instance_number + units = 1 + dimensions = () + type = integer +'''.format(inst=inst_local) + return build_flat_host_dict([], _parse(src), []) + + def _hd_without_inst(self): + # An empty host_dict — no instance_number, no number_of_instances. + return {} + + def test_no_template_pass_through(self): + hd = self._hd_with_inst() + self.assertEqual( + _substitute_instance_idx('ncols', hd), 'ncols', + ) + self.assertEqual( + _substitute_instance_idx('gfs%phii(lb:ub)', hd), + 'gfs%phii(lb:ub)', + ) + + def test_template_resolved_to_local_name(self): + hd = self._hd_with_inst(inst_local='inst') + self.assertEqual( + _substitute_instance_idx( + 'instance_data(instance_number)%data_array2', hd, + ), + 'instance_data(inst)%data_array2', + ) + + def test_template_uses_local_name_alias(self): + hd = self._hd_with_inst(inst_local='my_inst') + self.assertEqual( + _substitute_instance_idx( + 'instance_data(instance_number)', hd, + ), + 'instance_data(my_inst)', + ) + + def test_template_resolves_to_one_when_no_pair(self): + # Host did not declare instance_number; single-instance API, + # internal arrays sized to 1. + hd = self._hd_without_inst() + self.assertEqual( + _substitute_instance_idx( + 'instance_data(instance_number)%data_array2', hd, + ), + 'instance_data(1)%data_array2', + ) + + +######################################################################## +# Tests: _root_symbol +######################################################################## + +class TestRootSymbol(unittest.TestCase): + + def test_plain(self): + self.assertEqual(_root_symbol('ncols'), 'ncols') + + def test_component(self): + self.assertEqual(_root_symbol('gfs_statein%phii'), 'gfs_statein') + + def test_subscript(self): + self.assertEqual(_root_symbol('gfs_statein(instance_number)%phii'), 'gfs_statein') + + def test_deep(self): + self.assertEqual(_root_symbol('outer%middle%inner'), 'outer') + + +######################################################################## +# Tests: _local_name_conflict +######################################################################## + +class TestLocalNameConflict(unittest.TestCase): + + def test_no_conflict(self): + self.assertEqual(_local_name_conflict('phii_l', set()), 'phii_l') + + def test_conflict_adds_2(self): + self.assertEqual(_local_name_conflict('phii_l', {'phii_l'}), 'phii_2_l') + + def test_conflict_adds_3(self): + self.assertEqual( + _local_name_conflict('phii_l', {'phii_l', 'phii_2_l'}), 'phii_3_l' + ) + + +######################################################################## +# Tests: _resolve_one_arg (single argument) +######################################################################## + +class TestResolveOneArg(unittest.TestCase): + + def _host_dict(self): + return _load_full_host_dict() + + def _scheme_var(self, local, std_name, intent='in', units='1', + dims='()', type_='integer', kind='', optional=False): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar(local, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', units, ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', type_, ctx) + v.set_attr('intent', intent, ctx) + if kind: + v.set_attr('kind', kind, ctx) + if optional: + v.set_attr('optional', 'True', ctx) + return v + + def test_case1_direct_host(self): + """Case 1: scalar host variable, no transform.""" + hd = self._host_dict() + sv = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count') + arg = _resolve_one_arg(sv, 'run', hd, {}, 'my_scheme', set()) + self.assertEqual(arg.source, 'host') + self.assertEqual(arg.transform_case, 1) + self.assertEqual(arg.call_expr, 'im') + self.assertFalse(arg.needs_transform) + + def test_case1_control_var(self): + """Control variable → source='control', no USE module.""" + hd = self._host_dict() + sv = self._scheme_var('thread_num', 'thread_number', 'in', '1') + arg = _resolve_one_arg(sv, 'run', hd, {}, 'my_scheme', set()) + self.assertEqual(arg.source, 'control') + self.assertIsNone(arg.module_name) + + def test_case1_2d_array_run(self): + """2D array in run phase → subscript applied.""" + hd = self._host_dict() + sv = self._scheme_var('temp', 'air_temperature', 'inout', 'K', + '(horizontal_loop_extent, vertical_layer_dimension)', + 'real', 'kind_phys') + arg = _resolve_one_arg(sv, 'run', hd, {}, 'my_scheme', set()) + # access_path = 'gt0', subscript = '(lb:ub, 1:nlev)' + self.assertEqual(arg.call_expr, 'gt0(lb:ub, 1:nlev)') + self.assertEqual(arg.transform_case, 1) + + def test_case2_suite_owned(self): + """Case 2: not in host, first use intent(out) → creates SuiteVar.""" + hd = self._host_dict() + sv = self._scheme_var('new_var', 'brand_new_standard_name', 'out', 'K', + '()', 'real', 'kind_phys') + suite_vars: dict = {} + arg = _resolve_one_arg(sv, 'run', hd, suite_vars, 'my_scheme', set()) + self.assertEqual(arg.source, 'suite') + self.assertIn('brand_new_standard_name', suite_vars) + self.assertIsNotNone(arg.suite_var) + + def test_case3_not_found_intent_in_raises(self): + """Case 3: not in host, intent(in) → CCPPError.""" + hd = self._host_dict() + sv = self._scheme_var('missing', 'totally_missing_stdname', 'in', 'K') + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'bad_scheme', set()) + self.assertIn('totally_missing_stdname', str(cm.exception)) + + def test_case4_suite_data_reuse(self): + """Case 4: variable already in suite_vars (from prior scheme).""" + hd = self._host_dict() + sv_creator = self._scheme_var('new_var', 'interstitial_var', 'out', 'K', + '()', 'real', 'kind_phys') + suite_vars: dict = {} + _resolve_one_arg(sv_creator, 'run', hd, suite_vars, 'scheme_a', set()) + + sv_reader = self._scheme_var('interstitial_in', 'interstitial_var', 'in', 'K', + '()', 'real', 'kind_phys') + arg = _resolve_one_arg(sv_reader, 'run', hd, suite_vars, 'scheme_b', set()) + self.assertEqual(arg.source, 'suite') + self.assertIsNotNone(arg.suite_var) + + def test_unit_transform_detected(self): + """Units differ → transformation required.""" + hd = self._host_dict() + # air_temperature is in K, request it in Pa... but there's no K→Pa conversion. + # Use a case with a known conversion: host has 'K', scheme expects 'K' → no xform. + # Let's not test K→Pa (no conversion), test a successful mismatch. + # Instead, add a variable with Pa units to the host dict. + src = ''' +[ccpp-table-properties] + name = press_mod + type = host +[ccpp-arg-table] + name = press_mod + type = host +[ pres ] + standard_name = air_pressure + units = Pa + dimensions = () + type = real + kind = kind_phys +''' + extra_tbls = _parse(src) + extra_hd = build_flat_host_dict(extra_tbls, [], {}) + combined = {**hd, **extra_hd} + + sv = self._scheme_var('p_hpa', 'air_pressure', 'in', 'hPa', '()', 'real', 'kind_phys') + arg = _resolve_one_arg(sv, 'run', combined, {}, 'my_scheme', set()) + self.assertTrue(arg.needs_unit_transform) + self.assertEqual(arg.transform_case, 3) + self.assertIn('temp_name', arg.__dataclass_fields__) # has temp_name field + self.assertTrue(arg.temp_name) + + def test_no_transform_same_units(self): + """Identical units → no transformation.""" + hd = self._host_dict() + sv = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count') + arg = _resolve_one_arg(sv, 'run', hd, {}, 'my_scheme', set()) + self.assertFalse(arg.needs_transform) + + def test_unknown_unit_mismatch_raises(self): + """Units differ but no conversion known → CCPPError.""" + src = ''' +[ccpp-table-properties] + name = mod + type = host +[ccpp-arg-table] + name = mod + type = host +[ val ] + standard_name = some_value + units = xyz_unit + dimensions = () + type = real + kind = kind_phys +''' + hd = build_flat_host_dict(_parse(src), [], {}) + sv = self._scheme_var('v', 'some_value', 'in', 'abc_unit', '()', 'real', 'kind_phys') + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'bad_scheme', set()) + self.assertIn('xyz_unit', str(cm.exception)) + + def test_optional_sets_ptr_name(self): + """Optional argument → ptr_name set.""" + hd = self._host_dict() + sv = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count', + optional=True) + arg = _resolve_one_arg(sv, 'run', hd, {}, 'my_scheme', set()) + self.assertTrue(arg.is_optional) + self.assertTrue(arg.ptr_name) + self.assertEqual(arg.transform_case, 2) + + +######################################################################## +# Tests: vertical-flip transform (top_at_one mismatch) +######################################################################## + +class TestVerticalFlipTransform(unittest.TestCase): + """When host and scheme disagree on ``top_at_one`` for a variable that + carries a vertical dimension, the resolver emits a flipped host-side + subscript and turns on the temp/transform pipeline so the call site + copies data through a contiguous local in scheme order. + """ + + def _build_host_and_scheme( + self, + host_top_at_one: bool = False, + scheme_top_at_one: bool = False, + host_units: str = 'K', + scheme_units: str = 'K', + intent: str = 'inout', + ): + host_src = ''' +[ccpp-table-properties] + name = mod + type = host +[ccpp-arg-table] + name = mod + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ im ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer +[ gt0 ] + standard_name = air_temperature + units = {units} + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + top_at_one = {top} +'''.format(units=host_units, top='True' if host_top_at_one else 'False') + + ctrl_src = ''' +[ccpp-table-properties] + name = ctrl + type = control +[ccpp-arg-table] + name = ctrl + type = control +[ lb ] + standard_name = horizontal_loop_begin + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + units = index + dimensions = () + type = integer +''' + hd = build_flat_host_dict(_parse(host_src), _parse(ctrl_src), []) + + from metadata.metadata_table import MetaVar + ctx = _ctx() + sv = MetaVar('temp', ctx) + sv.set_attr('standard_name', 'air_temperature', ctx) + sv.set_attr('units', scheme_units, ctx) + sv.set_attr('dimensions', + '(horizontal_loop_extent, vertical_layer_dimension)', ctx) + sv.set_attr('type', 'real', ctx) + sv.set_attr('kind', 'kind_phys', ctx) + sv.set_attr('intent', intent, ctx) + if scheme_top_at_one: + sv.set_attr('top_at_one', 'True', ctx) + return hd, sv + + def test_no_flip_when_both_false(self): + hd, sv = self._build_host_and_scheme(False, False) + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertFalse(arg.needs_vert_flip) + self.assertEqual(arg.transform_case, 1) + self.assertEqual(arg.call_expr, 'gt0(lb:ub, 1:nlev)') + + def test_no_flip_when_both_true(self): + hd, sv = self._build_host_and_scheme(True, True) + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertFalse(arg.needs_vert_flip) + self.assertEqual(arg.transform_case, 1) + self.assertEqual(arg.call_expr, 'gt0(lb:ub, 1:nlev)') + + def test_flip_when_host_false_scheme_true(self): + hd, sv = self._build_host_and_scheme(False, True) + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertTrue(arg.needs_vert_flip) + self.assertTrue(arg.needs_transform) + # Transform pipeline: temp local, transform_case 3, no unit conv. + self.assertEqual(arg.transform_case, 3) + self.assertTrue(arg.temp_name) + self.assertFalse(arg.needs_unit_transform) + self.assertFalse(arg.needs_kind_transform) + # Host-side subscript carries reverse stride at the vdim position. + self.assertEqual(arg.call_expr, 'gt0(lb:ub, nlev:1:-1)') + # Forward (pre-call) copies host → temp; backward (post-call) + # copies temp → host using the same flipped LHS. + self.assertEqual(arg.unit_forward, 'gt0(lb:ub, nlev:1:-1)') + self.assertEqual(arg.unit_backward, 'temp_l') + + def test_flip_when_host_true_scheme_false(self): + hd, sv = self._build_host_and_scheme(True, False) + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertTrue(arg.needs_vert_flip) + self.assertEqual(arg.call_expr, 'gt0(lb:ub, nlev:1:-1)') + + def test_flip_composes_with_unit_conversion(self): + """Mismatched top_at_one AND a unit conversion → the unit-forward + formula is applied to the flipped call_expr; the temp pattern is + a single combined assignment.""" + hd, sv = self._build_host_and_scheme(False, True, + host_units='Pa', scheme_units='hPa') + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertTrue(arg.needs_vert_flip) + self.assertTrue(arg.needs_unit_transform) + self.assertEqual(arg.transform_case, 3) + # The flipped subscript is embedded inside the unit-conversion expr. + self.assertIn('gt0(lb:ub, nlev:1:-1)', arg.unit_forward) + # And backward uses the temp's regular order multiplied by the + # inverse factor; LHS at the post-call site uses the same flipped + # subscript. + self.assertIn('temp_l', arg.unit_backward) + self.assertEqual(arg.call_expr, 'gt0(lb:ub, nlev:1:-1)') + + def test_intent_in_only_emits_forward(self): + hd, sv = self._build_host_and_scheme(False, True, intent='in') + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertTrue(arg.needs_vert_flip) + self.assertEqual(arg.unit_forward, 'gt0(lb:ub, nlev:1:-1)') + self.assertEqual(arg.unit_backward, '') + + def test_intent_out_only_emits_backward(self): + hd, sv = self._build_host_and_scheme(False, True, intent='out') + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertTrue(arg.needs_vert_flip) + self.assertEqual(arg.unit_forward, '') + self.assertEqual(arg.unit_backward, 'temp_l') + + def test_no_flip_on_scalar(self): + """top_at_one on a scalar (no vertical dim) is a no-op — flip needs + a vertical-axis dimension to operate on. + """ + host_src = ''' +[ccpp-table-properties] + name = mod + type = host +[ccpp-arg-table] + name = mod + type = host +[ scalar_thing ] + standard_name = some_scalar + units = 1 + dimensions = () + type = real + kind = kind_phys + top_at_one = True +''' + hd = build_flat_host_dict(_parse(host_src), [], []) + + from metadata.metadata_table import MetaVar + ctx = _ctx() + sv = MetaVar('s', ctx) + sv.set_attr('standard_name', 'some_scalar', ctx) + sv.set_attr('units', '1', ctx) + sv.set_attr('dimensions', '()', ctx) + sv.set_attr('type', 'real', ctx) + sv.set_attr('kind', 'kind_phys', ctx) + sv.set_attr('intent', 'in', ctx) + # Scheme leaves top_at_one at default (False). + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertFalse(arg.needs_vert_flip) + + +######################################################################## +# Tests: character kind (len=) validation +######################################################################## + +class TestCharacterKindResolution(unittest.TestCase): + """len=* in scheme is always compatible; mismatched specific lengths are errors.""" + + def _host_with_char(self, kind='len=512'): + """Build a host dict with a character variable of the given kind.""" + src = ''' +[ccpp-table-properties] + name = hmod + type = host +[ccpp-arg-table] + name = hmod + type = host +[ msg ] + standard_name = my_message + units = none + dimensions = () + type = character + kind = {kind} +'''.format(kind=kind) + tbls = _parse(src) + return build_flat_host_dict(tbls, [], []) + + def _scheme_var_char(self, local, std_name, kind): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar(local, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', 'none', ctx) + v.set_attr('dimensions', '()', ctx) + v.set_attr('type', 'character', ctx) + v.set_attr('kind', kind, ctx) + v.set_attr('intent', 'out', ctx) + return v + + def test_len_star_compatible_with_len_512(self): + """len=* in scheme is always compatible — no transform, no error.""" + hd = self._host_with_char('len=512') + sv = self._scheme_var_char('msg', 'my_message', 'len=*') + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertFalse(arg.needs_kind_transform) + self.assertEqual(arg.transform_case, 1) + self.assertEqual(arg.temp_name, '') + + def test_len_match_compatible(self): + """Same specific len=N in both host and scheme — no transform.""" + hd = self._host_with_char('len=512') + sv = self._scheme_var_char('msg', 'my_message', 'len=512') + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertFalse(arg.needs_kind_transform) + + def test_len_star_in_host_no_error(self): + """len=* in the host is also fine (assumed-length dummy everywhere).""" + hd = self._host_with_char('len=*') + sv = self._scheme_var_char('msg', 'my_message', 'len=*') + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertFalse(arg.needs_kind_transform) + + def test_mismatched_specific_lengths_raises(self): + """Specific len=128 vs len=512 is a metadata error.""" + hd = self._host_with_char('len=512') + sv = self._scheme_var_char('msg', 'my_message', 'len=128') + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'bad_scheme', set()) + self.assertIn('len=512', str(cm.exception)) + self.assertIn('len=128', str(cm.exception)) + + def test_len_star_host_specific_scheme_raises(self): + """len=* in host but specific len=256 in scheme — error.""" + hd = self._host_with_char('len=*') + sv = self._scheme_var_char('msg', 'my_message', 'len=256') + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'bad_scheme', set()) + self.assertIn('len=256', str(cm.exception)) + + +######################################################################## +# Integration tests: resolve_suite +######################################################################## + +class TestResolveSuite(unittest.TestCase): + + def _resolve(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + return resolve_suite(suite, store, hd), hd + + def test_groups_present(self): + sr, _ = self._resolve() + self.assertEqual(sr.suite_name, 'test_simple') + self.assertEqual(len(sr.groups), 1) + self.assertEqual(sr.groups[0].group_name, 'physics') + + def test_run_phase_calls(self): + sr, _ = self._resolve() + rg = sr.groups[0] + self.assertIn('run', rg.phase_calls) + calls = rg.phase_calls['run'] + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0].scheme_name, 'temp_calc_adjust') + + def test_run_phase_args(self): + sr, _ = self._resolve() + calls = sr.groups[0].phase_calls['run'] + args = {a.scheme_local_name: a for a in calls[0].args} + # im = horizontal_dimension: scalar arg is synthesised from the loop + # bounds so the scheme sees the per-call chunk extent, not host ncols. + self.assertIn('im', args) + self.assertEqual(args['im'].call_expr, '(ub - lb + 1)') + # temp = air_temperature (2D, run phase subscript) + self.assertIn('temp', args) + self.assertEqual(args['temp'].call_expr, 'gt0(lb:ub, 1:nlev)') + # errmsg and errflg + self.assertIn('errmsg', args) + self.assertIn('errflg', args) + + def test_init_phase_calls(self): + sr, _ = self._resolve() + rg = sr.groups[0] + self.assertIn('init', rg.phase_calls) + calls = rg.phase_calls['init'] + self.assertEqual(calls[0].scheme_name, 'temp_calc_adjust') + + def test_init_phase_horizontal_subscript(self): + """In init phase, scalar horizontal_dimension does not produce an lb:ub slice.""" + sr, _ = self._resolve() + rg = sr.groups[0] + # temp_calc_adjust_init doesn't have temp, but the general rule should hold + # for any 2D variable in a non-run phase: no lb:ub slice in init args. + # (Scalar horizontal_dimension args are synthesised as (ub - lb + 1), + # which collapses to ncols in non-run phases but never contains + # the substring 'lb:ub'.) + calls = rg.phase_calls.get('init', []) + for rc in calls: + for arg in rc.args: + self.assertNotIn('lb:ub', arg.call_expr) + + def test_no_suite_vars(self): + """All variables in temp_calc_adjust are provided by the host.""" + sr, _ = self._resolve() + self.assertEqual(sr.suite_vars, {}) + + def test_used_modules(self): + sr, _ = self._resolve() + calls = sr.groups[0].phase_calls['run'] + mods = calls[0].used_modules + # host_phys should appear (air_temperature, horizontal_dimension, etc.) + self.assertIn('host_phys', mods) + + def test_control_args_no_module(self): + sr, _ = self._resolve() + calls = sr.groups[0].phase_calls['run'] + ctrl = [a for a in calls[0].args if a.source == 'control'] + for c in ctrl: + self.assertIsNone(c.module_name) + + +class TestDedupSchemeNames(unittest.TestCase): + """Unit tests for the non-run-phase dedup helper.""" + + def test_no_duplicates_passthrough(self): + self.assertEqual(_dedup_scheme_names(['a', 'b', 'c']), ['a', 'b', 'c']) + + def test_consecutive_duplicate_collapsed(self): + self.assertEqual(_dedup_scheme_names(['a', 'a', 'b']), ['a', 'b']) + + def test_non_consecutive_duplicate_collapsed(self): + # First occurrence kept, later ones dropped. + self.assertEqual( + _dedup_scheme_names(['a', 'b', 'a', 'c', 'b']), + ['a', 'b', 'c'], + ) + + def test_empty_input(self): + self.assertEqual(_dedup_scheme_names([]), []) + + +class TestResolveSuiteInitFinalSchemes(unittest.TestCase): + """Suite-level ```` / ```` schemes resolve to + ``ResolvedCall`` objects attached to ``SuiteResolution.suite_init_call`` + and ``.suite_final_call`` respectively. A missing init/final phase + on the named scheme raises ``CCPPError`` at resolve time.""" + + def _build_store(self, scheme_meta_text: str): + from metadata.metadata_table import _parse_lines + from metadata.variable_resolver import SchemeStore + tbls = _parse_lines( + scheme_meta_text.splitlines(keepends=True), 'sch.meta', + ) + return SchemeStore.build_from(tbls) + + def _resolve(self, suite_xml: str, scheme_meta_text: str): + import tempfile, os, logging + from test_suite_resolver import _load_full_host_dict + from generator.suite_xml import parse_suite_xml + from generator.suite_resolver import resolve_suite + hd = _load_full_host_dict() + store = self._build_store(scheme_meta_text) + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, 's.xml') + with open(path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(path, tmp, logging.getLogger('t'), + skip_validation=True) + return resolve_suite(suite, store, hd) + + _SCHEME_META = ( + '[ccpp-table-properties]\n' + ' name = init_final_test\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = init_final_test_init\n' + ' type = scheme\n' + '[errmsg]\n' + ' standard_name = ccpp_error_message\n' + ' units = none\n' + ' dimensions = ()\n' + ' type = character\n' + ' kind = len=512\n' + ' intent = out\n' + '[errflg]\n' + ' standard_name = ccpp_error_code\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = out\n' + '[ccpp-arg-table]\n' + ' name = init_final_test_final\n' + ' type = scheme\n' + '[errmsg]\n' + ' standard_name = ccpp_error_message\n' + ' units = none\n' + ' dimensions = ()\n' + ' type = character\n' + ' kind = len=512\n' + ' intent = out\n' + '[errflg]\n' + ' standard_name = ccpp_error_code\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = out\n' + ) + + def test_init_call_attached(self): + suite_xml = ( + '\n' + '\n' + ' init_final_test\n' + ' \n' + '\n' + ) + sr = self._resolve(suite_xml, self._SCHEME_META) + self.assertIsNotNone(sr.suite_init_call) + self.assertEqual(sr.suite_init_call.scheme_name, 'init_final_test') + self.assertEqual(sr.suite_init_call.phase, 'init') + self.assertIsNone(sr.suite_final_call) + + def test_final_call_attached(self): + suite_xml = ( + '\n' + '\n' + ' \n' + ' init_final_test\n' + '\n' + ) + sr = self._resolve(suite_xml, self._SCHEME_META) + self.assertIsNone(sr.suite_init_call) + self.assertIsNotNone(sr.suite_final_call) + self.assertEqual(sr.suite_final_call.scheme_name, 'init_final_test') + self.assertEqual(sr.suite_final_call.phase, 'final') + + def test_both_attached(self): + suite_xml = ( + '\n' + '\n' + ' init_final_test\n' + ' \n' + ' init_final_test\n' + '\n' + ) + sr = self._resolve(suite_xml, self._SCHEME_META) + self.assertIsNotNone(sr.suite_init_call) + self.assertIsNotNone(sr.suite_final_call) + + def test_init_scheme_without_init_phase_raises(self): + """If the named scheme has no ``init`` phase in its metadata, + the resolver errors out with a clear message — silent drop + would let the SDF declaration go unused at runtime.""" + # Same scheme metadata but with init removed: only ``final``. + run_only = ( + '[ccpp-table-properties]\n' + ' name = init_final_test\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = init_final_test_run\n' + ' type = scheme\n' + '[errmsg]\n' + ' standard_name = ccpp_error_message\n' + ' units = none\n' + ' dimensions = ()\n' + ' type = character\n' + ' kind = len=512\n' + ' intent = out\n' + '[errflg]\n' + ' standard_name = ccpp_error_code\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = out\n' + ) + suite_xml = ( + '\n' + '\n' + ' init_final_test\n' + ' \n' + '\n' + ) + with self.assertRaises(CCPPError) as cm: + self._resolve(suite_xml, run_only) + msg = str(cm.exception) + self.assertIn('init_final_test', msg) + self.assertIn('init', msg) + + +class TestResolveSuiteDuplicateScheme(unittest.TestCase): + """Resolve a suite where one scheme appears twice in the same group. + + Run phase must preserve both call sites (the scheme runs once per + iteration, e.g. once per constituent); non-run phases must collapse to + a single call so register/init/final entry points fire exactly once + per group. Matches the advection-test pattern where + ``apply_constituent_tendencies`` is listed twice in ``physics``. + """ + + _SUITE_XML = ( + '\n' + '\n' + ' \n' + ' temp_calc_adjust\n' + ' temp_calc_adjust\n' + ' \n' + '\n' + ) + + def _resolve(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + from generator.suite_xml import parse_suite_xml + import logging + logger = logging.getLogger('test') + with tempfile.TemporaryDirectory() as tmpdir: + xml_path = os.path.join(tmpdir, 'suite_dup.xml') + with open(xml_path, 'w') as fh: + fh.write(self._SUITE_XML) + suite = parse_suite_xml(xml_path, tmpdir, logger, + skip_validation=True) + return resolve_suite(suite, store, hd) + + def test_resolve_does_not_raise(self): + # The pre-fix behaviour was a CCPPError on the second occurrence. + self.assertIsNotNone(self._resolve()) + + def test_run_phase_preserves_both_calls(self): + sr = self._resolve() + run_calls = list(iter_phase_calls(sr.groups[0].phase_calls['run'])) + self.assertEqual(len(run_calls), 2) + self.assertEqual( + [c.scheme_name for c in run_calls], + ['temp_calc_adjust', 'temp_calc_adjust'], + ) + + def test_init_phase_dedupes(self): + sr = self._resolve() + init_calls = list(iter_phase_calls(sr.groups[0].phase_calls['init'])) + self.assertEqual(len(init_calls), 1) + self.assertEqual(init_calls[0].scheme_name, 'temp_calc_adjust') + + def test_final_phase_dedupes(self): + sr = self._resolve() + final_calls = list(iter_phase_calls(sr.groups[0].phase_calls['final'])) + self.assertEqual(len(final_calls), 1) + + +######################################################################## +# Tests: group cap output +######################################################################## + +class TestDimDeclLocal(unittest.TestCase): + """``_dim_decl_local`` — local-name dim resolution with horizontal-chunk + special case. The horizontal dim must emit ``lb:ub`` (using the + host's local names for horizontal_loop_begin / horizontal_loop_end) + because the temp must match the chunk slice the scheme receives at + the call site, not the full extent.""" + + def setUp(self): + self.hd = _load_full_host_dict() + + def test_empty(self): + self.assertEqual(_dim_decl_local([], self.hd), '') + + def test_horizontal_dimension_uses_chunk_bounds(self): + # control_full.meta has horizontal_loop_begin → 'lb', + # horizontal_loop_end → 'ub'. + self.assertEqual( + _dim_decl_local(['horizontal_dimension'], self.hd), + ', dimension(lb:ub)', + ) + + def test_horizontal_loop_extent_uses_chunk_bounds(self): + # Same special case for the alternative dim std name. + self.assertEqual( + _dim_decl_local(['horizontal_loop_extent'], self.hd), + ', dimension(lb:ub)', + ) + + def test_vertical_dim_uses_local_name(self): + # No special case for vertical dims — host's local name only. + self.assertEqual( + _dim_decl_local(['vertical_layer_dimension'], self.hd), + ', dimension(nlev)', + ) + + def test_mixed_horiz_vert(self): + self.assertEqual( + _dim_decl_local( + ['horizontal_dimension', 'vertical_layer_dimension'], self.hd, + ), + ', dimension(lb:ub, nlev)', + ) + + def test_unknown_dim_falls_back_to_std_name(self): + # No entry in host_dict → use the std name verbatim. + self.assertEqual( + _dim_decl_local(['some_unknown_dim'], self.hd), + ', dimension(some_unknown_dim)', + ) + + +class TestCollectKindsUsed(unittest.TestCase): + """``_collect_kinds_used`` must collect only kind *symbols* — integer + literals and ``len=...`` specifiers are NOT module symbols and must + not enter the ``use ccpp_kinds, only: ...`` list, even though they + do flow through to the temp declarations and numeric-literal kind + suffixes (which is correct Fortran).""" + + def _fake_arg(self, kind_scheme: str = '', kind_host: str = '', + temp_name: str = 'foo_l'): + """Build a minimal ResolvedArg stand-in with the fields + ``_collect_kinds_used`` reads.""" + from unittest.mock import MagicMock + a = MagicMock() + a.temp_name = temp_name + a.kind_scheme = kind_scheme + host = MagicMock() + host.kind = kind_host + a.host_entry = host + return a + + def _fake_rg(self, args): + from unittest.mock import MagicMock + # ``iter_phase_calls`` does ``isinstance(item, ResolvedCall)`` so a + # MagicMock won't pass; build a real ResolvedCall (the only field + # ``_collect_kinds_used`` reads is ``args``). + rc = ResolvedCall(scheme_name='s', phase='run', args=args) + rg = MagicMock() + rg.phase_calls = {'run': [rc]} + return rg + + def test_keeps_kind_symbols(self): + args = [self._fake_arg(kind_scheme='kind_phys'), + self._fake_arg(kind_scheme='kind_dyn')] + self.assertEqual(_collect_kinds_used(self._fake_rg(args)), + ['kind_dyn', 'kind_phys']) + + def test_drops_integer_literal_kinds(self): + """``kind = 8`` is valid Fortran but not a module symbol — must + not appear in the USE list.""" + args = [self._fake_arg(kind_scheme='8')] + self.assertEqual(_collect_kinds_used(self._fake_rg(args)), []) + + def test_drops_character_len_specifiers(self): + args = [self._fake_arg(kind_scheme='len=512')] + self.assertEqual(_collect_kinds_used(self._fake_rg(args)), []) + + def test_mixed_set(self): + args = [self._fake_arg(kind_scheme='kind_phys'), + self._fake_arg(kind_scheme='8'), + self._fake_arg(kind_scheme='len=*'), + self._fake_arg(kind_scheme='kind_phys')] + self.assertEqual(_collect_kinds_used(self._fake_rg(args)), + ['kind_phys']) + + def test_no_temp_name_skipped(self): + # Args without a transformation temp are irrelevant — no kind + # parameter is emitted into a declaration for them. + args = [self._fake_arg(kind_scheme='kind_dyn', temp_name='')] + self.assertEqual(_collect_kinds_used(self._fake_rg(args)), []) + + +class TestTransformComment(unittest.TestCase): + """The trailing inline comment must list every active transform, but + must suppress "unit conversion" when the rendered formula is the + identity (formula ``'{var}'`` for dimensionally-equivalent units). + """ + + def _arg(self, **kwargs): + from unittest.mock import MagicMock + a = MagicMock() + a.needs_unit_transform = kwargs.get('needs_unit_transform', False) + a.needs_kind_transform = kwargs.get('needs_kind_transform', False) + a.needs_vert_flip = kwargs.get('needs_vert_flip', False) + a.unit_forward = kwargs.get('unit_forward', '') + a.unit_backward = kwargs.get('unit_backward', '') + a.call_expr = kwargs.get('call_expr', '') + a.temp_name = kwargs.get('temp_name', '') + a.kind_host = kwargs.get('kind_host', '') + a.kind_scheme = kwargs.get('kind_scheme', '') + return a + + def test_no_transforms_returns_empty(self): + self.assertEqual(_transform_comment(self._arg()), '') + + def test_identity_forward_suppressed(self): + """Forward formula returns the call_expr unchanged → no comment.""" + a = self._arg( + needs_unit_transform=True, + unit_forward='gt0(lb:ub, 1:nlev)', + call_expr='gt0(lb:ub, 1:nlev)', + kind_host='kind_phys', kind_scheme='kind_phys', + ) + self.assertEqual(_transform_comment(a, reverse=False), '') + + def test_identity_backward_suppressed(self): + """Backward formula returns the temp_name unchanged → no comment.""" + a = self._arg( + needs_unit_transform=True, + unit_backward='foo_l', + temp_name='foo_l', + kind_host='kind_phys', kind_scheme='kind_phys', + ) + self.assertEqual(_transform_comment(a, reverse=True), '') + + def test_non_identity_forward_emitted(self): + """Forward formula scales the call_expr → comment lists the + unit conversion.""" + a = self._arg( + needs_unit_transform=True, + unit_forward='1.0E-3_kind_phys*gt0(lb:ub)', + call_expr='gt0(lb:ub)', + kind_host='kind_phys', kind_scheme='kind_phys', + ) + self.assertIn('unit conversion', _transform_comment(a, reverse=False)) + + def test_non_identity_backward_emitted(self): + a = self._arg( + needs_unit_transform=True, + unit_backward='1.0E+3_kind_phys*foo_l', + temp_name='foo_l', + kind_host='kind_phys', kind_scheme='kind_phys', + ) + self.assertIn('unit conversion', _transform_comment(a, reverse=True)) + + def test_vert_flip_alone_emits_flip_only(self): + """A pure vertical flip (identity unit conversion, no kind change) + still gets a comment — and it mentions only the flip, not a + spurious "unit conversion".""" + a = self._arg( + needs_vert_flip=True, + unit_forward='gt0(lb:ub, nlev:1:-1)', + call_expr='gt0(lb:ub, nlev:1:-1)', + ) + self.assertEqual(_transform_comment(a, reverse=False), + '! vertical flip (top_at_one mismatch)') + + def test_unit_and_flip_both_listed(self): + a = self._arg( + needs_unit_transform=True, + needs_vert_flip=True, + unit_forward='1.0E-3_kind_phys*gt0(lb:ub, nlev:1:-1)', + call_expr='gt0(lb:ub, nlev:1:-1)', + kind_host='kind_phys', kind_scheme='kind_phys', + ) + comment = _transform_comment(a, reverse=False) + self.assertIn('unit conversion', comment) + self.assertIn('vertical flip', comment) + + +class TestFortranTypeStr(unittest.TestCase): + + def test_real_with_kind(self): + self.assertEqual(_fortran_type_str('real', 'kind_phys'), 'real(kind=kind_phys)') + + def test_integer(self): + self.assertEqual(_fortran_type_str('integer', ''), 'integer') + + def test_character_with_len(self): + self.assertEqual(_fortran_type_str('character', 'len=512'), 'character(len=512)') + + def test_ddt_gets_type_wrap(self): + self.assertEqual(_fortran_type_str('my_ddt_type', ''), 'type(my_ddt_type)') + + def test_ddt_already_wrapped(self): + self.assertEqual(_fortran_type_str('type(my_ddt)', ''), 'type(my_ddt)') + + +class TestGenerateGroupCap(unittest.TestCase): + + def _resolve_and_generate(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + sr = resolve_suite(suite, store, hd) + rg = sr.groups[0] + lines = _generate_group_cap('test_simple', 'physics', rg, hd) + return lines + + def test_module_header(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('module ccpp_test_simple_physics_cap', text) + self.assertIn('end module ccpp_test_simple_physics_cap', text) + + def test_header_comment(self): + lines = self._resolve_and_generate() + self.assertTrue(lines[0].startswith('!')) + + def test_use_statements_present(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('use host_phys', text) + + def test_implicit_none_private(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('implicit none', text) + self.assertIn('private', text) + + def test_public_subroutines(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('public :: ccpp_test_simple_physics_run', text) + + def test_contains_block(self): + lines = self._resolve_and_generate() + self.assertIn('contains', lines) + + def test_run_subroutine(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('subroutine ccpp_test_simple_physics_run', text) + self.assertIn('end subroutine ccpp_test_simple_physics_run', text) + + def test_scheme_call_present(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('call temp_calc_adjust_run', text) + + def test_keyword_args_in_call(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + # scheme local name 'im' (horizontal_dimension) is synthesised from + # the loop-bound control vars, not taken directly from host ncols. + self.assertIn('im=(ub - lb + 1)', text) + self.assertIn('timestep=dt', text) + self.assertIn('temp=gt0(lb:ub, 1:nlev)', text) + + def test_errflg_check(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('if (errflg /= 0) return', text) + + def test_init_subroutine(self): + lines = self._resolve_and_generate() + text = '\n'.join(lines) + self.assertIn('subroutine ccpp_test_simple_physics_init', text) + self.assertIn('call temp_calc_adjust_init', text) + + def test_write_group_cap(self): + """write_group_cap writes the file and returns its path.""" + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + sr = resolve_suite(suite, store, hd) + rg = sr.groups[0] + with tempfile.TemporaryDirectory() as tmpdir: + path = write_group_cap('test_simple', 'physics', rg, hd, tmpdir) + self.assertTrue(os.path.isfile(path)) + self.assertEqual(os.path.basename(path), 'ccpp_test_simple_physics_cap.F90') + with open(path) as fh: + content = fh.read() + self.assertIn('module ccpp_test_simple_physics_cap', content) + self.assertIn('call temp_calc_adjust_run', content) + + +######################################################################## +# Tests: subcycle resolution +######################################################################## + +class TestSubcycleResolution(unittest.TestCase): + + def _resolve_subcycle(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_subcycle.xml') + return resolve_suite(suite, store, hd) + + def test_run_phase_has_subcycle(self): + sr = self._resolve_subcycle() + rg = sr.groups[0] + run_items = rg.phase_calls['run'] + self.assertEqual(len(run_items), 1) + self.assertIsInstance(run_items[0], ResolvedSubcycle) + + def test_subcycle_loop_count(self): + sr = self._resolve_subcycle() + sub = sr.groups[0].phase_calls['run'][0] + self.assertEqual(sub.loop, '3') + + def test_subcycle_contains_scheme(self): + sr = self._resolve_subcycle() + sub = sr.groups[0].phase_calls['run'][0] + self.assertEqual(len(sub.calls), 1) + self.assertEqual(sub.calls[0].scheme_name, 'temp_calc_adjust') + + def test_init_phase_is_flat(self): + """Init phase flattens subcycles — no ResolvedSubcycle in init.""" + sr = self._resolve_subcycle() + rg = sr.groups[0] + for item in rg.phase_calls.get('init', []): + self.assertNotIsInstance(item, ResolvedSubcycle) + + def test_iter_phase_calls_flattens(self): + sr = self._resolve_subcycle() + rg = sr.groups[0] + all_calls = list(iter_phase_calls(rg.phase_calls['run'])) + self.assertEqual(len(all_calls), 1) + self.assertEqual(all_calls[0].scheme_name, 'temp_calc_adjust') + + +class TestNestedSubcycleResolution(unittest.TestCase): + """SDFs may nest ```` inside ````. The resolver + must preserve the nesting (it determines the effective iteration + count product: ``outer * inner1 * inner2 * ...``). Original capgen + semantics.""" + + def _resolve_nested(self, suite_xml: str): + import tempfile, os, logging + from test_suite_resolver import ( + _load_full_host_dict, _load_scheme_store, + ) + from generator.suite_xml import parse_suite_xml + from generator.suite_resolver import resolve_suite + hd = _load_full_host_dict() + store = _load_scheme_store() + with tempfile.TemporaryDirectory() as tmp: + xml_path = os.path.join(tmp, 's.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), + skip_validation=True) + return resolve_suite(suite, store, hd) + + def test_two_deep_nesting_preserved(self): + suite_xml = ( + '\n' + '\n' + ' \n' + ' \n' + ' \n' + ' temp_calc_adjust\n' + ' \n' + ' \n' + ' \n' + '\n' + ) + sr = self._resolve_nested(suite_xml) + run = sr.groups[0].phase_calls['run'] + # Outer subcycle. + self.assertEqual(len(run), 1) + outer = run[0] + self.assertIsInstance(outer, ResolvedSubcycle) + self.assertEqual(outer.loop, '3') + # Inside the outer is one inner subcycle, NOT a flat scheme list. + self.assertEqual(len(outer.calls), 1) + inner = outer.calls[0] + self.assertIsInstance(inner, ResolvedSubcycle) + self.assertEqual(inner.loop, '2') + # And the scheme lives inside the inner subcycle. + self.assertEqual(len(inner.calls), 1) + self.assertIsInstance(inner.calls[0], ResolvedCall) + self.assertEqual(inner.calls[0].scheme_name, 'temp_calc_adjust') + + def test_three_deep_nesting_preserved(self): + suite_xml = ( + '\n' + '\n' + ' \n' + ' \n' + ' \n' + ' \n' + ' temp_calc_adjust\n' + ' \n' + ' \n' + ' \n' + ' \n' + '\n' + ) + sr = self._resolve_nested(suite_xml) + run = sr.groups[0].phase_calls['run'] + outer = run[0] + mid = outer.calls[0] + inner = mid.calls[0] + self.assertEqual([outer.loop, mid.loop, inner.loop], ['3', '2', '2']) + # Leaf is the scheme call. + self.assertEqual(len(inner.calls), 1) + self.assertIsInstance(inner.calls[0], ResolvedCall) + + def test_iter_phase_calls_recurses_through_nesting(self): + """``iter_phase_calls`` must walk through every nesting level + and yield the leaf scheme calls — used everywhere a phase needs + a flat view of its scheme calls (USE collection, etc.).""" + suite_xml = ( + '\n' + '\n' + ' \n' + ' \n' + ' \n' + ' temp_calc_adjust\n' + ' \n' + ' \n' + ' \n' + '\n' + ) + sr = self._resolve_nested(suite_xml) + run = sr.groups[0].phase_calls['run'] + calls = list(iter_phase_calls(run)) + # One scheme call, reachable through two subcycle wrappers. + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0].scheme_name, 'temp_calc_adjust') + + +######################################################################## +# Tests: _resolve_subcycle_loop_bound +######################################################################## + +class TestResolveSubcycleLoopBound(unittest.TestCase): + """Subcycle ``loop=`` attribute resolution. + + ``loop=""`` passes through verbatim; ``loop=""`` + must resolve to the host's local Fortran name (or fail with a clear + error). The returned standard name drives USE-statement / dummy-arg + threading in the group cap. + """ + + def _hd(self): + from metadata.metadata_table import _parse_lines + host_src = ''' +[ccpp-table-properties] + name = host_mod + type = host +[ccpp-arg-table] + name = host_mod + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ n_sub ] + standard_name = num_subcycles_for_effr + units = count + dimensions = () + type = integer +''' + ctrl_src = ''' +[ccpp-table-properties] + name = ctrl + type = control +[ccpp-arg-table] + name = ctrl + type = control +[ n_ctrl ] + standard_name = number_of_iterations + units = count + dimensions = () + type = integer +''' + return build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'h.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'c.meta'), + [], + ) + + def test_none_returns_one(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + self.assertEqual( + _resolve_subcycle_loop_bound(None, self._hd()), + ('1', ''), + ) + + def test_empty_string_returns_one(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + self.assertEqual( + _resolve_subcycle_loop_bound(' ', self._hd()), + ('1', ''), + ) + + def test_integer_literal_passes_through(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + self.assertEqual( + _resolve_subcycle_loop_bound('3', self._hd()), + ('3', ''), + ) + self.assertEqual( + _resolve_subcycle_loop_bound(' 42 ', self._hd()), + ('42', ''), + ) + + def test_std_name_resolves_to_host_local(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + local, std = _resolve_subcycle_loop_bound( + 'num_subcycles_for_effr', self._hd(), + ) + self.assertEqual(local, 'n_sub') + self.assertEqual(std, 'num_subcycles_for_effr') + + def test_std_name_case_insensitive(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + local, std = _resolve_subcycle_loop_bound( + 'Num_Subcycles_For_Effr', self._hd(), + ) + self.assertEqual(local, 'n_sub') + self.assertEqual(std, 'num_subcycles_for_effr') + + def test_std_name_resolves_to_control_local(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + local, std = _resolve_subcycle_loop_bound( + 'number_of_iterations', self._hd(), + ) + self.assertEqual(local, 'n_ctrl') + self.assertEqual(std, 'number_of_iterations') + + def test_unresolved_std_name_raises(self): + from generator.suite_resolver import _resolve_subcycle_loop_bound + with self.assertRaises(CCPPError) as cm: + _resolve_subcycle_loop_bound('totally_made_up_name', self._hd()) + msg = str(cm.exception) + self.assertIn('totally_made_up_name', msg) + self.assertIn('standard name', msg) + + def test_std_name_resolving_to_ddt_component_uses_access_path(self): + """When a CCPP standard name resolves to a DDT-component entry + (i.e. the host declares the variable inside a DDT instance), the + generated Fortran must use the *full access path* + (``phys_state%num_subcycles``), not just the bare component name + (``num_subcycles``). The bare local_name wouldn't be in scope + from inside the cap.""" + from metadata.metadata_table import _parse_lines + from generator.suite_resolver import _resolve_subcycle_loop_bound + + ddt_src = ''' +[ccpp-table-properties] + name = physics_state + type = ddt +[ccpp-arg-table] + name = physics_state + type = ddt +[num_sub] + standard_name = num_subcycles_for_effr + units = count + dimensions = () + type = integer +''' + host_src = ''' +[ccpp-table-properties] + name = test_host_mod + type = host +[ccpp-arg-table] + name = test_host_mod + type = host +[ phys_state ] + standard_name = physics_state_ddt_instance + units = ddt + dimensions = () + type = physics_state +''' + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'h.meta'), + [], + _parse_lines(ddt_src.splitlines(keepends=True), 'd.meta'), + ) + expr, std = _resolve_subcycle_loop_bound( + 'num_subcycles_for_effr', hd, + ) + self.assertEqual(expr, 'phys_state%num_sub') + self.assertEqual(std, 'num_subcycles_for_effr') + + +class TestSubcycleGroupCapOutput(unittest.TestCase): + + def setUp(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_subcycle.xml') + sr = resolve_suite(suite, store, hd) + rg = sr.groups[0] + self.lines = _generate_group_cap('test_subcycle', 'physics', rg, hd) + self.text = '\n'.join(self.lines) + + def test_do_loop_present(self): + self.assertIn('do ccpp_loop_counter = 1, 3', self.text) + + def test_end_do_present(self): + self.assertIn('end do', self.text) + + def test_loop_counter_declared(self): + self.assertIn('integer :: ccpp_loop_counter', self.text) + + def test_scheme_call_inside_loop(self): + loop_start = self.text.index('do ccpp_loop_counter = 1, 3') + loop_end = self.text.index('end do') + loop_body = self.text[loop_start:loop_end] + self.assertIn('call temp_calc_adjust_run', loop_body) + + def test_init_not_in_do_loop(self): + """Init phase is flat — no do loop.""" + self.assertNotIn('do ccpp_loop_counter', self.text.split('subroutine ccpp_test_subcycle_physics_init')[1].split('end subroutine')[0]) + + +######################################################################## +# Tests: state machine in group cap +######################################################################## + +class TestStateMachineGroupCap(unittest.TestCase): + + def setUp(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + sr = resolve_suite(suite, store, hd) + rg = sr.groups[0] + self.lines = _generate_group_cap('test_simple', 'physics', rg, hd) + self.text = '\n'.join(self.lines) + + def test_state_constants_declared(self): + self.assertIn('CCPP_GROUP_UNINITIALIZED = 0', self.text) + self.assertIn('CCPP_GROUP_INITIALIZED = 1', self.text) + self.assertIn('CCPP_GROUP_IN_TIMESTEP = 2', self.text) + + def test_state_array_declared(self): + self.assertIn('integer, private, allocatable :: ccpp_group_state(:)', self.text) + + def test_state_alloc_public(self): + self.assertIn('public :: ccpp_test_simple_physics_state_alloc', self.text) + + def test_state_dealloc_public(self): + self.assertIn('public :: ccpp_test_simple_physics_state_dealloc', self.text) + + def test_init_idempotent_skip(self): + # init returns silently when already INITIALIZED. + self.assertIn( + 'if (ccpp_group_state(inst_num) == CCPP_GROUP_INITIALIZED) return', + self.text, + ) + + def test_init_errors_on_invalid_state(self): + # init must error if the state is anything other than UNINITIALIZED + # or INITIALIZED (idempotent skip). + init_sub = self.text.split('subroutine ccpp_test_simple_physics_init')[1] + init_sub = init_sub.split('end subroutine')[0] + self.assertIn( + 'ccpp_group_state(inst_num) /= CCPP_GROUP_UNINITIALIZED', init_sub + ) + self.assertIn('errflg = 1', init_sub) + + def test_init_sets_state(self): + self.assertIn('ccpp_group_state(inst_num) = CCPP_GROUP_INITIALIZED', self.text) + + def test_final_resets_state(self): + self.assertIn('ccpp_group_state(inst_num) = CCPP_GROUP_UNINITIALIZED', self.text) + + def test_run_guards_on_in_timestep(self): + # run requires IN_TIMESTEP; otherwise sets errflg and returns. + run_sub = self.text.split('subroutine ccpp_test_simple_physics_run')[1] + run_sub = run_sub.split('end subroutine')[0] + self.assertIn( + 'ccpp_group_state(inst_num) /= CCPP_GROUP_IN_TIMESTEP', run_sub + ) + self.assertIn('errflg = 1', run_sub) + + def test_state_alloc_subroutine(self): + # state_alloc always takes number_of_instances as explicit arg. + self.assertIn( + 'subroutine ccpp_test_simple_physics_state_alloc(number_of_instances, errmsg, errflg)', + self.text, + ) + self.assertIn('allocate(ccpp_group_state(number_of_instances))', self.text) + + def test_ninstances_not_used_in_group_cap(self): + # number_of_instances is no longer USEd by the group cap module; + # it is passed as an explicit argument to state_alloc instead. + preamble = self.text.split('contains')[0] + self.assertNotIn('ninstances', preamble) + + def test_state_dealloc_subroutine(self): + self.assertIn('subroutine ccpp_test_simple_physics_state_dealloc(errmsg, errflg)', self.text) + self.assertIn('if (allocated(ccpp_group_state)) deallocate(ccpp_group_state)', self.text) + + def test_inst_num_in_init_args(self): + # inst_num (the local name for instance_number) must be a dummy arg of init. + init_sub = self.text.split('subroutine ccpp_test_simple_physics_init')[1] + init_sub = init_sub.split('end subroutine')[0] + self.assertIn('inst_num', init_sub) + + def test_inst_num_in_final_args(self): + final_sub = self.text.split('subroutine ccpp_test_simple_physics_final')[1] + final_sub = final_sub.split('end subroutine')[0] + self.assertIn('inst_num', final_sub) + + +######################################################################## +# Tests: suite cap calls state_alloc/dealloc +######################################################################## + +class TestSuiteCapStateCalls(unittest.TestCase): + """Suite cap calls state_alloc/dealloc — multi-instance host (host_full.meta).""" + + def setUp(self): + from generator.suite_cap import _generate_suite_cap + hd = _load_full_host_dict() + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + sr = resolve_suite(suite, store, hd) + # Pass host_dict so number_of_instances flows through. + lines = _generate_suite_cap('test_simple', sr, store, hd) + self.text = '\n'.join(lines) + + def test_init_calls_state_alloc_with_ninstances(self): + # host_full.meta has ninstances → number_of_instances. + self.assertIn( + 'call ccpp_test_simple_physics_state_alloc(ninstances, errmsg, errflg)', self.text + ) + + def test_init_subroutine_has_ninstances_arg(self): + init_sub = self.text.split('subroutine test_simple_init')[1].split('end subroutine')[0] + self.assertIn('ninstances', init_sub) + + def test_final_calls_state_dealloc(self): + self.assertIn( + 'call ccpp_test_simple_physics_state_dealloc(errmsg, errflg)', self.text + ) + + def test_state_alloc_imported_in_suite_cap(self): + self.assertIn('ccpp_test_simple_physics_state_alloc', self.text.split('contains')[0]) + + def test_state_dealloc_imported_in_suite_cap(self): + self.assertIn('ccpp_test_simple_physics_state_dealloc', self.text.split('contains')[0]) + + +class TestSuiteCapStateCallsSingleInstance(unittest.TestCase): + """Suite cap falls back to passing literal 1 when host has no number_of_instances.""" + + def setUp(self): + from generator.suite_cap import _generate_suite_cap + # Use full host_dict but remove number_of_instances to simulate a + # single-instance host model. + hd_full = _load_full_host_dict() + hd = {k: v for k, v in hd_full.items() if k != 'number_of_instances'} + store = _load_scheme_store() + suite = _parse_suite('suite_test_simple.xml') + sr = resolve_suite(suite, store, hd) + lines = _generate_suite_cap('test_simple', sr, store, hd) + self.text = '\n'.join(lines) + + def test_init_calls_state_alloc_with_literal_1(self): + self.assertIn( + 'call ccpp_test_simple_physics_state_alloc(1, errmsg, errflg)', self.text + ) + + def test_init_subroutine_has_no_ninstances_arg(self): + init_sub = self.text.split('subroutine test_simple_init')[1].split('end subroutine')[0] + self.assertNotIn('number_of_instances', init_sub) + + +######################################################################## +# Register-phase + suite-owned scalar dimension flow +######################################################################## + +def _load_register_dim_scheme_store(): + """Load the register_dim_producer + register_dim_consumer schemes.""" + tables = [] + tables.extend(parse_metadata_file(_sf('scheme_register_dim_producer.meta'))) + tables.extend(parse_metadata_file(_sf('scheme_register_dim_consumer.meta'))) + return SchemeStore.build_from(tables) + + +class TestRegisterPhaseSuiteOwnedDim(unittest.TestCase): + """A scheme writes a suite-owned scalar dimension during ``_register``; + a later scheme uses it as a dimension for an interstitial array.""" + + def setUp(self): + self.hd = _load_full_host_dict() + self.store = _load_register_dim_scheme_store() + self.suite = _parse_suite('suite_register_dim.xml') + self.sr = resolve_suite(self.suite, self.store, self.hd) + + def test_dim_inter_promoted_to_suite_var(self): + # The register-phase intent=out arg becomes a suite-owned variable. + self.assertIn( + 'dimension_for_interstitial_variable', self.sr.suite_vars, + ) + sv = self.sr.suite_vars['dimension_for_interstitial_variable'] + self.assertEqual(sv.type_, 'integer') + self.assertEqual(sv.dimensions, []) + self.assertEqual(sv.source_phase, 'register') + + def test_interstitial_var_promoted_to_suite_var(self): + # The run-phase intent=out array also becomes a suite var, dimensioned + # by the register-set scalar. + self.assertIn( + 'output_only_interstitial_variable', self.sr.suite_vars, + ) + sv = self.sr.suite_vars['output_only_interstitial_variable'] + self.assertEqual(sv.dimensions, ['dimension_for_interstitial_variable']) + + def test_register_phase_call_resolved(self): + # Group's register phase has a ResolvedCall for the producer scheme. + rg = self.sr.groups[0] + register_calls = list(iter_phase_calls(rg.phase_calls.get('register', []))) + self.assertEqual(len(register_calls), 1) + self.assertEqual(register_calls[0].scheme_name, 'register_dim_producer') + + def test_run_phase_dim_resolves_via_suite_var(self): + # The run-phase consumer call's interstitial_var arg's call_expr + # must reference ccpp_suite_data(...)%dim_inter as the upper bound. + rg = self.sr.groups[0] + run_calls = list(iter_phase_calls(rg.phase_calls.get('run', []))) + consumer = next(rc for rc in run_calls + if rc.scheme_name == 'register_dim_consumer') + inter_arg = next(a for a in consumer.args + if a.standard_name == 'output_only_interstitial_variable') + self.assertIn('1:ccpp_suite_data', inter_arg.subscript) + self.assertIn('dim_inter', inter_arg.subscript) + + +class TestRegisterPhaseSuiteCapEmission(unittest.TestCase): + """Suite cap emits the register-phase scheme call inside _register.""" + + def setUp(self): + from generator.suite_cap import _generate_suite_cap + self.hd = _load_full_host_dict() + self.store = _load_register_dim_scheme_store() + self.suite = _parse_suite('suite_register_dim.xml') + self.sr = resolve_suite(self.suite, self.store, self.hd) + self.text = '\n'.join( + _generate_suite_cap('reg_dim', self.sr, self.store, self.hd) + ) + + def test_register_subroutine_emits_scheme_call(self): + register_body = self.text.split('subroutine reg_dim_register')[1].split( + 'end subroutine reg_dim_register' + )[0] + self.assertIn('call register_dim_producer_register', register_body) + + def test_register_subroutine_uses_scheme_module(self): + # USE statement is required so the suite cap can reference _register. + self.assertIn('use register_dim_producer', self.text) + + def test_register_subroutine_uses_suite_data(self): + # Suite-owned vars (dim_inter) are accessed through ccpp__data. + self.assertIn('use ccpp_reg_dim_data', self.text) + + def test_state_transitions_to_registered(self): + register_body = self.text.split('subroutine reg_dim_register')[1].split( + 'end subroutine reg_dim_register' + )[0] + self.assertIn( + 'ccpp_suite_state(inst_num) = CCPP_SUITE_REGISTERED', + register_body, + ) + + +######################################################################## +# Constituent registration (opt-in via type=host) +######################################################################## + +def _load_constituent_host_dict(): + """Load host_with_constituents.meta + control_full.meta + the framework + constituent DDT metadata into a host dict.""" + host_tbls = parse_metadata_file(_sf('host_with_constituents.meta')) + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + # Pull in the framework's DDT definitions for ccpp_constituent_properties_t + # / ccpp_model_constituents_t so the host's ccpp_model_constituents_object + # resolves cleanly. These are passed as DDT tables, not host tables. + ddt_tbls = [] + fw_meta = os.path.join( + os.path.dirname(os.path.dirname(__file__)), + 'capgen-ng', 'src', 'ccpp_constituent_prop_mod.meta', + ) + if os.path.isfile(fw_meta): + ddt_tbls = parse_metadata_file(fw_meta) + return build_flat_host_dict(host_tbls, ctrl_tbls, ddt_tbls) + + +def _load_constituent_scheme_store(): + tables = parse_metadata_file(_sf('scheme_register_constituents.meta')) + return SchemeStore.build_from(tables) + + +class TestRegisterConstituentsResolver(unittest.TestCase): + """Resolver detects intent=out ccpp_constituent_properties_t register args + and records them in suite_res.constituent_register_calls without promoting + to suite_vars.""" + + def setUp(self): + self.hd = _load_constituent_host_dict() + self.store = _load_constituent_scheme_store() + self.suite = _parse_suite('suite_register_constituents.xml') + self.sr = resolve_suite(self.suite, self.store, self.hd) + + def test_constituent_register_calls_recorded(self): + self.assertEqual( + self.sr.constituent_register_calls, + [('register_constituents', 'dyn_const')], + ) + + def test_constituent_arg_not_promoted_to_suite_var(self): + # The constituent array is per-scheme transient — never a SuiteVar. + self.assertNotIn( + 'dynamic_constituents_for_register_test', self.sr.suite_vars, + ) + + def test_constituent_arg_marked(self): + rg = self.sr.groups[0] + register_call = list(iter_phase_calls(rg.phase_calls['register']))[0] + const_arg = next(a for a in register_call.args if a.is_constituent_arg) + self.assertEqual(const_arg.scheme_local_name, 'dyn_const') + self.assertEqual(const_arg.call_expr, 'scheme_consts') + + +class TestRegisterConstituentsNoHostObjectRequired(unittest.TestCase): + """Under option A the constituent object is generator-owned, so the host + is NOT required to declare ``ccpp_model_constituents_object`` — a suite + that registers constituents resolves cleanly against a regular host.""" + + def test_missing_host_object_does_not_raise(self): + hd = _load_full_host_dict() + store = _load_constituent_scheme_store() + suite = _parse_suite('suite_register_constituents.xml') + sr = resolve_suite(suite, store, hd) + # The register-phase scheme is still recorded for the suite cap to + # populate the per-suite dynamic-constituent buffer. + self.assertEqual( + sr.constituent_register_calls, + [('register_constituents', 'dyn_const')], + ) + + +class TestRegisterConstituentsSuiteCap(unittest.TestCase): + """Under option A the suite cap packs each register-phase scheme's + constituent array into the per-suite ``_dynamic_constituents`` + buffer (owned by ccpp_host_constituents). The actual merge into the + host-wide constituent object happens later in + ``ccpp_register_constituents`` — NOT in the suite cap. + """ + + def setUp(self): + from generator.suite_cap import _generate_suite_cap + self.hd = _load_constituent_host_dict() + self.store = _load_constituent_scheme_store() + self.suite = _parse_suite('suite_register_constituents.xml') + self.sr = resolve_suite(self.suite, self.store, self.hd) + self.text = '\n'.join( + _generate_suite_cap('reg_consts', self.sr, self.store, self.hd) + ) + + def test_uses_constituent_prop_type(self): + # Only the property type is USE'd; the model_constituents_t DDT is + # owned by ccpp_host_constituents. + self.assertIn( + 'use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t', + self.text, + ) + + def test_uses_per_suite_buffer(self): + # The per-suite dynamic-constituent buffer is imported from the + # host_constituents module. + self.assertIn( + 'use ccpp_host_constituents, only: reg_consts_dynamic_constituents', + self.text, + ) + + def test_no_host_object_referenced(self): + # Suite cap no longer references the host-wide constituent object + # directly; that's the host_constituents module's job. + self.assertNotIn('host_consts_obj', self.text) + self.assertNotIn('%initialize_table', self.text) + self.assertNotIn('%lock_table', self.text) + self.assertNotIn('%new_field', self.text) + + def test_two_pass_packs_into_buffer(self): + register_body = self.text.split('subroutine reg_consts_register')[1].split( + 'end subroutine reg_consts_register' + )[0] + self.assertIn('First pass: count', register_body) + self.assertIn('Second pass: copy into per-suite buffer', register_body) + # The constituent-providing scheme is called twice (one per pass). + self.assertEqual( + register_body.count('call register_constituents_register'), 2, + ) + + def test_buffer_allocate(self): + self.assertIn( + 'allocate(reg_consts_dynamic_constituents(num_consts))', self.text, + ) + + def test_buffer_populate_loop(self): + self.assertIn( + 'reg_consts_dynamic_constituents(num_consts + i) = scheme_consts(i)', + self.text, + ) + + def test_final_tears_down_per_suite_buffer(self): + # The per-suite dynamic-constituent buffer is owned by the + # suite-cap lifecycle: filled in _register, torn down in + # _final's last-to-leave block. ccpp_deallocate_dynamic_ + # constituents must NOT touch it (would break re-register). + final_body = self.text.split('subroutine reg_consts_final')[1].split( + 'end subroutine reg_consts_final' + )[0] + self.assertIn( + 'use ccpp_host_constituents, only: reg_consts_dynamic_constituents', + final_body, + ) + self.assertIn('if (all(ccpp_suite_state == CCPP_SUITE_UNREGISTERED))', + final_body) + self.assertIn( + 'if (allocated(reg_consts_dynamic_constituents)) ' + 'deallocate(reg_consts_dynamic_constituents)', + final_body, + ) + + def test_scheme_consts_temp_declared(self): + # Local temporary in the register subroutine. + register_body = self.text.split('subroutine reg_consts_register')[1].split( + 'end subroutine reg_consts_register' + )[0] + self.assertIn( + 'type(ccpp_constituent_properties_t), allocatable :: scheme_consts(:)', + register_body, + ) + + +######################################################################## +# Constituent auto-resolution (cam-sima-style consumer schemes) +######################################################################## + +def _load_constituent_consumer_store(): + tables = parse_metadata_file(_sf('scheme_consume_constituent.meta')) + return SchemeStore.build_from(tables) + + +class TestConstituentAutoResolution(unittest.TestCase): + """Resolver routes constituent-flagged scheme args without a host or + earlier-scheme provider through the framework's ``ccpp_constituents`` + and ``ccpp_constituent_tendencies`` arrays. The suite cap will own + those arrays; the resolver emits ``source='constituent'`` ResolvedArgs. + """ + + def setUp(self): + self.hd = _load_constituent_host_dict() + self.store = _load_constituent_consumer_store() + self.suite = _parse_suite('suite_consume_constituent.xml') + self.sr = resolve_suite(self.suite, self.store, self.hd) + run_calls = list(iter_phase_calls(self.sr.groups[0].phase_calls['run'])) + self.run_args = {a.scheme_local_name: a for a in run_calls[0].args} + + def test_uses_constituents_flag_set(self): + self.assertTrue(self.sr.uses_constituents) + + def test_constituent_index_names_enumerated(self): + # Both the base read and the tendency write reference the same + # base std name. + self.assertEqual( + self.sr.constituent_index_names, + ['cloud_liquid_water_mixing_ratio'], + ) + + def test_base_constituent_call_expr(self): + cldliq = self.run_args['cldliq'] + self.assertEqual(cldliq.source, 'constituent') + # Per-instance access: ccpp_model_constituents_obj()%vars_layer(...). + # host_with_constituents.meta declares instance_number with local + # name ``inst_num`` (via control_full.meta), and number_of_instances + # with local name ``ninstances`` — but only inst_num appears here. + self.assertEqual( + cldliq.call_expr, + 'ccpp_model_constituents_obj(inst_num)%vars_layer(lb:ub, ' + '1:nlev, index_of_cloud_liquid_water_mixing_ratio)', + ) + + def test_tendency_call_expr(self): + tend = self.run_args['tend_cldliq'] + self.assertEqual(tend.source, 'constituent') + self.assertEqual( + tend.call_expr, + 'ccpp_model_constituents_obj(inst_num)%vars_layer_tend(lb:ub, ' + '1:nlev, index_of_cloud_liquid_water_mixing_ratio)', + ) + + def test_instance_number_in_used_dim_std_names(self): + # Drives the group cap to inject instance_number as a dummy arg. + for name in ('cldliq', 'tend_cldliq'): + self.assertIn('instance_number', + self.run_args[name].used_dim_std_names) + + def test_constituent_module_name_set(self): + cldliq = self.run_args['cldliq'] + # Under option A all constituent symbols live in a single + # host-wide module, not per-suite. + self.assertEqual(cldliq.constituent_module_name, 'ccpp_host_constituents') + + def test_index_symbol_in_extras(self): + cldliq = self.run_args['cldliq'] + self.assertIn( + 'index_of_cloud_liquid_water_mixing_ratio', + cldliq.constituent_extra_symbols, + ) + + def test_constituent_args_excluded_from_introspection(self): + # source != 'host' — constituent args do not appear in suite + # input/output lists (validated indirectly via _collect_host_io + # in static_api tests; here we just confirm the source). + for arg in self.run_args.values(): + if arg.scheme_local_name in ('cldliq', 'tend_cldliq'): + self.assertNotEqual(arg.source, 'host') + + def test_no_number_of_ccpp_constituents_in_extras(self): + # Regression: under the per-instance design, the + # ``number_of_ccpp_constituents`` symbol is no longer module-level + # — its value is reached as ccpp_model_constituents_obj(inst)% + # num_layer_vars. Even when a scheme dim references it, the + # resolver MUST NOT add it to constituent_extra_symbols (it + # doesn't exist as a USE'd symbol in ccpp_host_constituents and + # would produce an "Symbol ... not found" Fortran error). It + # must also NOT leak into used_dim_std_names (would trigger + # spurious USE/dummy-arg plumbing in the group cap). + for arg in self.run_args.values(): + self.assertNotIn( + 'number_of_ccpp_constituents', + arg.constituent_extra_symbols, + 'arg {!r} leaked number_of_ccpp_constituents into ' + 'constituent_extra_symbols'.format(arg.scheme_local_name), + ) + self.assertNotIn( + 'number_of_ccpp_constituents', + arg.used_dim_std_names, + 'arg {!r} leaked number_of_ccpp_constituents into ' + 'used_dim_std_names (should live on ' + 'used_const_dim_std_names instead)'.format( + arg.scheme_local_name), + ) + + +class TestUsedConstDimStdNames(unittest.TestCase): + """``ResolvedArg.used_const_dim_std_names`` carries framework- + constituent dim refs (notably ``number_of_ccpp_constituents``) so + the introspection routines in :mod:`generator.static_api` can list + them as inputs without polluting the host-side + :attr:`used_dim_std_names` channel.""" + + def _scheme_var(self, local, std_name, dims, intent='in'): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar(local, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', 'none', ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', 'real', ctx) + v.set_attr('kind', 'kind_phys', ctx) + v.set_attr('intent', intent, ctx) + return v + + def test_ccpp_constituents_dim_lands_on_dedicated_field(self): + # Uses _load_full_host_dict (no ccpp_model_constituents_t DDT + # instance), so the host-wins gate does NOT fire and the + # resolver routes through capgen-ng's auto-provisioning path. + from generator.suite_resolver import _resolve_constituent_arg + hd = _load_full_host_dict() + sv = self._scheme_var( + 'consts', 'ccpp_constituents', + '(horizontal_dimension, vertical_layer_dimension, ' + 'number_of_ccpp_constituents)', + intent='in', + ) + arg = _resolve_constituent_arg( + sv, 'run', hd, {}, 'consts_user', 'mysuite', + ) + self.assertIsNotNone(arg) + self.assertEqual(arg.source, 'constituent') + # Goes on the dedicated channel. + self.assertEqual(arg.used_const_dim_std_names, + {'number_of_ccpp_constituents'}) + # Does NOT leak into the host-side channels. + self.assertNotIn('number_of_ccpp_constituents', + arg.used_dim_std_names) + self.assertNotIn('number_of_ccpp_constituents', + arg.constituent_extra_symbols) + + def test_no_const_dim_when_arg_does_not_reference_it(self): + # cldliq is 2D — no framework-constituent dim refs. Uses + # _load_full_host_dict so the resolver routes through the + # constituent auto-provisioning path (Path 2: is_constituent + + # intent=in). + from generator.suite_resolver import _resolve_constituent_arg + hd = _load_full_host_dict() + sv = self._scheme_var( + 'cldliq', 'cloud_liquid_water_mixing_ratio', + '(horizontal_dimension, vertical_layer_dimension)', + intent='in', + ) + sv.set_attr('advected', 'True', _ctx()) + arg = _resolve_constituent_arg( + sv, 'run', hd, {}, 'cldliq_user', 'mysuite', + ) + self.assertIsNotNone(arg) + self.assertEqual(arg.used_const_dim_std_names, set()) + + +class TestConstSubscriptHelper(unittest.TestCase): + """``_const_dim_part`` / ``_build_const_subscript``: + ``number_of_ccpp_constituents`` becomes ``':'`` and is routed + through the dedicated ``used_const_dim_std`` channel — NOT through + ``used_host_std`` (which would imply USE/dummy-arg plumbing) and + NOT through ``used_const_std`` (which would imply a USE'd symbol). + """ + + def setUp(self): + from generator.suite_resolver import ( + _const_dim_part, _build_const_subscript, + ) + self._const_dim_part = _const_dim_part + self._build_const_subscript = _build_const_subscript + self.hd = _load_full_host_dict() + + def test_dim_part_returns_colon(self): + part, used_host, used_const, used_const_dim = self._const_dim_part( + 'number_of_ccpp_constituents', 'run', self.hd, + ) + self.assertEqual(part, ':') + # NOT a host dim — _collect_group_uses / _extra_dim_ctrl_entries + # would mishandle it if it leaked here. + self.assertEqual(used_host, set()) + # NOT a USE'd symbol — it isn't a public name on + # ccpp_host_constituents in the per-instance design. + self.assertEqual(used_const, set()) + # The introspection-only channel. + self.assertEqual(used_const_dim, {'number_of_ccpp_constituents'}) + + def test_dim_part_handles_explicit_lower_bound(self): + part, used_host, used_const, used_const_dim = self._const_dim_part( + 'ccpp_constant_one:number_of_ccpp_constituents', 'run', self.hd, + ) + self.assertEqual(part, ':') + self.assertEqual(used_host, set()) + self.assertEqual(used_const, set()) + self.assertEqual(used_const_dim, {'number_of_ccpp_constituents'}) + + def test_full_3d_subscript(self): + # Mirrors apply_constituent_tendencies.meta's + # (horizontal_dimension, vertical_layer_dimension, + # number_of_ccpp_constituents). + sub, used_host, used_const, used_const_dim = self._build_const_subscript( + ['horizontal_dimension', 'vertical_layer_dimension', + 'number_of_ccpp_constituents'], + 'run', self.hd, + ) + self.assertEqual(sub, '(lb:ub, 1:nlev, :)') + # Host dims (horizontal_*, vertical_*) live in used_host; + # number_of_ccpp_constituents lives in used_const_dim. + self.assertNotIn('number_of_ccpp_constituents', used_host) + self.assertEqual(used_const, set()) + self.assertEqual(used_const_dim, {'number_of_ccpp_constituents'}) + + +class TestConstituentResolverErrors(unittest.TestCase): + """Mismatched constituent-flag + intent + std-name combinations error.""" + + _SCHEME_TEMPLATE = ( + '[ccpp-table-properties]\n' + ' name = bad_scheme\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = bad_scheme_run\n' + ' type = scheme\n' + '[ x ]\n' + ' standard_name = {std}\n' + ' units = kg kg-1\n' + ' dimensions = (horizontal_loop_extent, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = {intent}\n' + ' {flag} = .true.\n' + ) + + _SUITE_XML = ( + '\n' + '\n' + ' bad_scheme\n' + '\n' + ) + + def _build_store(self, std, intent, flag): + with tempfile.NamedTemporaryFile('w', suffix='.meta', delete=False) as fh: + fh.write(self._SCHEME_TEMPLATE.format( + std=std, intent=intent, flag=flag, + )) + path = fh.name + try: + tables = parse_metadata_file(path) + finally: + os.unlink(path) + return SchemeStore.build_from(tables) + + def _resolve(self, std, intent, flag): + hd = _load_constituent_host_dict() + store = self._build_store(std, intent, flag) + with tempfile.TemporaryDirectory() as tmpdir: + xml = os.path.join(tmpdir, 's.xml') + with open(xml, 'w') as f: + f.write(self._SUITE_XML) + from generator.suite_xml import parse_suite_xml + import logging + suite = parse_suite_xml(xml, tmpdir, logging.getLogger('t'), + skip_validation=True) + return resolve_suite(suite, store, hd) + + def test_base_constituent_intent_out_raises(self): + # advected + intent=out on a non-tendency std name → reject. + with self.assertRaises(CCPPError) as ctx: + self._resolve('air_temperature_extra', 'out', 'advected') + self.assertIn("'tendency_of_'", str(ctx.exception)) + + def test_tendency_intent_in_raises(self): + # constituent flag + intent=in on a tendency_of_* std name → reject. + with self.assertRaises(CCPPError) as ctx: + self._resolve('tendency_of_air_temperature', 'in', 'constituent') + self.assertIn('intent=out', str(ctx.exception)) + + def test_tendency_intent_inout_raises(self): + with self.assertRaises(CCPPError) as ctx: + self._resolve('tendency_of_air_temperature', 'inout', 'constituent') + self.assertIn('intent=out', str(ctx.exception)) + + +class TestHostDeclaredIndexOfWinsOverConstituents(unittest.TestCase): + """Regression: when the host declares an ``index_of_`` integer as a + regular host variable, scheme references to that std_name must resolve + to the host's short local name — NOT route through the constituent + auto-provisioning path (which would emit a parallel module-level + integer named after the full std_name in ``ccpp_host_constituents`` + and, for SCM-style long std_names, blow the Fortran 63-char identifier + limit). + """ + + _HOST_SRC = ( + '[ccpp-table-properties]\n' + ' name = scm_host_mod\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = scm_host_mod\n' + ' type = host\n' + # Mirrors SCM's GFS_typedefs: short Fortran local name `ntcw` + # paired with a long index_of_* standard name. + '[ ntcw ]\n' + ' standard_name = index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array\n' + ' units = index\n' + ' type = integer\n' + ' protected = True\n' + ' dimensions = ()\n' + ) + + def _scheme_var(self, local, std_name, intent='in'): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar(local, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', 'index', ctx) + v.set_attr('dimensions', '()', ctx) + v.set_attr('type', 'integer', ctx) + v.set_attr('intent', intent, ctx) + return v + + def test_host_index_of_resolves_to_host_local_name(self): + hd = build_flat_host_dict(_parse(self._HOST_SRC), [], []) + sv = self._scheme_var( + 'ntcw', + 'index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array', + intent='in', + ) + arg = _resolve_one_arg(sv, 'run', hd, {}, 'gfs_mp_generic_pre', set()) + # Host metadata wins: source=host, short local name, NO leakage of + # the long std_name into ccpp_host_constituents. + self.assertEqual(arg.source, 'host') + self.assertEqual(arg.call_expr, 'ntcw') + self.assertIsNotNone(arg.host_entry) + self.assertEqual(arg.host_entry.local_name, 'ntcw') + + def test_unclaimed_index_of_still_routes_to_constituents(self): + """The framework auto-provisioning path is preserved for + ``index_of_`` names the host does NOT declare — required for + capgen-ng-owned constituent flows (cf. the advection e2e test).""" + hd = build_flat_host_dict(_parse(self._HOST_SRC), [], []) + sv = self._scheme_var( + 'idx_other', 'index_of_some_other_constituent_not_in_host', + intent='in', + ) + arg = _resolve_one_arg(sv, 'run', hd, {}, 'some_scheme', set()) + self.assertEqual(arg.source, 'constituent') + self.assertEqual(arg.call_expr, + 'index_of_some_other_constituent_not_in_host') + + +######################################################################## +# Doctest loader +######################################################################## + +def load_tests(loader, tests, ignore): + import generator.suite_resolver as sr + import generator.group_cap as gc + tests.addTests(doctest.DocTestSuite(sr)) + tests.addTests(doctest.DocTestSuite(gc)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_suite_xml.py b/unit-tests/test_suite_xml.py new file mode 100644 index 00000000..a20660bf --- /dev/null +++ b/unit-tests/test_suite_xml.py @@ -0,0 +1,767 @@ +#!/usr/bin/env python3 + +"""Unit tests for :mod:`generator.suite_xml`. + +Tests cover: + +1. The in-memory data model: :class:`~generator.suite_xml.SuiteScheme`, + :class:`~generator.suite_xml.SuiteSubcycle`, + :class:`~generator.suite_xml.SuiteSubcol`, + :class:`~generator.suite_xml.SuiteGroup`, + :class:`~generator.suite_xml.Suite`. +2. The XML-to-object builder :func:`~generator.suite_xml._build_suite`. +3. The public API :func:`~generator.suite_xml.parse_suite_xml` — including + nested-suite expansion, expanded-XML writing, and error detection. + +The test cases for parsing and expansion are ported from +``test/unit_tests/test_sdf.py`` in the legacy capgen test suite, updated +for the new package layout. + +Run with:: + + python -m pytest capgen-ng/tests/test_suite_xml.py -v + +or include ``--doctest-modules capgen-ng/generator/suite_xml.py``. +""" + +import filecmp +import glob +import logging +import os +import sys +import tempfile +import unittest +import xml.etree.ElementTree as ET + +# ---- path setup ------------------------------------------------------------ +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_REPO_ROOT = os.path.dirname(_TESTS_DIR) +_PKG_ROOT = os.path.join(_REPO_ROOT, 'capgen-ng') +for _p in (_PKG_ROOT, _REPO_ROOT): + if _p not in sys.path: + sys.path.insert(0, _p) + +# ---- imports ---------------------------------------------------------------- +from metadata.parse_tools import ( + CCPPError, + init_log, + set_log_to_null, + read_xml_file, + find_schema_version, + expand_nested_suites, + write_xml_file, +) +from metadata.parse_tools.xml_tools import validate_xml_file +from generator.suite_xml import ( + Suite, + SuiteGroup, + SuiteScheme, + SuiteSubcol, + SuiteSubcycle, + _build_suite, + _parse_group_items, + parse_suite_xml, +) + +_SAMPLE_DIR = os.path.join(_TESTS_DIR, 'sample_suite_files') +_SCHEMA_DIR = os.path.join(_PKG_ROOT, 'schema') + + +######################################################################## +# Helper — shared logger (quiet by default) +######################################################################## + +def _make_logger(name='test_suite_xml'): + log = init_log(name, level=logging.WARNING) + set_log_to_null(log) + return log + + +######################################################################## +# Helper — XML tree comparison (ported from test_sdf.py) +######################################################################## + +def _compare_text(name, txt1, txt2, typ): + """Return an error string if the text items differ, else None.""" + if txt1 and txt2: + if txt1.strip() != txt2.strip(): + return f"{name} {typ}, '{txt1}', does not match {typ}, '{txt2}'" + elif txt1: + return f"{name} {typ} is missing from string2" + elif txt2: + return f"{name} {typ} is missing from string1" + return None + + +def xml_diff(xt1, xt2): + """Return a list of difference strings between two ElementTree subtrees. + + Returns an empty list when the trees are identical. + """ + diffs = [] + if xt1.tag != xt2.tag: + diffs.append(f"Tags do not match: {xt1.tag} != {xt2.tag}") + return diffs + for name, value in xt1.attrib.items(): + if name not in xt2.attrib: + diffs.append(f"xt1 attribute, {name}, is missing in xt2") + elif xt2.attrib[name] != value: + diffs.append(f"Attributes for {name} do not match: " + f"{value!r} != {xt2.attrib[name]!r}") + for name in xt2.attrib: + if name not in xt1.attrib: + diffs.append(f"xt2 attribute, {name}, is missing in xt1") + tdiff = _compare_text(xt1.tag, xt1.text, xt2.text, "text") + if tdiff: + diffs.append(tdiff) + tdiff = _compare_text(xt1.tag, xt1.tail, xt2.tail, "tail") + if tdiff: + diffs.append(tdiff) + if len(xt1) != len(xt2): + diffs.append(f"Number of children differs: {len(xt1)} != {len(xt2)}") + else: + for c1, c2 in zip(xt1, xt2): + diffs.extend(xml_diff(c1, c2)) + return diffs + + +######################################################################## +# Data model tests +######################################################################## + +class TestSuiteScheme(unittest.TestCase): + """Tests for :class:`SuiteScheme`.""" + + def test_creation(self): + s = SuiteScheme('my_scheme') + self.assertEqual(s.name, 'my_scheme') + + def test_leading_trailing_spaces_stripped(self): + s = SuiteScheme(' spaced ') + self.assertEqual(s.name, 'spaced') + + def test_scheme_names(self): + self.assertEqual(SuiteScheme('foo').scheme_names(), ['foo']) + + def test_repr(self): + self.assertIn('my_scheme', repr(SuiteScheme('my_scheme'))) + + +class TestSuiteSubcycle(unittest.TestCase): + """Tests for :class:`SuiteSubcycle`.""" + + def test_literal_integer_loop(self): + sc = SuiteSubcycle(loop='2', items=[]) + self.assertEqual(sc.loop, '2') + self.assertTrue(sc.is_literal_count) + + def test_stdname_loop(self): + sc = SuiteSubcycle(loop='num_subcycles_for_ag', items=[]) + self.assertFalse(sc.is_literal_count) + + def test_none_loop_is_literal(self): + sc = SuiteSubcycle(loop=None, items=[]) + self.assertIsNone(sc.loop) + self.assertTrue(sc.is_literal_count) + + def test_scheme_names_from_items(self): + sc = SuiteSubcycle(loop='2', items=[ + SuiteScheme('scheme_a'), + SuiteScheme('scheme_b'), + ]) + self.assertEqual(sc.scheme_names(), ['scheme_a', 'scheme_b']) + + def test_nested_subcycle_scheme_names(self): + inner = SuiteSubcycle(loop='3', items=[SuiteScheme('inner_sch')]) + outer = SuiteSubcycle(loop='2', items=[SuiteScheme('outer_sch'), inner]) + self.assertEqual(outer.scheme_names(), ['outer_sch', 'inner_sch']) + + +class TestSuiteSubcol(unittest.TestCase): + """Tests for :class:`SuiteSubcol`.""" + + def test_creation(self): + sc = SuiteSubcol('gen_routine', 'avg_routine', [SuiteScheme('sch')]) + self.assertEqual(sc.gen_routine, 'gen_routine') + self.assertEqual(sc.avg_routine, 'avg_routine') + + def test_scheme_names(self): + sc = SuiteSubcol('g', 'a', [SuiteScheme('s1'), SuiteScheme('s2')]) + self.assertEqual(sc.scheme_names(), ['s1', 's2']) + + +class TestSuiteGroup(unittest.TestCase): + """Tests for :class:`SuiteGroup`.""" + + def test_creation(self): + g = SuiteGroup('physics', [SuiteScheme('sch1'), SuiteScheme('sch2')]) + self.assertEqual(g.name, 'physics') + self.assertEqual(len(g.items), 2) + + def test_scheme_names(self): + g = SuiteGroup('g', [ + SuiteScheme('a'), + SuiteSubcycle('2', [SuiteScheme('b'), SuiteScheme('a')]), + ]) + self.assertEqual(g.scheme_names(), ['a', 'b', 'a']) + + def test_unique_scheme_names(self): + g = SuiteGroup('g', [ + SuiteScheme('a'), + SuiteSubcycle('2', [SuiteScheme('b'), SuiteScheme('a')]), + ]) + self.assertEqual(g.unique_scheme_names(), ['a', 'b']) + + +class TestSuite(unittest.TestCase): + """Tests for :class:`Suite`.""" + + def _make_suite(self, groups=None, init=None, final=None): + groups = groups or [SuiteGroup('g', [SuiteScheme('s')])] + return Suite('my_suite', [2, 0], '/f.xml', groups, init, final) + + def test_name(self): + self.assertEqual(self._make_suite().name, 'my_suite') + + def test_group_names(self): + s = self._make_suite([ + SuiteGroup('g1', [SuiteScheme('s1')]), + SuiteGroup('g2', [SuiteScheme('s2')]), + ]) + self.assertEqual(s.group_names(), ['g1', 'g2']) + + def test_get_group_found(self): + grp = SuiteGroup('dynamics', [SuiteScheme('dyn')]) + s = self._make_suite([grp]) + self.assertIs(s.get_group('dynamics'), grp) + + def test_get_group_not_found(self): + self.assertIsNone(self._make_suite().get_group('nope')) + + def test_all_scheme_names_deduped(self): + s = self._make_suite([ + SuiteGroup('g1', [SuiteScheme('common'), SuiteScheme('a')]), + SuiteGroup('g2', [SuiteScheme('b'), SuiteScheme('common')]), + ]) + self.assertEqual(s.all_scheme_names(), ['common', 'a', 'b']) + + def test_init_final_schemes(self): + s = self._make_suite(init='suite_init', final='suite_final') + self.assertEqual(s.init_scheme, 'suite_init') + self.assertEqual(s.final_scheme, 'suite_final') + + def test_expanded_file_default_none(self): + self.assertIsNone(self._make_suite().expanded_file) + + +######################################################################## +# _parse_group_items tests +######################################################################## + +class TestParseGroupItems(unittest.TestCase): + """Tests for :func:`_parse_group_items`.""" + + def test_single_scheme(self): + xml = 'my_scheme' + el = ET.fromstring(xml) + items = _parse_group_items(el) + self.assertEqual(len(items), 1) + self.assertIsInstance(items[0], SuiteScheme) + self.assertEqual(items[0].name, 'my_scheme') + + def test_subcycle_with_loop(self): + xml = 's' + el = ET.fromstring(xml) + items = _parse_group_items(el) + self.assertEqual(len(items), 1) + sc = items[0] + self.assertIsInstance(sc, SuiteSubcycle) + self.assertEqual(sc.loop, '3') + self.assertEqual(len(sc.items), 1) + + def test_subcol(self): + xml = ('' + '' + 's') + el = ET.fromstring(xml) + items = _parse_group_items(el) + self.assertIsInstance(items[0], SuiteSubcol) + self.assertEqual(items[0].gen_routine, 'gen_r') + + def test_subcol_missing_avg_raises(self): + xml = ('' + '' + 's') + el = ET.fromstring(xml) + with self.assertRaises(CCPPError): + _parse_group_items(el) + + def test_empty_scheme_raises(self): + xml = ' ' + el = ET.fromstring(xml) + with self.assertRaises(CCPPError): + _parse_group_items(el) + + def test_nested_subcycles(self): + xml = ''' + + + inner + + + ''' + el = ET.fromstring(xml) + items = _parse_group_items(el) + outer = items[0] + self.assertIsInstance(outer, SuiteSubcycle) + inner = outer.items[0] + self.assertIsInstance(inner, SuiteSubcycle) + self.assertEqual(inner.loop, '3') + self.assertEqual(inner.items[0].name, 'inner') + + +######################################################################## +# _build_suite tests +######################################################################## + +class TestBuildSuite(unittest.TestCase): + """Tests for :func:`_build_suite`.""" + + def _parse(self, xml_str, source='test.xml', version=None): + root = ET.fromstring(xml_str) + v = version or [2, 0] + return _build_suite(root, source, v, _make_logger()) + + def test_basic_suite(self): + xml = ''' + + sch1 + + ''' + suite = self._parse(xml) + self.assertEqual(suite.name, 'my_suite') + self.assertEqual(suite.group_names(), ['physics']) + + def test_suite_with_init_final(self): + xml = ''' + suite_init_scheme + sch + suite_final_scheme + ''' + suite = self._parse(xml) + self.assertEqual(suite.init_scheme, 'suite_init_scheme') + self.assertEqual(suite.final_scheme, 'suite_final_scheme') + + def test_deprecated_finalize_rejected(self): + """ (old long form) is rejected with a clear error + pointing at the canonical short form.""" + xml = ''' + sch + final_scheme + ''' + with self.assertRaises(CCPPError) as cm: + self._parse(xml) + msg = str(cm.exception) + self.assertIn('finalize', msg) + self.assertIn('', msg) + + def test_deprecated_initalize_typo_rejected(self): + """ (the old schema's typo) is rejected with a clear + error pointing at the canonical short form.""" + xml = ''' + init_scheme + sch + ''' + with self.assertRaises(CCPPError) as cm: + self._parse(xml) + msg = str(cm.exception) + self.assertIn('initalize', msg) + self.assertIn('', msg) + + def test_deprecated_initialize_correct_spelling_rejected(self): + """ (the correctly-spelled long form) is also rejected. + Only the short is accepted.""" + xml = ''' + init_scheme + sch + ''' + with self.assertRaises(CCPPError) as cm: + self._parse(xml) + msg = str(cm.exception) + self.assertIn('initialize', msg) + self.assertIn('', msg) + + def test_missing_suite_name_raises(self): + xml = 's' + with self.assertRaises(CCPPError): + self._parse(xml) + + def test_duplicate_group_name_raises(self): + xml = ''' + a + b + ''' + with self.assertRaises(CCPPError) as cm: + self._parse(xml) + self.assertIn('dup', str(cm.exception)) + + def test_empty_init_raises(self): + xml = ''' + + s + ''' + with self.assertRaises(CCPPError): + self._parse(xml) + + def test_empty_final_raises(self): + xml = ''' + s + + ''' + with self.assertRaises(CCPPError): + self._parse(xml) + + def test_subcycle_in_group(self): + xml = ''' + + + scheme6 + + + ''' + suite = self._parse(xml) + grp = suite.groups[0] + sc = grp.items[0] + self.assertIsInstance(sc, SuiteSubcycle) + self.assertEqual(sc.loop, 'num_subcycles_for_scheme6') + self.assertFalse(sc.is_literal_count) + + +######################################################################## +# parse_suite_xml — valid files (ported from test_sdf.py) +######################################################################## + +def _sample(name): + return os.path.join(_SAMPLE_DIR, name) + + +class TestParseSuiteXmlValid(unittest.TestCase): + """Integration tests for :func:`parse_suite_xml` with valid SDFs. + + These tests port the ``test_good_v*`` cases from the legacy + ``test_sdf.py`` test suite. Expanded XML output is compared against + the ``*_exp.xml`` reference files. + """ + + def setUp(self): + self._tmp = tempfile.mkdtemp() + self._log = _make_logger() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmp, ignore_errors=True) + + def _parse(self, filename, skip_validation=True): + return parse_suite_xml( + _sample(filename), self._tmp, + logger=self._log, + schema_path=_SCHEMA_DIR, + skip_validation=skip_validation, + ) + + def _compare_expanded(self, suite, exp_filename): + """Assert that the expanded XML matches the reference file.""" + self.assertIsNotNone(suite.expanded_file) + self.assertTrue(os.path.isfile(suite.expanded_file)) + _, ref_root = read_xml_file(_sample(exp_filename), self._log) + _, got_root = read_xml_file(suite.expanded_file, self._log) + diffs = xml_diff(ref_root, got_root) + sep = '\n' + self.assertFalse( + diffs, + msg=f"Expanded XML differs from {exp_filename}:\n{sep.join(diffs)}" + ) + + # ---- v1 suites --------------------------------------------------------- + + def test_v1_suite_01(self): + """V1 SDF is read and written without expansion.""" + suite = self._parse('suite_good_v1_test01.xml') + self.assertEqual(suite.version[0], 1) + self.assertTrue(os.path.isfile(suite.expanded_file)) + + def test_v1_suite_02(self): + suite = self._parse('suite_good_v1_test02.xml') + self.assertEqual(suite.version[0], 1) + + # ---- v2 suites --------------------------------------------------------- + + def test_v2_suite_01_expand_group_of_nested_suite(self): + """Expand one group from a simple nested suite at group level.""" + suite = self._parse('suite_good_v2_test01.xml') + self.assertEqual(suite.name, 'ver_test_suite') + self.assertEqual(suite.version, [2, 0]) + self.assertEqual(suite.group_names(), ['group1']) + self._compare_expanded(suite, 'suite_good_v2_test01_exp.xml') + + def test_v2_suite_01_scheme_names(self): + """After expansion, correct scheme names are in group1.""" + suite = self._parse('suite_good_v2_test01.xml') + names = suite.get_group('group1').scheme_names() + # Expected after expansion: scheme5, scheme1i, scheme2i, scheme1i, scheme9 + self.assertEqual(names, ['scheme5', 'scheme1i', 'scheme2i', 'scheme1i', 'scheme9']) + + def test_v2_suite_02_expand_one_group_of_multigroup_nested_suite(self): + """Expand one group from a multi-group nested suite at group level.""" + suite = self._parse('suite_good_v2_test02.xml') + self.assertEqual(suite.name, 'v2_suite') + self._compare_expanded(suite, 'suite_good_v2_test02_exp.xml') + + def test_v2_suite_02_subcycle_preserved(self): + """Subcycle loop attribute survives the expansion.""" + suite = self._parse('suite_good_v2_test02.xml') + grp = suite.get_group('main_group') + # First item is the subcycle from the original suite + sc = grp.items[0] + self.assertIsInstance(sc, SuiteSubcycle) + self.assertEqual(sc.loop, 'num_subcycles_for_scheme6') + + def test_v2_suite_03_expand_multiple_nested_suites(self): + """Expand two nested suites at group level + full suite at suite level.""" + suite = self._parse('suite_good_v2_test03.xml') + self.assertEqual(suite.name, 'main_suite') + # Should have 3 groups after expansion: groupp, nested_group1, nested_group2 + self.assertEqual(len(suite.groups), 3) + self._compare_expanded(suite, 'suite_good_v2_test03_exp.xml') + + def test_v2_suite_04_expand_group_from_nested_full_suite(self): + """Expand two group-level nested suites + one group from nested suite at suite level.""" + suite = self._parse('suite_good_v2_test04.xml') + self.assertEqual(suite.name, 'ver_test_suite') + # 2 groups: main11 + nested_group2 (only group from nested_full_suite) + self.assertEqual(len(suite.groups), 2) + self._compare_expanded(suite, 'suite_good_v2_test04_exp.xml') + + def test_expanded_xml_written_to_output_root(self): + """The expanded XML must be written to the correct output path.""" + suite = self._parse('suite_good_v2_test01.xml') + expected_name = f"ccpp_{suite.name}_expanded.xml" + expected_path = os.path.join(self._tmp, expected_name) + self.assertEqual(suite.expanded_file, expected_path) + self.assertTrue(os.path.isfile(expected_path)) + + def test_all_scheme_names_unique(self): + """all_scheme_names() deduplicates across groups.""" + suite = self._parse('suite_good_v2_test03.xml') + all_names = suite.all_scheme_names() + self.assertEqual(len(all_names), len(set(all_names))) + + +######################################################################## +# parse_suite_xml — error cases +######################################################################## + +class TestParseSuiteXmlErrors(unittest.TestCase): + """Error handling tests for :func:`parse_suite_xml`.""" + + def setUp(self): + self._tmp = tempfile.mkdtemp() + self._log = _make_logger() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmp, ignore_errors=True) + + def _parse(self, filename, skip_validation=True): + return parse_suite_xml( + _sample(filename), self._tmp, + logger=self._log, + schema_path=_SCHEMA_DIR, + skip_validation=skip_validation, + ) + + def test_nonexistent_file_raises(self): + with self.assertRaises(CCPPError): + parse_suite_xml('/nonexistent/suite.xml', self._tmp, + logger=self._log, skip_validation=True) + + def test_bad_schema_version_formats(self): + """Malformed version attributes are detected on file read.""" + for fname in ('suite_bad_version01.xml', 'suite_bad_version02.xml', + 'suite_bad_version03.xml', 'suite_bad_version04.xml'): + with self.subTest(fname=fname): + with self.assertRaises(CCPPError): + self._parse(fname) + + def test_missing_version_raises(self): + with self.assertRaises(CCPPError): + self._parse('suite_missing_version.xml') + + def test_infinite_group_recursion_detected(self): + """Circular nested-suite references at group level are caught.""" + with self.assertRaises(CCPPError) as cm: + self._parse('suite_recurse_top1.xml') + self.assertIn('iterations', str(cm.exception)) + + def test_infinite_suite_recursion_detected(self): + """Circular nested-suite references at suite level are caught.""" + with self.assertRaises(CCPPError) as cm: + self._parse('suite_recurse_top2.xml') + self.assertIn('iterations', str(cm.exception)) + + def test_missing_nested_group_raises(self): + """Referencing a group that doesn't exist in the target suite raises.""" + with self.assertRaises(CCPPError) as cm: + self._parse('suite_missing_group.xml') + self.assertIn('not found', str(cm.exception)) + + def test_missing_loaded_suite_raises(self): + """Referencing a suite name that doesn't exist in the target file raises.""" + with self.assertRaises(CCPPError) as cm: + self._parse('suite_missing_loaded_suite.xml') + self.assertIn('not found', str(cm.exception)) + + +######################################################################## +# Schema validation tests (require xmllint) +######################################################################## + +class TestSchemaValidation(unittest.TestCase): + """Tests that exercise xmllint-based XML schema validation. + + These tests are skipped automatically when ``xmllint`` is not installed. + """ + + @classmethod + def setUpClass(cls): + import shutil + if not shutil.which('xmllint'): + raise unittest.SkipTest("xmllint not installed — skipping schema validation tests") + cls._log = _make_logger() + cls._tmp = tempfile.mkdtemp() + + @classmethod + def tearDownClass(cls): + import shutil + shutil.rmtree(cls._tmp, ignore_errors=True) + + def test_good_v2_suite_validates(self): + _, root = read_xml_file(_sample('suite_good_v2_test01.xml'), self._log) + version = find_schema_version(root) + result = validate_xml_file( + _sample('suite_good_v2_test01.xml'), 'suite', version, self._log, + schema_path=_SCHEMA_DIR + ) + self.assertTrue(result) + + def test_bad_suite_tag_rejected(self): + """A nested element violates the schema.""" + _, root = read_xml_file(_sample('suite_bad_v2_suite_tag.xml'), self._log) + version = find_schema_version(root) + try: + result = validate_xml_file( + _sample('suite_bad_v2_suite_tag.xml'), 'suite', version, + self._log, schema_path=_SCHEMA_DIR + ) + # Some xmllint versions return True even on error + except CCPPError as exc: + self.assertIn("not expected", str(exc)) + + def test_invalid_fortran_id_scheme_rejected(self): + """A scheme name that is not a valid Fortran ID is rejected.""" + _, root = read_xml_file( + _sample('suite_invalid_scheme_fortran_id.xml'), self._log + ) + version = find_schema_version(root) + with self.assertRaises(CCPPError) as cm: + validate_xml_file( + _sample('suite_invalid_scheme_fortran_id.xml'), 'suite', + version, self._log, schema_path=_SCHEMA_DIR + ) + self.assertIn("scheme-1", str(cm.exception)) + + def test_invalid_fortran_id_group_rejected(self): + _, root = read_xml_file( + _sample('suite_invalid_group_fortran_id.xml'), self._log + ) + version = find_schema_version(root) + with self.assertRaises(CCPPError) as cm: + validate_xml_file( + _sample('suite_invalid_group_fortran_id.xml'), 'suite', + version, self._log, schema_path=_SCHEMA_DIR + ) + self.assertIn("group-1", str(cm.exception)) + + def test_invalid_fortran_id_suite_rejected(self): + _, root = read_xml_file( + _sample('suite_invalid_suite_fortran_id.xml'), self._log + ) + version = find_schema_version(root) + with self.assertRaises(CCPPError) as cm: + validate_xml_file( + _sample('suite_invalid_suite_fortran_id.xml'), 'suite', + version, self._log, schema_path=_SCHEMA_DIR + ) + self.assertIn("ver-test-suite", str(cm.exception)) + + def test_duplicate_group_name_rejected_after_expansion(self): + """After nested-suite expansion a duplicate group name fails validation.""" + _, root = read_xml_file(_sample('suite_bad_v2_duplicate_group.xml'), self._log) + version = find_schema_version(root) + # Initial file validates OK + result = validate_xml_file( + _sample('suite_bad_v2_duplicate_group.xml'), 'suite', version, + self._log, schema_path=_SCHEMA_DIR + ) + self.assertTrue(result) + # After expansion the duplicated xs:ID triggers a validation error + expand_nested_suites(root, _SAMPLE_DIR, logger=self._log) + expanded_path = os.path.join(self._tmp, 'dup_group_expanded.xml') + write_xml_file(root, expanded_path, self._log) + with self.assertRaises(CCPPError) as cm: + validate_xml_file(expanded_path, 'suite', version, self._log, + schema_path=_SCHEMA_DIR) + self.assertIn('group1', str(cm.exception)) + + +######################################################################## +# xml_diff helper tests (ported from test_sdf.py) +######################################################################## + +class TestXmlDiff(unittest.TestCase): + """Tests for the :func:`xml_diff` helper.""" + + def test_matching_trees(self): + r1 = ET.fromstring('text') + r2 = ET.fromstring('text') + self.assertEqual(xml_diff(r1, r2), []) + + def test_tag_mismatch(self): + diffs = xml_diff(ET.fromstring('x'), + ET.fromstring('x')) + self.assertEqual(len(diffs), 1) + self.assertIn('Tags', diffs[0]) + + def test_text_mismatch(self): + diffs = xml_diff(ET.fromstring('a'), ET.fromstring('b')) + self.assertEqual(len(diffs), 1) + self.assertIn('does not match', diffs[0]) + + def test_attrib_mismatch(self): + r1 = ET.fromstring('') + r2 = ET.fromstring('') + diffs = xml_diff(r1, r2) + self.assertEqual(len(diffs), 3) + + def test_child_count_mismatch(self): + r1 = ET.fromstring('

') + r2 = ET.fromstring('

') + diffs = xml_diff(r1, r2) + self.assertEqual(len(diffs), 1) + self.assertIn('children', diffs[0]) + + +######################################################################## + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_validator.py b/unit-tests/test_validator.py new file mode 100644 index 00000000..587861a2 --- /dev/null +++ b/unit-tests/test_validator.py @@ -0,0 +1,654 @@ +"""Unit tests for ccpp_validator.""" + +import doctest +import os +import tempfile +import textwrap +import unittest + +import ccpp_validator as val_mod +from ccpp_validator import ( + _join_continuation, + _parse_subroutines, + _load_source_tree, + validate, +) + +_SAMPLE_DIR = os.path.join(os.path.dirname(__file__), 'sample_files') +_CORRECT_F90 = os.path.join(_SAMPLE_DIR, 'scheme_multipart_correct.F90') +_WRONG_F90 = os.path.join(_SAMPLE_DIR, 'scheme_multipart_wrong_args.F90') +_SCHEME_META = os.path.join(_SAMPLE_DIR, 'scheme_multipart.meta') + + +class TestJoinContinuation(unittest.TestCase): + + def test_no_continuation(self): + lines = [' foo\n', ' bar\n'] + self.assertEqual(_join_continuation(lines), [' foo', ' bar']) + + def test_single_continuation(self): + lines = [' foo &\n', ' bar\n'] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertIn('foo', result[0]) + self.assertIn('bar', result[0]) + + def test_multi_continuation(self): + lines = [' a &\n', ' b &\n', ' c\n'] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertIn('a', result[0]) + self.assertIn('c', result[0]) + + def test_comment_after_continuation(self): + lines = [' foo & ! comment\n', ' bar\n'] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertNotIn('comment', result[0]) + + def test_dual_form_strips_leading_ampersand(self): + # F77 / fixed-form: ``&`` at column 6 of the continuation line + # is a continuation marker and must be stripped before the + # continued expression is appended to the buffer. Without + # this, the joined logical line carries ``&`` glued into the + # middle of the expression. + lines = [' foo &\n', ' & bar &\n', ' & baz\n'] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertNotIn('&', result[0]) + self.assertIn('foo', result[0]) + self.assertIn('bar', result[0]) + self.assertIn('baz', result[0]) + + def test_dual_form_then_free_form_mixed(self): + # Files that use dual-form for one signature and free-form for + # another (or mix within a single signature) must still join + # correctly. + lines = [ + ' start &\n', # trailing & + ' & middle &\n', # leading & + trailing & + ' end\n', # no leading &, plain continuation tail + ] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertNotIn('&', result[0]) + for tok in ('start', 'middle', 'end'): + self.assertIn(tok, result[0]) + + def test_comment_line_between_continuation_lines(self): + # Real-world case (sfc_diff.f::stability): a comment-only line + # appears between two continuation lines. Fortran 90+ allows + # this; the join must skip the comment line, not terminate. + lines = [ + ' subroutine stability &\n', + '! --- inputs:\n', + ' & ( z1, zvfun, grav, &\n', + '! --- outputs:\n', + ' & rb, fm, fh, cm, ch, ustar)\n', + ] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + self.assertIn('subroutine stability', result[0]) + self.assertIn('z1', result[0]) + self.assertIn('ustar', result[0]) + # Closing paren survives. + self.assertIn(')', result[0]) + self.assertNotIn('&', result[0]) + + def test_blank_line_between_continuation_lines(self): + # Blank lines mid-continuation are also legal under F90+. + lines = [ + ' foo &\n', + '\n', + ' & bar &\n', + ' \n', + ' & baz\n', + ] + result = _join_continuation(lines) + self.assertEqual(len(result), 1) + for tok in ('foo', 'bar', 'baz'): + self.assertIn(tok, result[0]) + + def test_rrtmg_style_signature_round_trip(self): + # The real-world failure mode that motivated the fix: a + # fixed-form subroutine signature with 57 args spread across + # 16 dual-form continuation lines. After joining, ``(`` must + # appear immediately after the subroutine name (no stray + # ``&`` in between) so the signature regex can pick up the + # arg list. + src_lines = [ + ' subroutine rrtmg_lw_run &\n', + ' & ( plyr,plvl,tlyr,tlvl,qlyr,olyr, &\n', + ' & icseed,aeraod,aerssa, &\n', + ' & errmsg, errflg &\n', + ' & )\n', + ] + result = _join_continuation(src_lines) + self.assertEqual(len(result), 1) + self.assertNotIn('&', result[0]) + # ``subroutine rrtmg_lw_run`` is followed (after whitespace) by ``(``. + import re as _re + self.assertIsNotNone( + _re.search(r'subroutine\s+rrtmg_lw_run\s*\(', result[0]), + 'joined signature lacks ``subroutine NAME (`` shape — got: {!r}' + .format(result[0]), + ) + + +class TestParseSubroutines(unittest.TestCase): + + def test_simple_subroutine(self): + src = 'subroutine foo(a, b, c)\nend subroutine foo\n' + result = _parse_subroutines(src) + self.assertIn('foo', result) + self.assertEqual(result['foo'].args, ['a', 'b', 'c']) + self.assertEqual(result['foo'].optional, set()) + + def test_case_insensitive_name(self): + src = 'SUBROUTINE MyScheme_run(errmsg, errflg)\nend subroutine\n' + result = _parse_subroutines(src) + self.assertIn('myscheme_run', result) + + def test_no_args(self): + src = 'subroutine bar()\nend subroutine bar\n' + result = _parse_subroutines(src) + self.assertEqual(result['bar'].args, []) + + def test_pure_prefix(self): + src = 'pure subroutine baz(x)\nend subroutine\n' + result = _parse_subroutines(src) + self.assertIn('baz', result) + self.assertEqual(result['baz'].args, ['x']) + + def test_elemental_prefix(self): + src = 'elemental subroutine qux(y)\nend subroutine\n' + result = _parse_subroutines(src) + self.assertIn('qux', result) + + def test_continuation_args(self): + src = 'subroutine foo(a, b, &\n c, d)\nend subroutine\n' + result = _parse_subroutines(src) + self.assertIn('foo', result) + self.assertEqual(sorted(result['foo'].args), ['a', 'b', 'c', 'd']) + + def test_multiple_subroutines(self): + src = ( + 'subroutine foo(a)\nend subroutine foo\n' + 'subroutine bar(x, y)\nend subroutine bar\n' + ) + result = _parse_subroutines(src) + self.assertIn('foo', result) + self.assertIn('bar', result) + + def test_nested_subroutine_ignored_if_same_name(self): + # first occurrence wins + src = ( + 'subroutine foo(a)\n' + ' subroutine foo(b, c)\n' + ' end subroutine\n' + 'end subroutine foo\n' + ) + result = _parse_subroutines(src) + self.assertEqual(result['foo'].args, ['a']) + + def test_no_parentheses(self): + # some Fortran compilers allow omitting () for no-arg subs + src = 'subroutine foo\nend subroutine foo\n' + result = _parse_subroutines(src) + self.assertIn('foo', result) + self.assertEqual(result['foo'].args, []) + + def test_fixed_form_dual_continuation_signature(self): + # Regression: rrtmg_lw_run-style F77 signatures use ``&`` at + # both line ends. Pre-fix this gave ``args == []``. + src = ( + ' subroutine rrtmg_lw_run &\n' + ' & ( plyr,plvl,tlyr,tlvl,qlyr,olyr, &\n' + ' & icseed,aeraod,aerssa, sfemis, sfgtmp, &\n' + ' & errmsg, errflg &\n' + ' & )\n' + ' end subroutine rrtmg_lw_run\n' + ) + result = _parse_subroutines(src) + self.assertIn('rrtmg_lw_run', result) + args = result['rrtmg_lw_run'].args + self.assertEqual(args, [ + 'plyr', 'plvl', 'tlyr', 'tlvl', 'qlyr', 'olyr', + 'icseed', 'aeraod', 'aerssa', 'sfemis', 'sfgtmp', + 'errmsg', 'errflg', + ]) + + +class TestLoadSourceTree(unittest.TestCase): + + def test_loads_correct_file(self): + tree = _load_source_tree([_CORRECT_F90]) + self.assertIn('temp_calc_adjust_run', tree) + self.assertIn('temp_calc_adjust_init', tree) + self.assertIn('temp_calc_adjust_final', tree) + + def test_args_from_correct_file(self): + tree = _load_source_tree([_CORRECT_F90]) + run_args = tree['temp_calc_adjust_run'].args + self.assertEqual(sorted(run_args), ['errflg', 'errmsg', 'im', 'temp', 'timestep']) + + def test_merges_multiple_files(self): + src1 = textwrap.dedent("""\ + subroutine aaa(x) + end subroutine aaa + """) + src2 = textwrap.dedent("""\ + subroutine bbb(y, z) + end subroutine bbb + """) + with tempfile.NamedTemporaryFile(suffix='.F90', mode='w', delete=False) as f1: + f1.write(src1) + p1 = f1.name + with tempfile.NamedTemporaryFile(suffix='.F90', mode='w', delete=False) as f2: + f2.write(src2) + p2 = f2.name + try: + tree = _load_source_tree([p1, p2]) + self.assertIn('aaa', tree) + self.assertIn('bbb', tree) + finally: + os.unlink(p1) + os.unlink(p2) + + +class TestValidateCorrect(unittest.TestCase): + + def test_no_errors_on_correct_source(self): + errors = validate([_SCHEME_META], [_CORRECT_F90]) + self.assertEqual(errors, []) + + +class TestValidateWrongArgs(unittest.TestCase): + + def setUp(self): + self.errors = validate([_SCHEME_META], [_WRONG_F90]) + + def test_has_errors(self): + self.assertGreater(len(self.errors), 0) + + def test_arg_count_error_for_init(self): + init_errs = [e for e in self.errors if 'temp_calc_adjust_init' in e] + self.assertGreater(len(init_errs), 0) + + def test_renamed_arg_error_for_run(self): + run_errs = [e for e in self.errors if 'temp_calc_adjust_run' in e] + self.assertGreater(len(run_errs), 0) + + +class TestDegenerateParseHint(unittest.TestCase): + """When the Fortran signature parser finds a subroutine but extracts + zero args while metadata declares many, the error message must + surface a HINT pointing at the parser rather than masquerading as + a routine mismatch. Triggered most commonly by an unsupported + continuation style; reproduced here with a parens-less Fortran + sub paired with multi-arg metadata.""" + + _META = ( + '[ccpp-table-properties]\n' + ' name = bogus_scheme\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = bogus_scheme_run\n' + ' type = scheme\n' + '[ a ]\n' + ' standard_name = horizontal_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = in\n' + '[ b ]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = inout\n' + ) + + _F90 = ( + '! Subroutine has no parentheses → parser yields zero args, but\n' + '! the metadata declares two. Hint must fire.\n' + 'subroutine bogus_scheme_run\n' + 'end subroutine bogus_scheme_run\n' + ) + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.meta_path = os.path.join(self.tmpdir, 'bogus_scheme.meta') + self.f90_path = os.path.join(self.tmpdir, 'bogus_scheme.F90') + with open(self.meta_path, 'w') as fh: + fh.write(self._META) + with open(self.f90_path, 'w') as fh: + fh.write(self._F90) + + def tearDown(self): + import shutil + shutil.rmtree(self.tmpdir, ignore_errors=True) + + def test_hint_fires_on_zero_fortran_args(self): + errors = validate([self.meta_path], [self.f90_path]) + count_errs = [e for e in errors if 'Argument count mismatch' in e] + self.assertEqual(len(count_errs), 1, errors) + msg = count_errs[0] + self.assertIn('HINT', msg) + self.assertIn('zero arguments', msg) + self.assertIn('parser', msg) + + +class TestOptionalArgsParsing(unittest.TestCase): + """_parse_subroutines collects optional-attribute arg names.""" + + def test_optional_first_attr(self): + src = textwrap.dedent("""\ + subroutine foo(a, b) + integer, optional, intent(in) :: a + integer, intent(in) :: b + end subroutine foo + """) + sig = _parse_subroutines(src)['foo'] + self.assertEqual(sig.args, ['a', 'b']) + self.assertEqual(sig.optional, {'a'}) + + def test_optional_after_intent(self): + src = textwrap.dedent("""\ + subroutine foo(a, b) + integer, intent(out), optional :: b + integer, intent(in) :: a + end subroutine foo + """) + sig = _parse_subroutines(src)['foo'] + self.assertEqual(sig.optional, {'b'}) + + def test_multiple_vars_one_decl(self): + src = textwrap.dedent("""\ + subroutine foo(a, b, c) + real, optional :: a, b(:,:), c + end subroutine foo + """) + sig = _parse_subroutines(src)['foo'] + self.assertEqual(sig.optional, {'a', 'b', 'c'}) + + def test_no_optional(self): + src = textwrap.dedent("""\ + subroutine foo(a) + integer, intent(in) :: a + end subroutine foo + """) + sig = _parse_subroutines(src)['foo'] + self.assertEqual(sig.optional, set()) + + def test_optional_token_in_string_or_comment_ignored(self): + src = textwrap.dedent("""\ + subroutine foo(a) + integer, intent(in) :: a ! this is optional, but a comment + end subroutine foo + """) + sig = _parse_subroutines(src)['foo'] + self.assertEqual(sig.optional, set()) + + +class TestValidateOptionalArgs(unittest.TestCase): + """Optional Fortran-only args are silently allowed in validation.""" + + _META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ a ] + standard_name = std_a + units = none + dimensions = () + type = integer + intent = in + [ b ] + standard_name = std_b + units = none + dimensions = () + type = integer + intent = in + """) + + _F90_OK = textwrap.dedent("""\ + module my_scheme + contains + subroutine my_scheme_run(a, b, c, d, e) + integer, intent(in) :: a + integer, intent(in) :: b + integer, optional, intent(in) :: c + integer, optional, intent(out) :: d + integer, optional, intent(in) :: e + end subroutine my_scheme_run + end module my_scheme + """) + + _F90_REQUIRED_EXTRA = textwrap.dedent("""\ + module my_scheme + contains + subroutine my_scheme_run(a, b, c) + integer, intent(in) :: a + integer, intent(in) :: b + integer, intent(in) :: c + end subroutine my_scheme_run + end module my_scheme + """) + + def _write_files(self, fortran_src): + with tempfile.NamedTemporaryFile(suffix='.meta', mode='w', delete=False) as fm: + fm.write(self._META) + meta_path = fm.name + with tempfile.NamedTemporaryFile(suffix='.F90', mode='w', delete=False) as ff: + ff.write(fortran_src) + f90_path = ff.name + self._cleanup = [meta_path, f90_path] + return meta_path, f90_path + + def tearDown(self): + for f in getattr(self, '_cleanup', []): + os.unlink(f) + + def test_optional_fortran_only_args_allowed(self): + meta, f90 = self._write_files(self._F90_OK) + errors = validate([meta], [f90]) + self.assertEqual(errors, [], 'unexpected errors: ' + repr(errors)) + + def test_non_optional_extra_fortran_arg_errors(self): + meta, f90 = self._write_files(self._F90_REQUIRED_EXTRA) + errors = validate([meta], [f90]) + self.assertTrue(any('Non-optional arguments in Fortran' in e + for e in errors), + 'expected non-optional-extra error, got: ' + repr(errors)) + + +class TestValidateMissingSubroutine(unittest.TestCase): + + def setUp(self): + # Only supply a file with temp_calc_adjust_init, missing run and final. + src = textwrap.dedent("""\ + module temp_calc_adjust + contains + subroutine temp_calc_adjust_init(im, errmsg, errflg) + end subroutine temp_calc_adjust_init + end module temp_calc_adjust + """) + with tempfile.NamedTemporaryFile(suffix='.F90', mode='w', delete=False) as fh: + fh.write(src) + self._f90 = fh.name + + def tearDown(self): + os.unlink(self._f90) + + def test_missing_subroutine_reported(self): + errors = validate([_SCHEME_META], [self._f90]) + missing = [e for e in errors if 'not found' in e] + self.assertGreater(len(missing), 0) + + +class TestValidateMultipleSources(unittest.TestCase): + """Subroutines split across multiple files.""" + + def setUp(self): + init_src = textwrap.dedent("""\ + module a + contains + subroutine temp_calc_adjust_init(im, errmsg, errflg) + end subroutine + end module a + """) + run_src = textwrap.dedent("""\ + module b + contains + subroutine temp_calc_adjust_run(im, timestep, temp, errmsg, errflg) + end subroutine + end module b + """) + final_src = textwrap.dedent("""\ + module c + contains + subroutine temp_calc_adjust_final(errmsg, errflg) + end subroutine + end module c + """) + self._files = [] + for src in (init_src, run_src, final_src): + with tempfile.NamedTemporaryFile(suffix='.F90', mode='w', delete=False) as fh: + fh.write(src) + self._files.append(fh.name) + + def tearDown(self): + for f in self._files: + os.unlink(f) + + def test_no_errors_split_across_files(self): + errors = validate([_SCHEME_META], self._files) + self.assertEqual(errors, []) + + +class TestSourcePathAutoDiscovery(unittest.TestCase): + """Validator auto-discovers .F90 when source_files is omitted.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + # Write a .meta file whose source_path points to a subdirectory. + self._src_subdir = os.path.join(self._tmpdir, 'fortran') + os.makedirs(self._src_subdir) + + meta_src = textwrap.dedent("""\ + [ccpp-table-properties] + name = myscheme + type = scheme + source_path = fortran + [ccpp-arg-table] + name = myscheme_run + type = scheme + [ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out + [ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + """) + self._meta = os.path.join(self._tmpdir, 'myscheme.meta') + with open(self._meta, 'w') as fh: + fh.write(meta_src) + + # Correct .F90 in the source_path subdirectory. + fort_correct = textwrap.dedent("""\ + module myscheme + contains + subroutine myscheme_run(errmsg, errflg) + end subroutine myscheme_run + end module myscheme + """) + self._fort = os.path.join(self._src_subdir, 'myscheme.F90') + with open(self._fort, 'w') as fh: + fh.write(fort_correct) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_auto_discovers_correct_source(self): + errors = validate([self._meta]) + self.assertEqual(errors, []) + + def test_auto_discovers_wrong_source(self): + # Overwrite with wrong arg list. + with open(self._fort, 'w') as fh: + fh.write(textwrap.dedent("""\ + module myscheme + contains + subroutine myscheme_run(errmsg) + end subroutine myscheme_run + end module myscheme + """)) + errors = validate([self._meta]) + self.assertGreater(len(errors), 0) + + +class TestFortranFileForTable(unittest.TestCase): + """Tests for _fortran_file_for_table helper.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def _make_table(self, source_path=''): + from metadata.parse_tools import ParseContext + from metadata.metadata_table import MetadataTable + meta = os.path.join(self._tmpdir, 'foo.meta') + open(meta, 'w').close() + ctx = ParseContext(0, meta) + t = MetadataTable('foo', 'scheme', meta, ctx) + props = {} + if source_path: + props['source_path'] = source_path + t.apply_table_props(props) + return t + + def test_finds_F90_in_meta_dir(self): + fort = os.path.join(self._tmpdir, 'foo.F90') + open(fort, 'w').close() + t = self._make_table() + result = val_mod._fortran_file_for_table(t) + self.assertEqual(result, fort) + + def test_finds_F90_in_source_path(self): + subdir = os.path.join(self._tmpdir, 'src') + os.makedirs(subdir) + fort = os.path.join(subdir, 'foo.F90') + open(fort, 'w').close() + t = self._make_table(source_path='src') + result = val_mod._fortran_file_for_table(t) + self.assertEqual(result, fort) + + def test_returns_none_when_not_found(self): + t = self._make_table() + result = val_mod._fortran_file_for_table(t) + self.assertIsNone(result) + + +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(val_mod)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_variable_resolver.py b/unit-tests/test_variable_resolver.py new file mode 100644 index 00000000..c7aa33db --- /dev/null +++ b/unit-tests/test_variable_resolver.py @@ -0,0 +1,930 @@ +"""Unit tests for metadata.variable_resolver. + +Covers: +- Helper functions (_ddt_typename, _is_intrinsic, _is_external, _is_known_ddt, + _instance_subscript, _build_ddt_index) +- HostVarEntry construction and properties +- _flatten_ddt_instance (single-level and nested) +- build_flat_host_dict (plain vars, DDT expansion, control vars, duplicates) +- SchemeStore (build_from, has_scheme, phases_for, variables_for, duplicates) +""" + +import os +import sys +import unittest +import doctest + +# Path setup is handled by conftest.py (pytest) or run_tests.py; the imports +# below are enough for direct invocation via this file's __main__ block. +from metadata.metadata_table import _parse_lines, MetadataTable, MetaVar +from metadata.parse_tools import ParseContext, CCPPError +from metadata.variable_resolver import ( + _ddt_typename, + _is_intrinsic, + _is_external, + _is_known_ddt, + _instance_subscript, + _build_ddt_index, + _flatten_ddt_instance, + build_ddt_module_map, + HostVarEntry, + build_flat_host_dict, + SchemeStore, +) + +# --------------------------------------------------------------------------- +# Sample file directory +# --------------------------------------------------------------------------- +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_SAMPLES_DIR = os.path.join(_TESTS_DIR, 'sample_files') + + +def _sample(name: str) -> str: + return os.path.join(_SAMPLES_DIR, name) + + +def _parse_file(name: str): + """Parse a sample metadata file and return its tables.""" + from metadata.metadata_table import parse_metadata_file + return parse_metadata_file(_sample(name)) + + +def _ctx(lineno: int = 0, filename: str = 'test.meta') -> ParseContext: + return ParseContext(linenum=lineno, filename=filename) + + +def _make_simple_var(local_name: str, std_name: str, units: str = '1', + dims: str = '()', type_: str = 'integer', + kind: str = '') -> MetaVar: + """Quick helper to build a validated MetaVar.""" + ctx = _ctx() + v = MetaVar(local_name, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', units, ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', type_, ctx) + if kind: + v.set_attr('kind', kind, ctx) + return v + + +def _make_ddt_instance_var(local_name: str, std_name: str, type_name: str, + dims: str = '()') -> MetaVar: + """Build a DDT instance MetaVar with the given dimensions.""" + ctx = _ctx() + v = MetaVar(local_name, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', 'none', ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', type_name, ctx) + return v + + +######################################################################## +# Tests: helper functions +######################################################################## + +class TestDdtTypename(unittest.TestCase): + + def test_plain_name(self): + self.assertEqual(_ddt_typename('gfs_statein_type'), 'gfs_statein_type') + + def test_type_paren(self): + self.assertEqual(_ddt_typename('type(gfs_statein_type)'), 'gfs_statein_type') + + def test_type_paren_uppercase(self): + self.assertEqual(_ddt_typename('TYPE(MY_DDT)'), 'MY_DDT') + + def test_type_paren_whitespace(self): + self.assertEqual(_ddt_typename('type( my_type )'), 'my_type') + + def test_intrinsic_passthrough(self): + self.assertEqual(_ddt_typename('real'), 'real') + + def test_external_passthrough(self): + self.assertEqual(_ddt_typename('external:mpi_f08:mpi_comm'), + 'external:mpi_f08:mpi_comm') + + +class TestIsIntrinsic(unittest.TestCase): + + def test_real(self): + self.assertTrue(_is_intrinsic('real')) + + def test_integer(self): + self.assertTrue(_is_intrinsic('integer')) + + def test_character(self): + self.assertTrue(_is_intrinsic('character')) + + def test_logical(self): + self.assertTrue(_is_intrinsic('logical')) + + def test_complex(self): + self.assertTrue(_is_intrinsic('complex')) + + def test_ddt_name(self): + self.assertFalse(_is_intrinsic('gfs_statein_type')) + + def test_external_syntax(self): + self.assertFalse(_is_intrinsic('external:mpi_f08:mpi_comm')) + + +class TestIsExternal(unittest.TestCase): + + def test_external_lowercase(self): + self.assertTrue(_is_external('external:mpi_f08:mpi_comm')) + + def test_external_uppercase(self): + self.assertTrue(_is_external('EXTERNAL:mpi_f08:mpi_comm')) + + def test_real(self): + self.assertFalse(_is_external('real')) + + def test_ddt_name(self): + self.assertFalse(_is_external('my_ddt_type')) + + +class TestIsKnownDdt(unittest.TestCase): + + def _idx(self, name: str) -> dict: + ctx = _ctx() + tbl = MetadataTable(name, 'ddt', 'f.meta', ctx) + return {name: tbl} + + def test_known_ddt(self): + self.assertTrue(_is_known_ddt('my_type', self._idx('my_type'))) + + def test_type_paren_form(self): + self.assertTrue(_is_known_ddt('type(my_type)', self._idx('my_type'))) + + def test_unknown_name(self): + self.assertFalse(_is_known_ddt('other_type', self._idx('my_type'))) + + def test_intrinsic_not_ddt(self): + self.assertFalse(_is_known_ddt('real', self._idx('my_type'))) + + def test_external_not_ddt(self): + self.assertFalse(_is_known_ddt('external:mpi_f08:mpi_comm', self._idx('my_type'))) + + def test_empty_index(self): + self.assertFalse(_is_known_ddt('my_type', {})) + + +class TestInstanceSubscript(unittest.TestCase): + + def test_instance_dimension(self): + v = _make_ddt_instance_var('gs', 'gst', 'gs_type', '(instance_dimension)') + self.assertEqual(_instance_subscript(v), '(instance_number)') + + def test_number_of_instances(self): + v = _make_ddt_instance_var('gs', 'gst', 'gs_type', '(number_of_instances)') + self.assertEqual(_instance_subscript(v), '(instance_number)') + + def test_scalar_no_subscript(self): + v = _make_ddt_instance_var('gs', 'gst', 'gs_type', '()') + self.assertEqual(_instance_subscript(v), '') + + def test_horizontal_dim_no_subscript(self): + v = _make_simple_var('x', 'air_temperature', 'K', + '(horizontal_dimension, vertical_layer_dimension)', + 'real', 'kind_phys') + self.assertEqual(_instance_subscript(v), '') + + +######################################################################## +# Tests: HostVarEntry +######################################################################## + +class TestHostVarEntry(unittest.TestCase): + + def _make(self, stdname='air_temperature', local='temp', + path='temp', module='mymod'): + return HostVarEntry( + stdname, local, path, module, + 'real', 'kind_phys', 'K', + ['horizontal_dimension', 'vertical_layer_dimension'], + False, False, '' + ) + + def test_basic_attributes(self): + e = self._make() + self.assertEqual(e.standard_name, 'air_temperature') + self.assertEqual(e.local_name, 'temp') + self.assertEqual(e.access_path, 'temp') + self.assertEqual(e.module_name, 'mymod') + self.assertEqual(e.type, 'real') + self.assertEqual(e.kind, 'kind_phys') + self.assertEqual(e.units, 'K') + self.assertEqual(e.dimensions, + ['horizontal_dimension', 'vertical_layer_dimension']) + self.assertFalse(e.protected) + self.assertFalse(e.optional) + self.assertEqual(e.active, '') + + def test_is_control_false(self): + e = self._make() + self.assertFalse(e.is_control) + + def test_is_control_true(self): + e = HostVarEntry('thread_number', 'thread_num', 'thread_num', None, + 'integer', '', '1', [], False, False, '') + self.assertTrue(e.is_control) + + def test_repr(self): + e = self._make() + self.assertIn('air_temperature', repr(e)) + self.assertIn('temp', repr(e)) + + def test_dimensions_are_copied(self): + dims = ['horizontal_dimension'] + e = HostVarEntry('x', 'x', 'x', 'mod', 'integer', '', '1', + dims, False, False, '') + dims.append('extra') + self.assertEqual(len(e.dimensions), 1) + + def test_equality_by_standard_name(self): + e1 = self._make('air_temperature') + e2 = self._make('air_temperature') + e3 = self._make('pressure') + self.assertEqual(e1, e2) + self.assertNotEqual(e1, e3) + + def test_hash(self): + e1 = self._make('air_temperature') + e2 = self._make('air_temperature') + self.assertEqual(hash(e1), hash(e2)) + + +######################################################################## +# Tests: _build_ddt_index +######################################################################## + +class TestBuildDdtIndex(unittest.TestCase): + + def test_single_table(self): + ctx = _ctx() + tbl = MetadataTable('gfs_statein_type', 'ddt', 'f.meta', ctx) + idx = _build_ddt_index([tbl]) + self.assertIn('gfs_statein_type', idx) + self.assertIs(idx['gfs_statein_type'], tbl) + + def test_multiple_tables(self): + ctx = _ctx() + t1 = MetadataTable('type_a', 'ddt', 'a.meta', ctx) + t2 = MetadataTable('type_b', 'ddt', 'b.meta', ctx) + idx = _build_ddt_index([t1, t2]) + self.assertEqual(set(idx.keys()), {'type_a', 'type_b'}) + + def test_empty(self): + self.assertEqual(_build_ddt_index([]), {}) + + def test_from_file(self): + tables = _parse_file('ddt_simple.meta') + idx = _build_ddt_index(tables) + self.assertIn('gfs_statein_type', idx) + + +######################################################################## +# Tests: build_ddt_module_map +######################################################################## + +class TestBuildDdtModuleMap(unittest.TestCase): + + def test_ddt_co_located_with_scheme(self): + ctx = _ctx() + ddt_tbl = MetadataTable('vmr_type', 'ddt', 'make_ddt.meta', ctx) + sch_tbl = MetadataTable('make_ddt', 'scheme', 'make_ddt.meta', ctx) + result = build_ddt_module_map([ddt_tbl, sch_tbl]) + self.assertEqual(result, {'vmr_type': 'make_ddt'}) + + def test_ddt_co_located_with_host(self): + ctx = _ctx() + ddt_tbl = MetadataTable('payload_t', 'ddt', 'host.meta', ctx) + host_tbl = MetadataTable('my_host', 'host', 'host.meta', ctx) + result = build_ddt_module_map([ddt_tbl, host_tbl]) + self.assertEqual(result, {'payload_t': 'my_host'}) + + def test_ddt_alone_in_file_skipped(self): + ctx = _ctx() + orphan = MetadataTable('lonely_t', 'ddt', 'lonely.meta', ctx) + self.assertEqual(build_ddt_module_map([orphan]), {}) + + def test_multiple_files(self): + ctx = _ctx() + d1 = MetadataTable('t1', 'ddt', 'a.meta', ctx) + s1 = MetadataTable('mod_a', 'scheme', 'a.meta', ctx) + d2 = MetadataTable('t2', 'ddt', 'b.meta', ctx) + h2 = MetadataTable('mod_b', 'host', 'b.meta', ctx) + result = build_ddt_module_map([d1, s1, d2, h2]) + self.assertEqual(result, {'t1': 'mod_a', 't2': 'mod_b'}) + + def test_multiple_ddts_in_one_file(self): + ctx = _ctx() + d1 = MetadataTable('inner', 'ddt', 'm.meta', ctx) + d2 = MetadataTable('outer', 'ddt', 'm.meta', ctx) + s = MetadataTable('host_mod', 'host', 'm.meta', ctx) + result = build_ddt_module_map([d1, d2, s]) + self.assertEqual(result, {'inner': 'host_mod', 'outer': 'host_mod'}) + + def test_empty(self): + self.assertEqual(build_ddt_module_map([]), {}) + + +######################################################################## +# Tests: _flatten_ddt_instance +######################################################################## + +class TestFlattenDdtInstance(unittest.TestCase): + + def _load_ddt_gfs(self): + return _build_ddt_index(_parse_file('ddt_simple.meta')) + + def test_scalar_instance_paths(self): + """Scalar DDT instance (no dimensions) → field paths with % separator.""" + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('gfs_statein', 'gfs_statein', 'gfs_statein_type') + entries = _flatten_ddt_instance(var, 'CCPP_data', idx) + std_names = {e.standard_name for e in entries} + self.assertIn('gfs_statein', std_names) # instance itself + self.assertIn('geopotential_at_interface', std_names) + self.assertIn('geopotential', std_names) + + def test_scalar_instance_access_paths(self): + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('gfs_statein', 'gfs_statein', 'gfs_statein_type') + entries = {e.standard_name: e + for e in _flatten_ddt_instance(var, 'CCPP_data', idx)} + self.assertEqual(entries['geopotential_at_interface'].access_path, + 'gfs_statein%phii') + self.assertEqual(entries['geopotential'].access_path, + 'gfs_statein%phil') + + def test_arrayed_instance_subscript(self): + """DDT instance with instance dimension → access path has (instance_number).""" + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('gfs_statein', 'gfs_statein', + 'gfs_statein_type', '(number_of_instances)') + entries = {e.standard_name: e + for e in _flatten_ddt_instance(var, 'CCPP_data', idx)} + self.assertEqual(entries['geopotential_at_interface'].access_path, + 'gfs_statein(instance_number)%phii') + + def test_module_name_propagated(self): + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('gfs_statein', 'gfs_statein', 'gfs_statein_type') + for entry in _flatten_ddt_instance(var, 'GFS_typedefs', idx): + self.assertEqual(entry.module_name, 'GFS_typedefs') + + def test_unknown_ddt_type_raises(self): + idx = {} # empty — type not found + var = _make_ddt_instance_var('gs', 'gs', 'unknown_type') + with self.assertRaises(CCPPError) as cm: + _flatten_ddt_instance(var, 'mod', idx) + self.assertIn('unknown_type', str(cm.exception)) + + def test_field_type_and_kind(self): + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('gfs_statein', 'gfs_statein', 'gfs_statein_type') + entries = {e.standard_name: e + for e in _flatten_ddt_instance(var, 'CCPP_data', idx)} + phii = entries['geopotential_at_interface'] + self.assertEqual(phii.type, 'real') + self.assertEqual(phii.kind, 'kind_phys') + self.assertEqual(phii.units, 'm2 s-2') + + def test_prefix_propagated(self): + """access_prefix is prepended to all paths (used in nested DDT recursion).""" + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('inner', 'inner', 'gfs_statein_type') + entries = {e.standard_name: e + for e in _flatten_ddt_instance( + var, 'mod', idx, access_prefix='outer%')} + self.assertEqual(entries['geopotential_at_interface'].access_path, + 'outer%inner%phii') + + def test_max_depth_guard(self): + """A deeply nested structure beyond max_depth raises CCPPError.""" + idx = self._load_ddt_gfs() + var = _make_ddt_instance_var('x', 'x', 'gfs_statein_type') + with self.assertRaises(CCPPError) as cm: + _flatten_ddt_instance(var, 'mod', idx, depth=10, max_depth=8) + self.assertIn('depth', str(cm.exception)) + + +class TestFlattenNestedDdt(unittest.TestCase): + """End-to-end nested DDT flattening using sample files.""" + + def _load_nested(self): + outer_tables = _parse_file('ddt_nested_outer.meta') + inner_tables = _parse_file('ddt_nested_inner.meta') + return _build_ddt_index(outer_tables + inner_tables) + + def test_nested_field_standard_names(self): + idx = self._load_nested() + host_tables = _parse_file('host_with_nested_ddt.meta') + var = host_tables[0].sections()[0].variables[0] + entries = {e.standard_name: e + for e in _flatten_ddt_instance(var, 'nested_host_mod', idx)} + expected = { + 'outer_ddt_instance', # the outer DDT instance itself + 'outer_scalar_field', # plain field on the outer DDT + 'inner_ddt_instance', # the inner DDT instance (as a field) + 'inner_real_value', # field of the inner DDT + 'inner_integer_flag', # field of the inner DDT + } + self.assertEqual(set(entries.keys()), expected) + + def test_nested_access_paths(self): + idx = self._load_nested() + host_tables = _parse_file('host_with_nested_ddt.meta') + var = host_tables[0].sections()[0].variables[0] + entries = {e.standard_name: e + for e in _flatten_ddt_instance(var, 'nested_host_mod', idx)} + self.assertEqual(entries['outer_scalar_field'].access_path, + 'outer_inst%scalar_field') + self.assertEqual(entries['inner_real_value'].access_path, + 'outer_inst%inner_ddt%inner_value') + self.assertEqual(entries['inner_integer_flag'].access_path, + 'outer_inst%inner_ddt%inner_flag') + + def test_nested_module_name(self): + idx = self._load_nested() + host_tables = _parse_file('host_with_nested_ddt.meta') + var = host_tables[0].sections()[0].variables[0] + for entry in _flatten_ddt_instance(var, 'nested_host_mod', idx): + self.assertEqual(entry.module_name, 'nested_host_mod') + + +######################################################################## +# Tests: build_flat_host_dict +######################################################################## + +class TestBuildFlatHostDict(unittest.TestCase): + + def test_plain_host_vars(self): + host_tables = _parse_file('host_simple.meta') + d = build_flat_host_dict(host_tables, [], []) + self.assertIn('horizontal_dimension', d) + self.assertIn('vertical_layer_dimension', d) + # number_of_instances is a host-table var (USE'd by the suite cap) + self.assertIn('number_of_instances', d) + # loop bounds and error vars live in the control table, not the host table + self.assertNotIn('horizontal_loop_begin', d) + self.assertNotIn('horizontal_loop_end', d) + self.assertEqual(len(d), 3) + + def test_plain_host_access_paths(self): + host_tables = _parse_file('host_simple.meta') + d = build_flat_host_dict(host_tables, [], []) + # For plain vars the access path equals the local name. + self.assertEqual(d['horizontal_dimension'].access_path, 'ncols') + self.assertEqual(d['vertical_layer_dimension'].access_path, 'nlev') + + def test_plain_host_module_names(self): + host_tables = _parse_file('host_simple.meta') + d = build_flat_host_dict(host_tables, [], []) + for entry in d.values(): + self.assertEqual(entry.module_name, 'physics_data') + self.assertFalse(entry.is_control) + + def test_host_module_name_override(self): + """A ``type=host`` table that declares ``module_name`` in its + ``[ccpp-table-properties]`` should override the default convention + (module name = table name) so subsequent USE statements target the + actual Fortran module.""" + src = ''' +[ccpp-table-properties] + name = host_data + type = host + module_name = mod_host_data + +[ccpp-arg-table] + name = host_data + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +''' + tables = _parse_lines(src.splitlines(keepends=True), 'h.meta') + d = build_flat_host_dict(tables, [], []) + self.assertEqual(d['horizontal_dimension'].module_name, 'mod_host_data') + + def test_host_module_name_defaults_to_table_name(self): + """Without ``module_name`` the module defaults to the table name.""" + src = ''' +[ccpp-table-properties] + name = host_data + type = host + +[ccpp-arg-table] + name = host_data + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +''' + tables = _parse_lines(src.splitlines(keepends=True), 'h.meta') + d = build_flat_host_dict(tables, [], []) + self.assertEqual(d['horizontal_dimension'].module_name, 'host_data') + + def test_control_vars_no_module(self): + ctrl_tables = _parse_file('control_simple.meta') + d = build_flat_host_dict([], ctrl_tables, []) + self.assertIn('suite_name', d) + self.assertIn('group_name', d) + self.assertIn('thread_number', d) + self.assertIn('number_of_physics_threads', d) + for entry in d.values(): + self.assertIsNone(entry.module_name) + self.assertTrue(entry.is_control) + + def test_ddt_instance_expansion(self): + host_tables = _parse_file('host_with_ddt_instance.meta') + ddt_tables = _parse_file('ddt_simple.meta') + d = build_flat_host_dict(host_tables, [], ddt_tables) + # DDT instance itself + two fields + self.assertIn('gfs_statein', d) + self.assertIn('geopotential_at_interface', d) + self.assertIn('geopotential', d) + + def test_ddt_instance_subscript_in_path(self): + """DDT instance with (number_of_instances) → (instance_number) in path.""" + host_tables = _parse_file('host_with_ddt_instance.meta') + ddt_tables = _parse_file('ddt_simple.meta') + d = build_flat_host_dict(host_tables, [], ddt_tables) + self.assertEqual(d['geopotential_at_interface'].access_path, + 'gfs_statein(instance_number)%phii') + self.assertEqual(d['geopotential'].access_path, + 'gfs_statein(instance_number)%phil') + + def test_ddt_instance_module_name(self): + host_tables = _parse_file('host_with_ddt_instance.meta') + ddt_tables = _parse_file('ddt_simple.meta') + d = build_flat_host_dict(host_tables, [], ddt_tables) + self.assertEqual(d['geopotential_at_interface'].module_name, 'CCPP_data') + + def test_host_and_control_combined(self): + host_tables = _parse_file('host_simple.meta') + ctrl_tables = _parse_file('control_simple.meta') + d = build_flat_host_dict(host_tables, ctrl_tables, []) + self.assertIn('horizontal_dimension', d) + self.assertIn('suite_name', d) + self.assertFalse(d['horizontal_dimension'].is_control) + self.assertTrue(d['suite_name'].is_control) + + def test_duplicate_standard_name_raises(self): + """Same standard name in two host tables must raise CCPPError. + The message must include both access paths so the user can see + which two declarations collide.""" + src = ''' +[ccpp-table-properties] + name = mod_a + type = host +[ccpp-arg-table] + name = mod_a + type = host +[ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +''' + tables_a = _parse_lines(src.splitlines(keepends=True), 'a.meta') + tables_b = _parse_lines(src.replace('mod_a', 'mod_b') + .splitlines(keepends=True), 'b.meta') + with self.assertRaises(CCPPError) as cm: + build_flat_host_dict(tables_a + tables_b, [], []) + msg = str(cm.exception) + self.assertIn('horizontal_dimension', msg) + # Both module names must appear so the user can locate the duplicates. + self.assertIn('mod_a', msg) + self.assertIn('mod_b', msg) + # And both access paths. + self.assertIn('access path', msg) + + def test_duplicate_ddt_component_names_path_collision(self): + """Two sibling DDT instances of the same type inside one parent + DDT cause component standard names to collide. The error must + show both colliding access paths so the user can spot the issue + immediately (this is the scm_type_defs / GFS_interstitial_type + sliced-view-vs-array pattern).""" + # parent_ddt has two sibling fields of the same inner_ddt type: + # one bare and one with a slice in the local name. Both + # flatten into entries for ``foo_std`` (inner_ddt's only + # component) — the second insertion triggers the duplicate. + src = ''' +[ccpp-table-properties] + name = inner_ddt + type = ddt +[ccpp-arg-table] + name = inner_ddt + type = ddt +[ foo ] + standard_name = foo_std + units = 1 + dimensions = () + type = integer + +[ccpp-table-properties] + name = parent_ddt + type = ddt +[ccpp-arg-table] + name = parent_ddt + type = ddt +[ inner_a ] + standard_name = inner_ddt_instance_a + units = ddt + dimensions = () + type = inner_ddt +[ inner_b ] + standard_name = inner_ddt_instance_b + units = ddt + dimensions = () + type = inner_ddt + +[ccpp-table-properties] + name = my_host + type = host +[ccpp-arg-table] + name = my_host + type = host +[ parent ] + standard_name = parent_ddt_instance + units = ddt + dimensions = () + type = parent_ddt +''' + tables = _parse_lines(src.splitlines(keepends=True), 'm.meta') + ddt_tables = [t for t in tables if t.table_type == 'ddt'] + host_tables = [t for t in tables if t.table_type == 'host'] + with self.assertRaises(CCPPError) as cm: + build_flat_host_dict(host_tables, [], ddt_tables) + msg = str(cm.exception) + # The duplicated standard name appears. + self.assertIn('foo_std', msg) + # Both colliding access paths appear so the user can diagnose. + self.assertIn('parent%inner_a%foo', msg) + self.assertIn('parent%inner_b%foo', msg) + # Hint about sibling-DDT-instance pattern is present. + self.assertIn('sibling DDT instances', msg) + + def test_missing_ddt_table_raises(self): + """DDT instance without corresponding DDT table → CCPPError.""" + host_tables = _parse_file('host_with_ddt_instance.meta') + with self.assertRaises(CCPPError) as cm: + build_flat_host_dict(host_tables, [], []) + self.assertIn('gfs_statein_type', str(cm.exception)) + + def test_host_vars_not_control(self): + # Host vars are is_control=False; loop bounds are now control vars (control table). + host_tables = _parse_file('host_simple.meta') + d = build_flat_host_dict(host_tables, [], []) + self.assertFalse(d['horizontal_dimension'].is_control) + self.assertFalse(d['vertical_layer_dimension'].is_control) + # Control vars from the control table should be is_control=True. + ctrl_tables = _parse_file('control_simple.meta') + dc = build_flat_host_dict([], ctrl_tables, []) + self.assertTrue(dc['horizontal_loop_begin'].is_control) + self.assertTrue(dc['horizontal_loop_end'].is_control) + + def test_nested_ddt_full_expansion(self): + outer_tables = _parse_file('ddt_nested_outer.meta') + inner_tables = _parse_file('ddt_nested_inner.meta') + host_tables = _parse_file('host_with_nested_ddt.meta') + d = build_flat_host_dict(host_tables, [], outer_tables + inner_tables) + self.assertIn('outer_ddt_instance', d) + self.assertIn('outer_scalar_field', d) + self.assertIn('inner_real_value', d) + self.assertIn('inner_integer_flag', d) + self.assertEqual(d['inner_real_value'].access_path, + 'outer_inst%inner_ddt%inner_value') + + def test_empty_inputs(self): + d = build_flat_host_dict([], [], []) + self.assertEqual(d, {}) + + +######################################################################## +# Tests: SchemeStore +######################################################################## + +_SIMPLE_SCHEME_SRC = '''\ +[ccpp-table-properties] + name = my_scheme + type = scheme + +[ccpp-arg-table] + name = my_scheme_run + type = scheme +[ im ] + standard_name = horizontal_loop_extent + units = count + dimensions = () + type = integer + intent = in +[ temp ] + standard_name = air_temperature + units = K + dimensions = (horizontal_loop_extent, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + +[ccpp-arg-table] + name = my_scheme_init + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + + +class TestSchemeStore(unittest.TestCase): + + def _build(self, src=_SIMPLE_SCHEME_SRC): + tables = _parse_lines(src.splitlines(keepends=True), 's.meta') + return SchemeStore.build_from(tables) + + def test_build_from_single_scheme(self): + store = self._build() + self.assertTrue(store.has_scheme('my_scheme')) + + def test_has_scheme_false(self): + store = self._build() + self.assertFalse(store.has_scheme('nonexistent')) + + def test_phases_for(self): + store = self._build() + self.assertEqual(sorted(store.phases_for('my_scheme')), ['init', 'run']) + + def test_phases_for_unknown(self): + store = self._build() + self.assertEqual(store.phases_for('nonexistent'), []) + + def test_variables_for_run(self): + store = self._build() + vars_ = store.variables_for('my_scheme', 'run') + self.assertIsNotNone(vars_) + std_names = [v.standard_name for v in vars_] + self.assertEqual(std_names, ['horizontal_loop_extent', 'air_temperature']) + + def test_variables_for_init(self): + store = self._build() + vars_ = store.variables_for('my_scheme', 'init') + self.assertIsNotNone(vars_) + std_names = [v.standard_name for v in vars_] + self.assertIn('ccpp_error_message', std_names) + self.assertIn('ccpp_error_code', std_names) + + def test_variables_for_absent_phase(self): + store = self._build() + self.assertIsNone(store.variables_for('my_scheme', 'final')) + + def test_variables_for_unknown_scheme(self): + store = self._build() + self.assertIsNone(store.variables_for('unknown', 'run')) + + def test_module_for_defaults_to_scheme_name(self): + store = self._build() + self.assertEqual(store.module_for('my_scheme'), 'my_scheme') + + def test_module_for_honors_table_property(self): + """``module_name`` in ``[ccpp-table-properties]`` overrides the + default (scheme-name) module.""" + src = ''' +[ccpp-table-properties] + name = effr_pre + type = scheme + module_name = mod_effr_pre + +[ccpp-arg-table] + name = effr_pre_run + type = scheme +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + tables = _parse_lines(src.splitlines(keepends=True), 's.meta') + store = SchemeStore.build_from(tables) + self.assertEqual(store.module_for('effr_pre'), 'mod_effr_pre') + + def test_module_for_unknown_scheme_returns_name(self): + store = self._build() + self.assertEqual(store.module_for('unknown_scheme'), 'unknown_scheme') + + def test_scheme_names_sorted(self): + src2 = _SIMPLE_SCHEME_SRC + ''' +[ccpp-table-properties] + name = another_scheme + type = scheme + +[ccpp-arg-table] + name = another_scheme_run + type = scheme +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + tables = _parse_lines(src2.splitlines(keepends=True), 's.meta') + store = SchemeStore.build_from(tables) + self.assertEqual(store.scheme_names(), ['another_scheme', 'my_scheme']) + + def test_non_scheme_tables_skipped(self): + host_src = '''\ +[ccpp-table-properties] + name = host_mod + type = host + +[ccpp-arg-table] + name = host_mod + type = host +[ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +''' + tables = (_parse_lines(_SIMPLE_SCHEME_SRC.splitlines(keepends=True), 's.meta') + + _parse_lines(host_src.splitlines(keepends=True), 'h.meta')) + store = SchemeStore.build_from(tables) + self.assertFalse(store.has_scheme('host_mod')) + self.assertTrue(store.has_scheme('my_scheme')) + + def test_duplicate_phase_raises(self): + src_dup = _SIMPLE_SCHEME_SRC + '''\ +[ccpp-arg-table] + name = my_scheme_run + type = scheme +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + tables = _parse_lines(src_dup.splitlines(keepends=True), 's.meta') + with self.assertRaises(CCPPError) as cm: + SchemeStore.build_from(tables) + self.assertIn('run', str(cm.exception)) + self.assertIn('my_scheme', str(cm.exception)) + + def test_build_from_scheme_files(self): + """Integration: build SchemeStore from the multipart scheme sample file.""" + from metadata.metadata_table import parse_metadata_file + tables = parse_metadata_file(_sample('scheme_multipart.meta')) + store = SchemeStore.build_from(tables) + self.assertTrue(store.has_scheme('temp_calc_adjust')) + self.assertEqual(sorted(store.phases_for('temp_calc_adjust')), + ['final', 'init', 'run']) + + def test_repr(self): + store = self._build() + self.assertIn('my_scheme', repr(store)) + + def test_variable_list_is_copy(self): + """Mutating the returned list must not affect the store's internal state.""" + store = self._build() + vars1 = store.variables_for('my_scheme', 'run') + vars1.append(None) + vars2 = store.variables_for('my_scheme', 'run') + self.assertEqual(len(vars2), 2) + + +######################################################################## +# Doctest loader +######################################################################## + +def load_tests(loader, tests, ignore): + """Auto-discover doctests from variable_resolver and related modules.""" + import metadata.variable_resolver as vr + tests.addTests(doctest.DocTestSuite(vr)) + return tests + + +######################################################################## +# main +######################################################################## + +if __name__ == '__main__': + unittest.main(verbosity=2) From 43fb5b24b02f3880fcbc9eea464dc80c1d764c2e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 07:33:47 -0600 Subject: [PATCH 04/74] Add doc/ --- doc/constituents.md | 1029 +++++++ doc/constituents_overhaul.md | 829 ++++++ doc/migration.md | 545 ++++ ...sign_analysis - original 202060505T2044.md | 2489 ++++++++++++++++ doc/redesign_analysis.md | 2639 +++++++++++++++++ ...ign_analysis.md - updated 20260513T0733.md | 2639 +++++++++++++++++ ...edesign_prompt - original 20260505T2044.md | 814 +++++ doc/redesign_prompt.md | 1172 ++++++++ ...esign_prompt.md - updated 20260513T0733.md | 1172 ++++++++ 9 files changed, 13328 insertions(+) create mode 100644 doc/constituents.md create mode 100644 doc/constituents_overhaul.md create mode 100644 doc/migration.md create mode 100644 doc/redesign_analysis - original 202060505T2044.md create mode 100644 doc/redesign_analysis.md create mode 100644 doc/redesign_analysis.md - updated 20260513T0733.md create mode 100644 doc/redesign_prompt - original 20260505T2044.md create mode 100644 doc/redesign_prompt.md create mode 100644 doc/redesign_prompt.md - updated 20260513T0733.md diff --git a/doc/constituents.md b/doc/constituents.md new file mode 100644 index 00000000..8e5a7b77 --- /dev/null +++ b/doc/constituents.md @@ -0,0 +1,1029 @@ +# CCPP capgen-ng — Constituents Reference + +*Last revised: 2026-05-11.* + +This document is the authoritative reference for **constituent variables** in +capgen-ng — what they are, how scheme authors declare them in metadata, what +the host model has to do to plumb them through, what the generator emits, and +how the per-instance lifecycle works. + +> If you are migrating a host or scheme from the original capgen, jump to +> [§9 Differences from original capgen](#9-differences-from-original-capgen) +> first. + +--- + +## Table of Contents + +1. [What is a constituent?](#1-what-is-a-constituent) +2. [The four rules (scheme-author conventions)](#2-the-four-rules-scheme-author-conventions) +3. [Required host metadata + Fortran](#3-required-host-metadata--fortran) +4. [Host-side lifecycle (call sequence)](#4-host-side-lifecycle-call-sequence) +5. [Public API reference](#5-public-api-reference) +6. [Generated code structure](#6-generated-code-structure) +7. [Multi-instance design](#7-multi-instance-design) +8. [Limitations and gotchas](#8-limitations-and-gotchas) +9. [Differences from original capgen](#9-differences-from-original-capgen) +10. [Worked example](#10-worked-example) + +--- + +## 1. What is a constituent? + +A **constituent** is a model variable owned by the host's dynamical core (or +its constituent infrastructure) that is read and updated by physics schemes — +typically a tracer / mass mixing ratio (water vapor, cloud liquid, ozone, +chemistry species) — together with its **tendency**, the rate of change that +physics writes back so the dycore can advect/integrate it forward. + +In capgen-ng, the constituent layer has three concerns: + +1. **Registration** — declaring at model startup which constituents exist + (their standard name, units, vertical layout, advection flag, …). +2. **Storage** — the framework owns one `ccpp_model_constituents_t` DDT per + host instance (see [§7](#7-multi-instance-design)) which holds the + constituent values (`%vars_layer`), tendencies (`%vars_layer_tend`), and + metadata (`%const_metadata`). +3. **Access** — schemes reference constituents by standard name in their + metadata; the resolver translates those references to + `ccpp_model_constituents_obj(inst)%vars_layer(slice, index_of_)` + subscripts at code-gen time. + +All constituent state lives in **one generated module**: +`ccpp_host_constituents.F90` (one per generator run, emitted only when at +least one suite touches constituent state). Public symbols from this module +are also re-exported by `ccpp_static_api`, so most host code only needs + +```fortran +use ccpp_static_api, only: ccpp_register_constituents, ccpp_initialize_constituents, & + ccpp_constituents_array, ccpp_const_get_index, ... +``` + +--- + +## 2. The four rules (scheme-author conventions) + +These four rules govern every scheme-arg metadata pattern related to +constituents. They derive from a 2026-05-11 audit of all 12 cam-sima scheme +metadata files that touch constituent attributes. + +### Rule 1 — Register a new constituent (register phase) + +A scheme that creates a new constituent declares it in the **register** +phase via an `intent=out, allocatable` array of +`ccpp_constituent_properties_t`: + +``` +[ccpp-arg-table] + name = my_scheme_register + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_my_scheme + long_name = per-scheme constituent array + units = none + dimensions = (:) + type = ccpp_constituent_properties_t + allocatable = True + intent = out +[ errmsg ] + ... +[ errflg ] + ... +``` + +The scheme's Fortran register routine `allocate`s this array, populates +each entry via `%instantiate(std_name=..., long_name=..., units=..., +vertical_dim=..., advected=..., ...)` and returns it. The framework +captures every register-phase scheme's array, packs them into a per-suite +buffer (`_dynamic_constituents`), and merges them into each +host-instance's constituent object during `ccpp_register_constituents`. + +This is the **only path** for declaring a new constituent. + +### Rule 2 — Consume a base constituent (any physics phase) + +A scheme that reads (or reads + writes) an existing base constituent +declares the variable with `is_constituent` set (any of `advected`, +`constituent`, or `molar_mass` non-default) and `intent=in` or `intent=inout`: + +``` +[ cldliq ] + standard_name = cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in ! or inout + advected = true +``` + +The resolver translates this scheme arg to +`ccpp_model_constituents_obj()%vars_layer(, index_of_)` +in the generated group cap. No host metadata declaration is needed for +the variable. + +### Rule 3 — Produce a tendency (any physics phase) + +A scheme that writes a constituent tendency declares the variable with +`is_constituent` set, `intent=out`, and a standard name that **starts +with `tendency_of_`**: + +``` +[ tend_cldliq ] + standard_name = tendency_of_cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = true +``` + +The resolver translates this scheme arg to +`ccpp_model_constituents_obj()%vars_layer_tend(, index_of_)` +where `` is the std_name with the `tendency_of_` prefix stripped. +The tendency variable is implicitly tied to the base constituent of the +same name. + +### Rule 4 — Mismatched combinations are hard errors + +Two combinations are explicitly rejected by the resolver at code-gen time: + +| Mismatch | Error | +|---|---| +| `is_constituent=True` + `intent=out` + std_name does NOT start with `tendency_of_` | *"Physics phases may only produce constituent tendencies; new base constituents must be declared via a `ccpp_constituent_properties_t` argument in a register-phase scheme."* | +| `is_constituent=True` + `intent in (in, inout)` + std_name starts with `tendency_of_` | *"Constituent tendency arg must be declared with intent=out; physics phases only produce tendencies, never consume them."* | + +### Direct framework-array access + +A scheme may also access the framework's bulk arrays directly by +declaring an arg with one of these standard names: + +| Standard name | Maps to | +|---|---| +| `ccpp_constituents` | `ccpp_model_constituents_obj()%vars_layer` (3D) | +| `ccpp_constituent_tendencies` | `ccpp_model_constituents_obj()%vars_layer_tend` (3D) | +| `ccpp_constituent_properties` | `ccpp_model_constituents_obj()%const_metadata` (1D of `ccpp_constituent_prop_ptr_t`) | +| `number_of_ccpp_constituents` | `ccpp_model_constituents_obj()%num_layer_vars` (scalar integer) | +| `index_of_` | module-level `integer :: index_of_` (no per-instance access — the index is identical for every instance) | + +The trailing dimension `number_of_ccpp_constituents` in a 3D scheme arg +is emitted as `:` (whole-axis slice). + +--- + +## 3. Required host metadata + Fortran + +### Host metadata (`type=host` table) + +The host **must** declare: + +``` +[ ] + standard_name = number_of_instances + units = count + dimensions = () + type = integer +``` + +… **only when the host actually wants multi-instance support**. When +absent, every per-instance allocation falls back to size `1` and the +host effectively runs single-instance. + +The host **does not** need to declare: + +- `ccpp_model_constituents_object` — the constituent object is owned + by the generator (in `ccpp_host_constituents`); the host doesn't + declare it in metadata. +- `ccpp_constituents`, `ccpp_constituent_tendencies`, + `ccpp_constituent_properties`, `number_of_ccpp_constituents`, + `index_of_` — all auto-provided by the generator. + +#### Host metadata wins over auto-provisioning + +If the host **does** declare any of the framework-named standard +names above as a regular host variable, the resolver uses the host's +declaration instead of auto-provisioning. This matters for legacy +hosts (GFS / SCM) that own their own tracer indices: + +```meta +[ ntcw ] + standard_name = index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array + units = index + type = integer + protected = True + dimensions = () +``` + +A scheme arg requesting the same standard name resolves to the host's +short local name (`ntcw`), not a parallel module-level integer in +`ccpp_host_constituents` named after the full standard name (which +would blow the Fortran 63-character identifier limit). Auto-provisioning +only fires for framework-named standard names the host has **not** +claimed. + +### Host control-table requirements + +The host's `type=control` table must declare: + +``` +[ ] + standard_name = instance_number + units = 1 + dimensions = () + type = integer +``` + +… so the framework signature knows the index for per-instance state. +Same caveat as `number_of_instances` — required only when multi-instance +is wanted. + +### Host Fortran code + +The host's Fortran code only needs to: + +1. Maintain its own `integer :: ` for `number_of_instances` + in a module that's USE'd by the generator. (Same module that owns + the metadata.) +2. Build its **host constituents** array (water vapor, ozone, etc. — + the constituents that the host model owns directly, separately from + any scheme-registered ones). Pass this to + `ccpp_register_constituents`. + +The host does **not** need to allocate or own a +`type(ccpp_model_constituents_t)` variable. + +--- + +## 4. Host-side lifecycle (call sequence) + +``` + ┌─ host startup ─┐ + │ + ▼ + ┌──────────────────────────────────────┐ + │ for each instance: │ + │ ccpp_register(suite_name, │ + │ errmsg, errflg, │ + │ instance_number) │ ─── per-instance ───┐ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ allocate host_constituents(:) │ │ + │ host_constituents(1)%instantiate( │ ─── once ─────────┘ + │ std_name='water_vapor_specific_humidity', ...) │ + │ ... │ │ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_register_constituents( │ │ + │ host_constituents, │ │ + │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_initialize_constituents( │ │ + │ ncols, num_layers, │ │ + │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_init(suite_name, │ │ + │ errmsg, errflg, │ │ + │ instance_number) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ (model time-stepping) │ + ┌──────────────────────────────────────┐ │ + │ ccpp_physics_*(...) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ (host shutdown) │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_deallocate_dynamic_constituents( │ + │ instance_number) │ ─── per-instance ──┤ + │ ccpp_final(suite_name, │ │ + │ errmsg, errflg, │ │ + │ instance_number) │ │ + └──────────────────────────────────────┘ │ + │ + ┌────────────────────────────────────────┘ + │ ◀── last-to-leave dealloc fires + │ automatically inside the per-instance + │ calls when the final instance finishes. + ▼ +``` + +### Important ordering rules + +- `ccpp_register_constituents` **must** be called *after* `ccpp_register` + (per instance). The latter populates the per-suite dynamic-constituent + buffers via `_register`; the former merges them into the + per-instance constituent object. +- `ccpp_initialize_constituents` **must** be called *after* + `ccpp_register_constituents` (per instance). It calls `%lock_data` + on the per-instance object — which can only happen once + `%lock_table` has fired (which `ccpp_register_constituents` does). +- Physics phases (`ccpp_init`, `ccpp_physics_run`, etc.) require + the constituent state to be locked + bound (i.e., + `ccpp_initialize_constituents` already called). +- `ccpp_deallocate_dynamic_constituents` is per-instance with + last-to-leave teardown. Once the last instance calls it, the shared + per-suite buffers and the constituent object array are deallocated + automatically. + +### Built-in constituents vs scheme-registered constituents + +`ccpp_register_constituents` takes one explicit argument: an array of +`ccpp_constituent_properties_t` describing the **host's own constituents** +(typically water vapor and any other tracers the dycore carries +intrinsically). The framework then merges those entries with every +suite's per-suite dynamic-constituent buffer (populated during +`ccpp_register` from each register-phase scheme's output). + +Pass an empty (zero-size) array if the host has no built-in constituents +of its own. + +--- + +## 5. Public API reference + +All routines below live in `ccpp_host_constituents` and are also +re-exported from `ccpp_static_api` for convenience. The dummy-argument +name `instance_number` is the **standard name**; the actual emitted +dummy uses the host's local name for it (typically also +`instance_number` or `inst_num`). + +### `ccpp_register_constituents(host_constituents, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `host_constituents` | `type(ccpp_constituent_properties_t), target, intent(in) :: (:)` | Host-owned constituent declarations (water vapor, etc.). May be zero-size. | +| `instance_number` | `integer, intent(in)` | Per-instance index. | +| `errflg` | `integer, intent(out)` | Error flag (0 = success). | +| `errmsg` | `character(len=*), intent(out)` | Error message. | + +**Effect**: +- On the first call across instances, allocates + `ccpp_model_constituents_obj(number_of_instances)`. +- Calls `obj(instance_number)%initialize_table(num_consts)` where + `num_consts = size(host_constituents) + sum(size(_dynamic_constituents))`. +- Iterates `host_constituents` first, then every suite's + `_dynamic_constituents` buffer, calling + `obj(instance_number)%new_field(const_prop, errcode=errflg, errmsg=errmsg)` + for each entry. +- Calls `obj(instance_number)%lock_table(...)`. + +**Preconditions**: every `_register` call (across all suites) for +this instance has already happened (so the per-suite buffers are +populated). + +### `ccpp_initialize_constituents(ncols, num_layers, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `ncols` | `integer, intent(in)` | Horizontal extent for this instance's chunk. | +| `num_layers` | `integer, intent(in)` | Vertical layer count. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +**Effect**: +- Calls `obj(instance_number)%lock_data(ncols, num_layers, ...)` — + allocates `obj(inst)%vars_layer` and `%vars_layer_tend` arrays. +- Registers a singleton pointer with + `ccpp_scheme_utils.ccpp_initialize_constituent_ptr(obj(inst))` so + cam-sima schemes that call `ccpp_constituent_index` see the + constituent table. **First instance wins** — see + [§8 Limitations](#8-limitations-and-gotchas). +- Queries `obj(instance_number)%const_index(index_of_, '', ...)` for + every constituent `` known at code-gen time; populates the + module-level integer `index_of_`. These integers are identical + across instances; the last call to set them wins (benign — the + constituent table is the same per instance). + +**Preconditions**: `ccpp_register_constituents` has been called for this +instance. + +### `ccpp_is_scheme_constituent(var_name, constituent_exists, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `var_name` | `character(len=*), intent(in)` | Standard name to query. | +| `constituent_exists` | `logical, intent(out)` | True iff *var_name* matches one of the constituent std names known to capgen at code-gen time. | +| `errflg` / `errmsg` | `intent(out)` | | + +**No `instance_number`** — the data lookup is against the module-level +`character(len=N), parameter :: ccpp_model_const_stdnames(K)` array +(compile-time constant, identical across instances). + +### `ccpp_number_constituents(num_flds, advected, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `num_flds` | `integer, intent(out)` | Constituent count returned. | +| `advected` | `logical, optional, intent(in)` | If `.true.`, count advected only. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%num_constituents(num_flds, advected=advected, ...)`. + +> Even though every `obj(i)` returns the same count (registration is +> identical across instances), `instance_number` is part of the +> signature so the caller can guarantee they're querying an +> already-locked instance. Useful for hosts that lifecycle one +> instance at a time. + +### `ccpp_gather_constituents(const_array, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `const_array` | `real(kind=kind_phys), intent(out) :: (:,:,:)` | Destination buffer for constituent values. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%copy_in(const_array, ...)`. Use this to +pull the per-instance constituent values into a host-side array. + +### `ccpp_update_constituents(const_array, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `const_array` | `real(kind=kind_phys), intent(in) :: (:,:,:)` | Source buffer with updated constituent values. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%copy_out(const_array, ...)`. Use this to +push host-side updates back into the per-instance constituent object. + +### `ccpp_const_get_index(stdname, const_index, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `stdname` | `character(len=*), intent(in)` | Constituent standard name to look up. | +| `const_index` | `integer, intent(out)` | Returned index into the constituent array (or `int_unassigned` on miss). | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%const_index(standard_name=stdname, +index=const_index, ...)`. For constituents whose std names are known +at code-gen time, prefer using the module-level `index_of_` integer +directly (no call needed; it's bound during +`ccpp_initialize_constituents`). + +### `ccpp_constituents_array(instance_number) result(const_ptr)` + +Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → +`obj(instance_number)%field_data_ptr()`. + +### `ccpp_advected_constituents_array(instance_number) result(const_ptr)` + +Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → +`obj(instance_number)%advected_constituents_ptr()`. Subset of the +full constituent array containing only those flagged `advected=.true.`. + +### `ccpp_model_const_properties(instance_number) result(const_ptr)` + +Returns `type(ccpp_constituent_prop_ptr_t), pointer :: const_ptr(:)` → +`obj(instance_number)%constituent_props_ptr()`. + +### `ccpp_deallocate_dynamic_constituents(instance_number)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `instance_number` | `integer, intent(in)` | | + +**Per-instance reset + last-to-leave teardown**: +1. `obj(instance_number)%reset()` — unlocks the table for this instance. +2. Iterates every `obj(i)` and checks `%const_props_locked()`. If any + instance is still locked, the routine returns. +3. If **every** instance has been reset (none still locked), the routine + tears down the shared state: + - Deallocates every `_dynamic_constituents` buffer. + - Deallocates `ccpp_model_constituents_obj(:)`. + - Resets every `index_of_` integer to 0. + +The host should call this for every instance that successfully called +`ccpp_register_constituents`. + +--- + +## 6. Generated code structure + +When any suite touches constituent state, capgen-ng emits one extra +module per generator run: **`ccpp_host_constituents.F90`**. + +### Module declarations + +```fortran +module ccpp_host_constituents + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: & + ccpp_model_constituents_t, & + ccpp_constituent_properties_t, & + ccpp_constituent_prop_ptr_t + + implicit none + private + + ! ----- public state ---------------------------------------------------- + public :: ccpp_model_constituents_obj + public :: index_of_ ! one per known constituent std name + public :: index_of_ + public :: ccpp_model_const_stdnames ! parameter array + + ! ----- public routines (also re-exported from ccpp_static_api) -------- + public :: ccpp_register_constituents + public :: ccpp_initialize_constituents + public :: ccpp_is_scheme_constituent + public :: ccpp_number_constituents + public :: ccpp_gather_constituents + public :: ccpp_update_constituents + public :: ccpp_const_get_index + public :: ccpp_constituents_array + public :: ccpp_advected_constituents_array + public :: ccpp_model_const_properties + public :: ccpp_deallocate_dynamic_constituents + public :: _dynamic_constituents ! one per suite with register-phase producers + public :: _dynamic_constituents + + ! ----- module-level state --------------------------------------------- + type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) + type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) + type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) + integer :: index_of_ = 0 + integer :: index_of_ = 0 + character(len=N), parameter :: ccpp_model_const_stdnames(K) = (/ & + ' ', & + ' ' /) + +contains + ! ... routines as documented in §5 ... +end module ccpp_host_constituents +``` + +### Suite-cap responsibilities + +`ccpp__cap.F90` does NOT own constituent state. Its +`_register` routine packs each register-phase scheme's +constituent array into the suite's `_dynamic_constituents` +buffer (USE'd from `ccpp_host_constituents`): + +```fortran +if (.not. allocated(_dynamic_constituents)) then + ! First-instance-only two-pass count + populate. + num_consts = 0 + call _register(scheme_consts=scheme_consts, ...) + num_consts = num_consts + size(scheme_consts, 1) + deallocate(scheme_consts) + ... + allocate(_dynamic_constituents(num_consts)) + num_consts = 0 + call _register(scheme_consts=scheme_consts, ...) + do i = 1, size(scheme_consts, 1) + _dynamic_constituents(num_consts + i) = scheme_consts(i) + end do + num_consts = num_consts + size(scheme_consts, 1) + deallocate(scheme_consts) + ... +end if +``` + +The buffer is **shared across instances** (registration is identical +per instance); only the first instance to call `_register` +populates it. The host-wide merge happens in +`ccpp_register_constituents`. + +### Group-cap call sites + +`ccpp___cap.F90` USE's the constituent symbols it needs +from `ccpp_host_constituents`: + +```fortran +use ccpp_host_constituents, only: ccpp_model_constituents_obj, & + index_of_cloud_liquid_water_mixing_ratio +``` + +… and emits scheme call sites with the per-instance access expression: + +```fortran +call cld_liq_run( & + ... + cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer(lb:ub, 1:nlev, & + index_of_cloud_liquid_water_mixing_ratio), & + tend_cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer_tend(lb:ub, 1:nlev, & + index_of_cloud_liquid_water_mixing_ratio), & + ...) +``` + +The `instance_number` dummy is auto-injected into the group-cap +subroutine signatures by `_extra_dim_ctrl_entries` because the +resolver adds `instance_number` to every constituent arg's +`used_dim_std_names`. + +### Framework F90 dependencies + +`ccpp_host_constituents.F90` and the suite caps depend on these +framework files (listed under `` in `datatable.xml`): + +| File | Why | +|---|---| +| `ccpp_constituent_prop_mod.F90` | Provides `ccpp_model_constituents_t`, `ccpp_constituent_properties_t`, `ccpp_constituent_prop_ptr_t`. | +| `ccpp_hashable.F90` | Transitive dep of `ccpp_constituent_prop_mod`. | +| `ccpp_hash_table.F90` | Transitive dep. | +| `ccpp_scheme_utils.F90` | Provides `ccpp_initialize_constituent_ptr` + `ccpp_constituent_index` (used by cam-sima rrtmgp / mmm schemes). | + +The host's CMake should query `ccpp_datafile.py --utility-files` to +get the absolute paths to these files at the right output location. + +--- + +## 7. Multi-instance design + +In capgen-ng, **per-instance state** means: each "instance" (typically +an OpenMP team / chunk-domain partition) has its own copy of the +state arrays, indexed by `instance_number ∈ [1, number_of_instances]`. + +### What's per-instance + +| State | Storage | +|---|---| +| Constituent values + tendencies | `ccpp_model_constituents_obj(:)` — one DDT per instance | +| Suite-level state machine | `ccpp_suite_state(:)` — declared in each suite cap | +| Suite-owned data | `ccpp_suite_data(:)` — declared in each suite-data module | +| Group-level state machine | `ccpp_group_state(:)` — declared in each group cap | + +### What's shared across instances + +| State | Reason | +|---|---| +| `_dynamic_constituents(:)` per-suite buffers | Registration is identical per instance — populated by the first instance to call `_register` and reused by the rest | +| `index_of_` integers | The constituent table is identical per instance, so the indices are too | +| `ccpp_model_const_stdnames` parameter array | Compile-time constant | + +### Sizing + +`number_of_instances` is the single source of truth. The host declares +it in metadata + Fortran; the generator USE's it from the host module +wherever per-instance allocation happens. See the prior memo +[*Where the total number of instances comes from*](#) for the call +chain (and matching values across all four state arrays: +`ccpp_suite_state`, `ccpp_suite_data`, `ccpp_group_state`, +`ccpp_model_constituents_obj`). + +If the host doesn't declare `number_of_instances`, every per-instance +allocation falls back to `1` and the framework runs single-instance. + +### Two host-side lifecycle patterns + +Both work; pick whichever fits your model. + +**Pattern A: all instances registered first** +``` +do isuite = 1, num_suites + do iinst = 1, num_instances + call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) + end do +end do +do iinst = 1, num_instances + call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) + call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) +end do +do isuite = 1, num_suites + do iinst = 1, num_instances + call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) + end do +end do +! ... time-stepping ... +do iinst = 1, num_instances + call ccpp_deallocate_dynamic_constituents(iinst) + ... +end do +``` + +**Pattern B: serial per instance** +``` +do iinst = 1, num_instances + do isuite = 1, num_suites + call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) + end do + call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) + call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) + do isuite = 1, num_suites + call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) + end do + ! ... per-instance time-stepping ... + call ccpp_deallocate_dynamic_constituents(iinst) +end do +``` + +### Last-to-leave teardown + +`ccpp_deallocate_dynamic_constituents(inst)`: +1. Per-instance `obj(inst)%reset()`. +2. Iterates every `obj(i)`; if any has `%const_props_locked() == .true.`, + returns early. +3. Otherwise (every instance reset): deallocates the shared per-suite + buffers, deallocates `ccpp_model_constituents_obj(:)`, and zeros every + `index_of_` integer. + +This works for both lifecycle patterns above. + +--- + +## 8. Limitations and gotchas + +> **Note (2026-05-12).** Several items in this section are under active +> discussion for an upcoming framework + generator overhaul. See +> `doc/constituents_overhaul.md` for the full architectural review and +> three reform proposals. + +### Framework property ownership (2026-05-12) + +The framework's `ccpp_constituent_properties_t` now carries a private +`framework_owns_me` flag (default `.false.`) with +`is_framework_owned()` getter and `set_framework_owned(value)` setter. +`ccpt_deallocate` only deallocates the underlying prop when the flag +is `.true.`; otherwise it just nullifies its pointer. + +Under capgen-ng's explicit-registration model, all +`ccpp_constituent_properties_t` objects are **target-owned by the +caller** (the host's `host_constituents(:)` array, or the per-suite +`_dynamic_constituents(:)` buffer). We never set the flag, so +the framework correctly skips deallocation. Hosts that hand-allocate +property objects on the heap and want the framework to free them must +call `set_framework_owned(.true.)` before passing to `%new_field`. + +### Missing setters (framework gap) + +The framework lacks setters for `advected`, `diagnostic_name`, +`default_value` (and `mixing_ratio_type`). This means once a +constituent is `%instantiate`d, those properties cannot be changed. +If your host needs to override a scheme-supplied `diagnostic_name` or +`advected` value, you currently cannot — open item in the constituents +overhaul proposal. + +### `ccpp_scheme_utils` singleton + +`ccpp_scheme_utils.ccpp_initialize_constituent_ptr` accepts only one +singleton pointer. It's a framework-level convenience used by cam-sima +schemes that call `ccpp_constituent_index` from `ccpp_scheme_utils`. + +`ccpp_initialize_constituents` calls +`ccpp_initialize_constituent_ptr(obj(inst))` on each instance, but +**only the first call across instances actually sets the pointer** +(the routine is internally guarded by an `initialized` flag). +Subsequent calls are silent no-ops. + +For multi-instance hosts, schemes that use +`ccpp_scheme_utils.ccpp_constituent_index` will see only the first +instance's object — a known limitation inherited from the framework +module's design. Schemes that use the per-instance accessors +(`obj(inst)%const_index(...)` via `ccpp_const_get_index`) are +unaffected. + +### Constituent metadata is identical across instances + +The constituent table (which constituents exist, their properties, the +`index_of_` mapping) is **identical** for every instance. Every +instance's `obj(i)` has the same hash table, populated identically by +its own `ccpp_register_constituents` call. + +This means: + +- `ccpp_number_constituents` returns the same value regardless of + `instance_number`. +- `ccpp_const_get_index` returns the same index regardless of + `instance_number`. +- The `index_of_` integers are populated identically by every + instance's `ccpp_initialize_constituents` (last-write-wins is fine + since every write is the same value). + +`instance_number` is still in the signatures of these routines — see +[§5](#5-public-api-reference) for the rationale. + +### Forbidden patterns recap + +These are rejected at code-gen time (Rule 4 of [§2](#2-the-four-rules-scheme-author-conventions)): + +- `is_constituent + intent=out + non-tendency std_name` — physics phases + may only produce tendencies, not new base constituents. +- `is_constituent + intent=in/inout + tendency_of_*` — tendencies are + write-only. + +### Subscript indices in sliced local_names must be standard names + +If a host metadata variable is declared with a sliced local name +like `q(:,:,index_of_water_vapor_specific_humidity)`, every subscript +token (other than `:` and integer literals) must be a known standard +name. Otherwise the resolver raises a `CCPPError` with a clear +message naming the offending token. + +### Open work items + +- **Unconditional `ccpp_host_constituents.F90` emission.** The + generator currently emits `ccpp_host_constituents.F90` for every + build, even when no scheme or host actually uses the constituent + system (no `ccpp_constituent_properties_t(:)` register-phase arg, + no `is_constituent`-flagged scheme arg, no framework-named + `index_of_` / `ccpp_constituents` / etc. claimed by capgen-ng). + When the host owns its own indices (SCM/GFS) and no scheme exercises + the constituent path, the generated file is dead code that should be + suppressed. Tracked as a deferred item; the `host_dict` precedence + rule above already keeps the file *correct* (empty) in that case. + +--- + +## 9. Differences from original capgen + +| Aspect | Original capgen | capgen-ng | +|---|---|---| +| Constituent object location | Generated `_ccpp_cap.F90` module | `ccpp_host_constituents.F90` (one per generator run) | +| Per-instance | No (single instance) | Yes (`obj(:)` allocatable, sized to `number_of_instances`) | +| Routine name prefix | `_ccpp_register_constituents`, etc. | `ccpp_register_constituents`, etc. (no host prefix; one set per generator run) | +| Routine signatures | No `instance_number` arg | `instance_number` in every per-instance routine | +| Host metadata for constituent obj | None (auto-created by generator) | None (still auto-created by generator) | +| Module-level pointers | `_constituents_array` etc. as functions returning pointers | Same idea, now per-instance via `instance_number` arg | +| Scheme std-name set | `_model_const_stdnames` parameter array | `ccpp_model_const_stdnames` parameter array (no host prefix) | +| Host-facing API surface | `_ccpp_register_constituents`, `_ccpp_initialize_constituents`, `_ccpp_number_constituents`, `_ccpp_is_scheme_constituent`, `_ccpp_gather_constituents`, `_ccpp_update_constituents`, `_ccpp_deallocate_dynamic_constituents`, `_constituents_array`, `_advected_constituents_array`, `_model_const_properties`, `_const_get_index` | Same surface, no `_` prefix | +| Dynamic constituent buffer dimensionality | 1D, per host | 1D, per generator run, **shared across instances** | +| Static suite constituents | Auto-cloned by `ConstituentVarDict.find_variable` and registered via `_constituents_copy_const` accessors | Tracked at code-gen time via `is_constituent` flag; included in the constituent table only if a register-phase scheme produces them (rule 1). Schemes that *consume* a base constituent (rule 2) don't trigger registration — the constituent must be registered by SOMEONE (host or another scheme's register) for the access to work at runtime. | + +### Migration notes for cam-sima hosts + +- **Scheme metadata**: no changes needed. Cam-sima follows rules 2 and + 3 already (audited 2026-05-11). The 4 schemes that register + constituents via `ccpp_constituent_properties_t` (rule 1) work + unchanged. +- **Host metadata**: drop any explicit declaration of + `ccpp_model_constituents_object` if you carried one over from a + previous capgen-ng experiment — the generator owns it now. +- **Host Fortran**: change all `_ccpp_*_constituents` calls to + the unprefixed names (`ccpp_register_constituents` etc.) and add + `instance_number` to every call site. + +--- + +## 10. Worked example + +A minimal cam-sima-style suite with one scheme that consumes a base +constituent and produces its tendency. + +### Scheme metadata (`consume_constituent.meta`) + +``` +[ccpp-table-properties] + name = consume_constituent + type = scheme + +[ccpp-arg-table] + name = consume_constituent_run + type = scheme +[ cldliq ] + standard_name = cloud_liquid_water_mixing_ratio + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in + advected = .true. +[ tend_cldliq ] + standard_name = tendency_of_cloud_liquid_water_mixing_ratio + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = .true. +[ errmsg ] + ... +[ errflg ] + ... +``` + +### Host metadata (`my_host.meta`) + +``` +[ccpp-table-properties] + name = my_host + type = host + +[ccpp-arg-table] + name = my_host + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + units = count + dimensions = () + type = integer +``` + +(Plus a `type=control` table declaring `instance_number`, +`horizontal_loop_begin`, `horizontal_loop_end`, `ccpp_error_message`, +`ccpp_error_code`, etc.) + +### Suite XML (`my_suite.xml`) + +```xml + + + + consume_constituent + + +``` + +### Generated `ccpp_host_constituents.F90` (excerpt) + +```fortran +module ccpp_host_constituents + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: & + ccpp_model_constituents_t, ccpp_constituent_properties_t, & + ccpp_constituent_prop_ptr_t + + implicit none + private + + public :: ccpp_model_constituents_obj + public :: index_of_cloud_liquid_water_mixing_ratio + public :: ccpp_register_constituents, ccpp_initialize_constituents + public :: ccpp_is_scheme_constituent, ccpp_number_constituents + public :: ccpp_gather_constituents, ccpp_update_constituents + public :: ccpp_const_get_index, ccpp_constituents_array + public :: ccpp_advected_constituents_array, ccpp_model_const_properties + public :: ccpp_deallocate_dynamic_constituents + public :: ccpp_model_const_stdnames + + type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) + integer :: index_of_cloud_liquid_water_mixing_ratio = 0 + character(len=31), parameter :: ccpp_model_const_stdnames(1) = (/ & + 'cloud_liquid_water_mixing_ratio' /) + +contains + ! ... full subroutine bodies as in §5 ... +end module ccpp_host_constituents +``` + +### Host code skeleton (single-instance illustration) + +```fortran +subroutine my_host_run() + use ccpp_static_api, only: ccpp_register, ccpp_register_constituents, & + ccpp_initialize_constituents, ccpp_init, & + ccpp_physics_run, ccpp_final, & + ccpp_deallocate_dynamic_constituents + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + type(ccpp_constituent_properties_t), allocatable :: host_consts(:) + integer :: errflg + character(len=512) :: errmsg + integer, parameter :: inst = 1 + + ! 1. Run register phase: populates per-suite dynamic-constituent buffers. + call ccpp_register('my_suite', errmsg, errflg, inst) + + ! 2. Build host's own constituent declarations (water vapor, etc.). + allocate(host_consts(1)) + call host_consts(1)%instantiate( & + std_name='water_vapor_specific_humidity', long_name='water vapor', & + units='kg kg-1', vertical_dim='vertical_layer_dimension', & + advected=.true., errcode=errflg, errmsg=errmsg) + + ! 3. Merge host + suite-side constituents into obj(inst). + call ccpp_register_constituents(host_consts, inst, errflg, errmsg) + + ! 4. Allocate vars_layer + bind cached indices. + call ccpp_initialize_constituents(ncols, nlev, inst, errflg, errmsg) + + ! 5. Framework init phase. + call ccpp_init('my_suite', errmsg, errflg, inst) + + ! 6. Time-stepping (omitted). + call ccpp_physics_run('my_suite', 'phys', col_start, col_end, & + thread_num, nthreads, nphys_threads, & + errflg, errmsg, inst) + + ! 7. Shutdown. + call ccpp_final('my_suite', errmsg, errflg, inst) + call ccpp_deallocate_dynamic_constituents(inst) + deallocate(host_consts) +end subroutine my_host_run +``` + +For multi-instance, wrap each per-instance call in +`do iinst = 1, ninstances ... end do` per the patterns in +[§7](#7-multi-instance-design). diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md new file mode 100644 index 00000000..7d177cae --- /dev/null +++ b/doc/constituents_overhaul.md @@ -0,0 +1,829 @@ +# CCPP Constituents — Architecture Review & Overhaul Discussion + +**Authors:** Dom Heinzeller (lead), Claude (assistant) +**Date drafted:** 2026-05-12 +**Intended audience:** CCPP framework team, CAM-SIMA team +**Status:** Discussion document — no decisions are final. + +--- + +## Executive summary + +CCPP's "constituent" mechanism — how schemes declare and how the framework +manages tracer species like water vapor, cloud liquid, prescribed ozone, +etc. — has grown organically over the last few years. The result works, +but it carries: + +- **A latent framework bug** in `ccpp_constituent_prop_mod` that crashes on + teardown of explicitly-registered (target-passed) constituent property + arrays. Fixed in capgen-ng's framework copy 2026-05-12; needs to land + upstream. +- **Architectural confusion** about which properties are *physics-portable* + (the scheme owns them) versus *host-configuration* (the host owns them). + Today schemes are forced to supply host-specific values (`diag_name` is + the worst offender) at `%instantiate` time. +- **Setter API gaps**: properties that the host wants to override after + scheme-side registration (`advected`, `diagnostic_name`, `default_value`) + have no setters; `is_match` is overly strict about properties hosts + should be free to change. +- **Two registration models** coexist — original capgen's auto-clone of + is_constituent scheme args, and capgen-ng's explicit register-phase + + host-side declaration. Capgen-ng deliberately dropped auto-clone. + +This document is a structured brief for a discussion this week. It does +NOT pre-commit to any decision; it lays out what exists, what's broken, +what we audited, and what proposals are on the table. + +--- + +## Table of contents + +1. [How original capgen handles constituents](#1-how-original-capgen-handles-constituents) +2. [How capgen-ng handles constituents](#2-how-capgen-ng-handles-constituents) +3. [What CAM-SIMA actually needs (audit)](#3-what-cam-sima-actually-needs-audit) +4. [Bugs and design flaws](#4-bugs-and-design-flaws) +5. [Property classification (Class A vs Class B)](#5-property-classification-class-a-vs-class-b) +6. [What to remove, replace, improve](#6-what-to-remove-replace-improve) +7. [Open design questions](#7-open-design-questions) +8. [Three proposals — minimal / clean / deep](#8-three-proposals--minimal--clean--deep) +9. [Appendix: framework setter inventory](#9-appendix-framework-setter-inventory) + +--- + +## 1. How original capgen handles constituents + +### 1.1 Mental model + +Original capgen treats constituents as a **separate scope** between +suite and host: + +``` +group → suite → ConstituentVarDict → host +``` + +A scheme arg flagged `constituent = True` in metadata is matched first +against group/suite/ConstituentVarDict, and only against host as a last +resort. The ConstituentVarDict is a synthetic dictionary whose entries +are auto-created by `find_variable()` when a scheme metadata declares a +constituent dependency. + +### 1.2 Auto-clone of `is_constituent` scheme args + +Every scheme arg with non-default `advected`, `constituent`, or +`molar_mass` is treated as a *registration*. The generator emits, into +the host cap, a routine `_constituents_ccpp_create_constituent_array` +that: + +1. Allocates a `ccpp_constituent_properties_t` pointer per scheme arg. +2. Calls `%instantiate(...)` populating fields **from the scheme + metadata directly** — `std_name`, `long_name`, `diagnostic_name`, + `units`, `default_value`, `advected`, `vertical_dim`, etc. (See + `scripts/constituents.py:565`.) +3. Adds it to the model constituents object via `%new_field`. + +After this auto-clone runs, the host's hand-written +`host_constituents(:)` array is appended, then `%lock_table` finalizes +the hash table. + +### 1.3 The host-cap-owned `ccpp_model_constituents_obj` + +Original capgen generates **one** `ccpp_model_constituents_obj` per +generator invocation, declared module-level in `_ccpp_cap.F90`. +Single global; not per-instance. (CAM-SIMA runs one host per +executable, so single-instance is fine for them.) + +### 1.4 Scheme-side `%instantiate` registration (the other path) + +A scheme may also register constituents via a register-phase argument: + +```fortran +type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) +``` + +The scheme allocates the array, calls `%instantiate` per entry, and +returns it. Original capgen wires this through a per-suite +"dynamic constituents" buffer and merges it during host-cap setup, +alongside the auto-cloned set. + +So original capgen really supports **three** registration sources: + +- Host: hand-written `host_constituents(:)` arg. +- Suite-dynamic: register-phase scheme args. +- Suite-static: auto-cloned from any `is_constituent` consumer. + +All three flow into one `%new_field` table. + +### 1.5 Lifecycle + +- `_ccpp_register_constituents(host_constituents, ...)` runs the + three-source merge. +- `ccpp_initialize_constituent_ptr(const_obj, ...)` (in + `ccpp_scheme_utils`) caches a pointer for the `ccpp_constituent_index` + lookup. +- Phase entry points access `vars_layer` / `vars_layer_tend` via cached + `index_of_` integers. + +### 1.6 What's good about original capgen's approach + +- Schemes declare a constituent dependency once in metadata; no manual + Fortran registration ever needed for "static" tracers. +- Host doesn't have to enumerate every species every scheme wants. +- Works for CAM-SIMA's current scheme catalog. + +### 1.7 What's painful about original capgen's approach + +- The auto-clone path is **invisible** to anyone reading the scheme + Fortran — the registration happens in generated code. +- `ConstituentVarDict` is a synthetic scope, conceptually subtle, and + doesn't generalize cleanly to multi-instance. +- The auto-clone path lifts `diagnostic_name` and `default_value` from + scheme metadata, but those values are often host-specific (see §4.4). +- Three sources of registration with overlap mean two registrations of + the same `std_name` may collide; original capgen relies on + `is_match` (units, advected, thermo_active, water_species) to dedup, + which means schemes accidentally diverge on `advected` and trip the + "incompatible constituent" error. + +--- + +## 2. How capgen-ng handles constituents + +### 2.1 Mental model + +No synthetic scope. Constituents are *one of four* sources for any +scheme arg: + +``` +control | host | suite | constituent +``` + +The resolver classifies each scheme arg into exactly one source. A +`constituent` source means the value will be accessed at runtime as +`ccpp_model_constituents_obj()%vars_layer(:, :, index_of_)` +(or `%vars_layer_tend(...)` for `tendency_of_` outputs). + +### 2.2 The four scheme-author rules + +(See `doc/constituents.md` for full details; this is the summary.) + +1. **Register** — register-phase scheme args of type + `ccpp_constituent_properties_t(:), intent=out, allocatable` declare + new constituents the scheme contributes. +2. **Consume** — physics-phase scheme args with `advected=true` (or + `molar_mass=...` or `constituent=true`) and `intent=in/inout`, with + the constituent's standard name, read the base species. +3. **Produce a tendency** — physics-phase scheme args with + `constituent=true`, `intent=out`, and standard name + `tendency_of_`, write the tendency. +4. **Mismatched combinations are errors** — `intent=out` on a base + constituent, or `intent=in` on a tendency, are codegen-time errors. + +### 2.3 Two registration sources (no auto-clone) + +- **Host**: hand-written `host_constituents(:)`, passed into + `ccpp_register_constituents(host_constituents, instance_number, ...)`. +- **Suite-dynamic**: register-phase scheme args, accumulated into a + per-suite buffer `_dynamic_constituents(:)` by `_register`, + drained into `ccpp_model_constituents_obj(inst)` by + `ccpp_register_constituents`. + +The auto-clone-from-metadata path is deliberately **gone**. If a scheme +declares `advected=true` on an arg but no source registers that +standard name, capgen-ng now emits a runtime check during +`ccpp_initialize_constituents` that errors with the missing name. + +### 2.4 Per-instance state + +Everything is per-instance: + +```fortran +type(ccpp_model_constituents_t), allocatable :: ccpp_model_constituents_obj(:) + ! indexed by instance_number +``` + +All host-facing entry points take `instance_number`: + +``` +ccpp_register_constituents (host_constituents, instance_number, errflg, errmsg) +ccpp_initialize_constituents (ncols, num_layers, instance_number, errflg, errmsg) +ccpp_number_constituents (num_flds, advected, instance_number, errflg, errmsg) +ccpp_gather_constituents (const_array, instance_number, errflg, errmsg) +ccpp_update_constituents (const_array, instance_number, errflg, errmsg) +ccpp_const_get_index (stdname, const_index, instance_number, errflg, errmsg) +ccpp_constituents_array (instance_number) => pointer +ccpp_advected_constituents_array (instance_number) => pointer +ccpp_model_const_properties (instance_number) => pointer +ccpp_deallocate_dynamic_constituents (instance_number, ...) +``` + +`ccpp_is_scheme_constituent(var_name, ...)` and the +`ccpp_model_const_stdnames(:)` parameter array are NOT per-instance — +the standard-name catalog is identical across instances. + +### 2.5 Lifecycle + +``` +ccpp_register(suite_name, instance_number, ...) + └─ _register → packs scheme-dynamic constituents into + _dynamic_constituents (shared buffer, + first instance wins) + ↓ +ccpp_register_constituents(host_constituents, instance_number, ...) + └─ initialize_table(num_host_consts + num_suite_consts) + └─ new_field(host_consts ...) + └─ new_field(_dynamic_constituents ...) + └─ lock_table + ↓ +ccpp_initialize_constituents(ncols, num_layers, instance_number, ...) + └─ lock_data (allocates vars_layer, vars_layer_tend, vars_minvalue) + └─ ccpp_initialize_constituent_ptr (first instance wins; documented limit) + └─ %const_index('') for each enumerated constituent + └─ post-lookup int_unassigned check → clear error message + ↓ +ccpp_init(suite_name, instance_number, ...) + └─ _init → binds module-level pointers + ↓ +... physics phases ... + ↓ +ccpp_final(suite_name, instance_number, ...) + └─ _final → nullifies + last-to-leave deallocates + ↓ +ccpp_deallocate_dynamic_constituents(instance_number, ...) + └─ ccp_model_constituents_obj(inst)%reset + ↓ (in _final, last-to-leave) + deallocate(_dynamic_constituents) +``` + +### 2.6 What's good + +- Explicit. Every constituent registration is visible in someone's + Fortran source. +- Multi-instance from day one. +- The "four rules" are small enough to fit on a slide. +- Resolver-time + codegen-time + runtime checks catch the most common + mistakes. + +### 2.7 What's still painful + +Covered in §4. + +--- + +## 3. What CAM-SIMA actually needs (audit) + +### 3.1 Scheme-side registration usage + +We audited `EXT/cam-sima/atmospheric_physics/schemes/` for use of the +register-phase `ccpp_constituent_properties_t(:)` pattern: + +| Scheme | File | Registers | +|---|---|---| +| RRTMGP constituents | `schemes/rrtmgp/rrtmgp_constituents.meta` | radiative-active species | +| MUSICA chemistry | `schemes/musica/musica_ccpp.meta` | chemical species from MUSICA | +| Prescribed aerosols | `schemes/chemistry/prescribed_aerosols.meta` | aerosol species | +| Prescribed ozone | `schemes/chemistry/prescribed_ozone.meta` | ozone | + +**Total: 4 of 128 schemes** in the atmospheric_physics tree use +scheme-side registration. The other 124 only **consume** constituents +(`advected=true` + `intent=in/inout` in metadata, accessed via the +framework's `vars_layer`). + +This is a small enough number that an alternative "host-only +registration" model is feasible: move those 4 register calls into the +host (or into helper modules the host calls), and the rest of the +catalog only consumes. + +### 3.2 Host-side patterns + +`EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` wraps +the framework setters and exposes: + +- `const_set_thermo_active(const_obj | const_ind, value)` +- `const_set_water_species(const_obj | const_ind, value)` +- `const_set_minimum(...)` + +CAM-SIMA actively **calls these setters at runtime** — schemes don't +supply `thermo_active` at instantiate time; the host configures it +afterwards. This is direct evidence that the "post-instantiation +override" pattern is real and used today, and that the framework's +setter API is load-bearing. + +### 3.3 What CAM-SIMA does **not** do + +- It does not rely on auto-clone for `diag_name`. The scheme-side + register calls in the 4 schemes do supply `diag_name`, but those + values are CAM-SIMA's; a different host would need different ones. +- It does not use `ccpp_constituent_index` (the + `ccpp_scheme_utils`-singleton-based lookup) extensively — most + access goes through the framework's `index_of_` integers. + +### 3.4 What CAM-SIMA's host-cap-owned constituent object looks like + +Because original capgen generates **one** `ccpp_model_constituents_obj` +per generator invocation, and CAM-SIMA uses one generator invocation per +executable, CAM-SIMA effectively runs single-instance today. A +multi-instance CAM-SIMA (sub-columns, ensembles) would expose the +single-global limitation immediately. + +--- + +## 4. Bugs and design flaws + +This section lists known issues across the three layers (framework, +original capgen, capgen-ng). Items marked **(FIXED)** were resolved +2026-05-12 and either are or will be PRs; items marked **(OPEN)** are +intentionally left for this discussion. + +### 4.1 Framework: `ccpt_deallocate` ownership bug (FIXED in capgen-ng tree, needs upstream PR) + +- **Location**: `src/ccpp_constituent_prop_mod.F90`, `ccpt_deallocate` + + `ccpt_set`. +- **Symptom**: `free(): invalid size` crash when + `ccp_model_const_reset` is called on a properly-locked table whose + entries came from pointer-assigned targets (the common pattern + under capgen-ng's explicit registration; also potentially under + original capgen's `host_constituents` path). +- **Root cause**: `ccpt_set` does pointer assignment (`this%prop => + const_ptr`); `ccpt_deallocate` does an unconditional + `deallocate(this%prop)`. The deallocate is correct only when the + caller allocated `const_ptr` on the heap and transferred ownership. +- **Why it didn't surface earlier**: original capgen's advection test + only calls `deallocate` once between a *failing* register and a + *successful* one — at that point `lock_table` has not populated + `const_metadata`, so the broken inner loop is skipped. Capgen-ng + triggers it because its teardown calls `reset` after a successful + lock. +- **Fix landed 2026-05-12**: added `framework_owns_me` private flag on + `ccpp_constituent_properties_t` (default `.false.`) with + `is_framework_owned()` getter and `set_framework_owned(value)` + setter; `ccpt_deallocate` now only deallocates when the flag is set. + Original capgen's auto-clone path in `scripts/constituents.py` + updated to call `set_framework_owned(.true.)` after `allocate`. + Diffs in `src/ccpp_constituent_prop_mod.F90` (and capgen-ng's + parallel copy) + `scripts/constituents.py`. +- **Status**: framework tests pass, capgen-ng tests pass (954). Needs + upstream PR to ccpp-framework + ccpp-capgen. + +### 4.2 Framework: missing setters (OPEN) + +| Property | Optional in `%instantiate`? | Has setter? | `is_match`-checked? | +|---|---|---|---| +| `std_name` | required | — | (lookup key) | +| `long_name` | required | — | no | +| `diag_name` | required | **NO** | no | +| `units` | required | — | **yes** | +| `vertical_dim` | required | — | no | +| `advected` | optional (default .false.) | **NO** | **yes** | +| `default_value` | optional | **NO** | no | +| `min_value` | optional | `set_minimum` | no | +| `molar_mass` | optional | `set_molar_mass` | no | +| `water_species` | optional (default .false.) | `set_water_species` | **yes** | +| `mixing_ratio_type`| optional | **NO** | no | +| `thermo_active` | not in instantiate | `set_thermo_active` | **yes** | +| `const_index` | internal | `set_const_index` | no | + +**Pain points**: + +- `advected` is `is_match`-checked AND has no setter. Once registered, + immutable. If a scheme and the host disagree, you get the + "incompatible constituent" error and you cannot reconcile from + Fortran. +- `diag_name` is required (cannot be omitted at instantiate) AND has + no setter. A scheme must pick a value at registration time; that + value is then frozen. +- `default_value` is silently optional. If omitted, the constituent + array initializes to `huge(real)` and downstream comparisons fail + in surprising ways (we burnt half a day on this 2026-05-12). +- `thermo_active` is the only property in the "post-instantiate-only" + shape: it has a setter but isn't a `%instantiate` arg. The + asymmetry is confusing. + +### 4.3 Framework: `is_match` is too strict (OPEN) + +`is_match` (in `ccp_is_match`) checks `units`, `advected`, +`thermo_active`, `water_species`. Three of those four (`advected`, +`thermo_active`, `water_species`) are properties the host legitimately +overrides post-registration. Two registrations of the same `std_name` +with the same `units` but different `advected` should be a +duplicate-dedup (host wins), not a hard error. + +### 4.4 Framework: `diag_name` portability problem (OPEN) + +Diagnostic output names are host-specific. CAM-SIMA names cloud +liquid mixing ratio `CLDLIQ`; UFS would call it something else. Yet +`%instantiate` makes `diag_name` a *required* arg, forcing schemes to +either: + +- Pick a host-specific value (couples the scheme to a host), or +- Pick a "neutral" default that no host's diagnostic tooling + recognizes. + +The current de-facto pattern in CAM-SIMA scheme code is to pick a +CAM-SIMA-flavoured value and ship it. Any port to UFS would need to +either monkey-patch or fork the scheme. + +A clean fix: +1. Make `diag_name` optional at `%instantiate` (default to empty + string or `std_name`). +2. Add `set_diagnostic_name(value)` setter. +3. Host overrides per-registration after `ccpp_register_constituents`. + +### 4.5 Original capgen: implicit registration (OPEN — observation) + +The auto-clone path is generator magic. Reading scheme metadata +doesn't tell you whether the scheme's args result in registration; you +have to know that `advected=true` triggers it. This is a documentation ++ comprehension problem more than a bug. + +### 4.6 Original capgen: single-instance `ccpp_model_constituents_obj` (OPEN — limitation) + +The host cap declares one global. Multi-instance hosts would need to +either generate one cap per instance or restructure. + +### 4.7 Original capgen: `ConstituentVarDict` complexity (OPEN — observation) + +The synthetic scope between suite and host serves correctness but +adds a code path that most contributors don't read. If we drop it +(capgen-ng has), the variable-matching algorithm shrinks. + +### 4.8 Capgen-ng: `_FRAMEWORK_CONST_DIM_INPUTS` is hand-curated (OPEN) + +`generator/static_api.py` carries a frozenset of standard names +(currently just `number_of_ccpp_constituents`) that introspection +treats specially. Cleaner: a dedicated `used_const_dim_std_names` +field on `ResolvedArg`. Marked REVISIT in the code. + +### 4.9 Capgen-ng: no codegen-time cross-check of scheme registration (OPEN) + +The resolver knows every `is_constituent` arg's standard name (in +`SuiteResolution.constituent_index_names`) but doesn't know what each +scheme's `_register` subroutine actually `%instantiate`s. Today's +guarantee is a runtime check (the `int_unassigned` validation we +added 2026-05-12). Stronger options: + +- (a) New metadata attribute `registers_std_names = a, b, c` on + register-phase tables; codegen errors at generation time. +- (b) Parse scheme `_register` Fortran for `%instantiate(std_name=…)` + calls and cross-check. +- (c) Keep runtime check as authoritative, document the gap. + +### 4.10 Capgen-ng: scheme-metadata `diagnostic_name` for is_constituent args is host-specific (OPEN) + +Same issue as §4.4 but in capgen-ng's metadata layer. Today's +`diagnostic_name` attribute on a scheme metadata arg flows into +`datatable.xml` and is then trusted as "the" diagnostic name. If we +adopt setter-based class-B overrides, this attribute should either be +dropped for constituent args or marked as a default-only hint. + +### 4.11 Capgen-ng: `ccpp_scheme_utils` singleton (OPEN — documented limit) + +`ccpp_initialize_constituent_ptr(const_obj)` stores a single module-level +pointer. Schemes that use `ccpp_constituent_index(stdname)` get that +pointer back. Under multi-instance, only the first instance's +pointer is retained — `ccpp_constituent_index` queries from +within a scheme will always reflect instance 1. CAM-SIMA's 4 +scheme-registering schemes don't rely on this; documented in +`doc/constituents.md` §8. Real fix requires either threading +`instance_number` through `ccpp_constituent_index` (interface +change) or maintaining a per-instance pointer table. + +--- + +## 5. Property classification (Class A vs Class B) + +Proposed in `design_constituents_mutability.md` 2026-05-12. Each +constituent property is conceptually owned by either the scheme +(physics-portable, immutable once instantiated) or the host +(host-configuration, mutable post-instantiation). + +### Class A — scheme-intrinsic (immutable) + +| Property | Why class A | +|---|---| +| `std_name` | Identity. Cannot change. | +| `long_name` | Human-readable name of the *species*. Not host-specific. | +| `units` | Physics correctness. `is_match`-checked. | +| `vertical_dim` | Scheme's structural expectation (interface vs layer). | +| `molar_mass` | Physical constant of the species. | +| `default_value` | (Debatable — see §7) Scheme-appropriate initial value. | + +### Class B — host-configuration (mutable post-instantiation) + +| Property | Why class B | +|---|---| +| `advected` | Whether the host's dycore advects this — host decision. | +| `diag_name` | Host-specific diagnostic system name. | +| `thermo_active` | Host model configuration. | +| `min_value` | Host runtime guardrail. | +| `water_species` | (Borderline — see §7) Physical classification but also host-config. | +| `mixing_ratio_type` | (Borderline — see §7) Depends on dycore convention. | + +### Consequences if adopted + +- `is_match` should check **only class A**. Today it checks 3 of 4 + class-B properties. +- Class B properties need setters. Today + `advected`, `diag_name`, (and `mixing_ratio_type` if it stays + class B) have none. +- `%instantiate` can demote class B from "required + optional" to + "all optional with sane defaults" — `diag_name=''`, + `advected=.false.`, etc. Schemes wouldn't need to set them at all. + +--- + +## 6. What to remove, replace, improve + +### Remove (or stop requiring) + +- **Scheme-metadata `diagnostic_name` on is_constituent args** — host + will override. Keep the attribute valid on non-constituent args + (where it's host tooling documentation, no portability issue). +- **`is_match` checks on advected / water_species / thermo_active** — + class B should not block dedup. +- **The `diag_name` requirement at `%instantiate`** — demote to + optional with `''` default. +- **(Not adopting)** Original capgen's auto-clone path. Already gone + in capgen-ng; this discussion does not propose bringing it back. + Listed for completeness because the option is in memory. + +### Replace + +- **`ConstituentVarDict`** as a concept — capgen-ng already runs + without it. If the framework or future generator code references + it, dropping is fine. +- **Single-global `ccpp_model_constituents_obj`** — capgen-ng's + per-instance array is the replacement. Original capgen could be + retrofitted, but the priority depends on whether multi-instance + enters the original capgen's roadmap. + +### Improve + +- **Add the missing setters**: `set_advected`, `set_diagnostic_name`, + `set_default_value` (if `default_value` becomes class B), + `set_mixing_ratio_type` (if class B). +- **Add a convenience routine** like + `ccpp_get_constituent_props_by_std_name(stdname, instance_number, prop_ptr, errflg, errmsg)` + so hosts can lookup a single constituent's property wrapper by + name without iterating. +- **Codegen-time cross-check** of scheme `_register` calls vs + metadata declarations (preferred: §4.9 option (a) — new + `registers_std_names` attr). +- **Document the lifecycle** clearly. `doc/constituents.md` is + ~960 lines; targeted additions for "register-then-override" + workflow once the new setters land. +- **Capgen-ng-internal cleanup**: replace + `_FRAMEWORK_CONST_DIM_INPUTS` with a `used_const_dim_std_names` + field on `ResolvedArg`. + +--- + +## 7. Open design questions + +These are the calls we need to make in the meeting. + +### Q1. `default_value` — class A or class B? + +- **Class A argument**: the scheme knows what the species + should be initialized to (zero for "starts empty"; small positive + for "starts at background"); the host doesn't typically override. +- **Class B argument**: hosts may want non-default starting values + (chemistry runs with prescribed initial profiles). +- **Today's reality**: framework has no setter, so it's de-facto + class A. The advection-test issue 2026-05-12 surfaced because we + removed the `default_value=0._kind_phys` from cld_liq.F90's + scheme-side register and had no way to put it back; restoring it + in the scheme fixed the test but cements the class-A treatment. +- **Recommendation**: leave class A for now. Revisit when a real + host-override use case appears. + +### Q2. `water_species` — class A or class B? + +- The current `is_match` check on `water_species` treats it as + identity-defining (class A semantics). But the actual *meaning* of + the bit is mostly host bookkeeping ("does the dycore treat this as + water?"). CAM-SIMA has a `set_water_species` wrapper and uses it. +- **Recommendation**: class B, with the caveat that schemes whose + numerics depend on a constituent *being* water should declare that + in metadata as a hard requirement (different mechanism — not the + `is_match` machinery). + +### Q3. `mixing_ratio_type` — class A or class B? + +- The scheme's calculations assume `wrt_dry` or `wrt_moist`; this + feels class A. +- But hosts using different dycores might want to interpret the + same `std_name` differently — feels class B. +- **Recommendation**: class A. The mismatch should manifest as + different `std_name`s (`cloud_liquid_dry_mixing_ratio` vs + `cloud_liquid_wet_mixing_ratio`), not the same name with a runtime + override. Need cam-sima input. + +### Q4. After `is_match` relaxation: what happens on disagreement? + +- If two registrations of the same std_name agree on class A but + disagree on class B (e.g., `advected=.false.` from a scheme, + `advected=.true.` from the host), the second registration's class + B values should win without error. Effectively: the host overrides + the scheme. +- Order matters: today the host appends *after* the dynamic + constituents. Should we reverse so the host appends *first*? + Probably not — the "first registration wins on class A; host + setters override class B" model is conceptually clearer. +- **Recommendation**: silently dedup on matching class A; for class + B disagreements, the *later* registration's class B values are + ignored. Hosts use setters to override after registration + finalizes. + +### Q5. Should `%instantiate` accept class-B args at all? + +- **Option Y**: keep `%instantiate` accepting class B args (with + defaults). Schemes can supply them as hints; hosts can override. + Backward-compatible. +- **Option N**: remove class-B args from `%instantiate`. Schemes + *must* leave them to the host. Breaks the 4 cam-sima + scheme-registering schemes. +- **Recommendation**: option Y. The cost of breaking 4 schemes for + marginal clarity isn't worth it. + +### Q6. `ccpp_scheme_utils` singleton + +- Today: `ccpp_initialize_constituent_ptr(const_obj)` stores one + pointer module-wide. First instance wins. +- Fix options: + - (a) Maintain a per-instance pointer table; threading + `instance_number` through `ccpp_constituent_index`. + - (b) Document the limitation, route around it (no scheme uses + `ccpp_constituent_index` under multi-instance — capgen-ng + already enforces `index_of_` everywhere). +- **Recommendation**: (b). It's a one-line doc note and zero code + change. + +--- + +## 8. Three proposals — minimal / clean / deep + +### Proposal A — bugfix only + +**Scope**: +- Land the `ccpt_deallocate` ownership fix (done 2026-05-12). +- Update `scripts/constituents.py` for original capgen's auto-clone + path to pass `owned=.true.` (done). +- Add the three missing setters (`set_advected`, + `set_diagnostic_name`, `set_default_value`) without changing + semantics. Doesn't touch `is_match` or `%instantiate`. +- Document the gaps in `doc/constituents.md`. + +**Cost**: ~50 lines framework code + tests. No cam-sima changes +required. + +**Benefit**: closes the immediate bug, gives hosts the override +mechanism they need today (specifically for `diag_name`), unblocks +the advection test's deferred-property pattern. + +**Limit**: leaves `is_match` strict — hosts that disagree with a +scheme on `advected` still hit the "incompatible constituent" error. + +### Proposal B — class A/B split + setters + +**Scope** (in addition to A): +- Relax `is_match` to check only class A (`units` and possibly + `mixing_ratio_type`). +- Make all class-B properties optional in `%instantiate` with sane + defaults; deprecate (but keep accepting) class-B kwargs. +- Adopt the recommendation in Q4: silently dedup; host setters + override. +- Update `doc/constituents.md` with the register-then-override + workflow. +- (capgen-ng) Reject `diagnostic_name` on `is_constituent=True` + scheme args at parse time, or downgrade it to a default-only hint. +- (capgen-ng) Replace `_FRAMEWORK_CONST_DIM_INPUTS` with a + `ResolvedArg.used_const_dim_std_names` field. + +**Cost**: ~150 lines framework + ~50 lines capgen-ng + tests. +CAM-SIMA host code can stay as-is (the 4 scheme-side registrations +continue to work with their existing class-B values; they're just +not enforced anymore). Optional: tidy the 4 schemes to pass class-A +only. + +**Benefit**: physics schemes become genuinely portable across +hosts. The class-B override pattern that CAM-SIMA already uses for +`thermo_active` and `water_species` generalizes. + +**Limit**: does not change the registration model (still +explicit-only in capgen-ng, still auto-clone in original capgen). + +### Proposal C — host-only registration + +**Scope** (in addition to B): +- Move the 4 cam-sima scheme-side register calls into a CAM-SIMA + helper module called from `cam_comp.F90`'s initialization. +- Drop register-phase `ccpp_constituent_properties_t(:)` support + from capgen-ng (and possibly original capgen). Schemes only + consume constituents; only the host registers. +- Codegen-time enforcement: any `advected=true` scheme arg whose + std_name is not in the host's enumeration → codegen error. +- Eliminates the `_dynamic_constituents` per-suite buffer + entirely. + +**Cost**: ~300 lines code total; requires coordinated PRs across +ccpp-framework, ccpp-capgen, ccpp-capgen-ng, atmospheric_physics, and +CAM-SIMA. The 4 schemes need their `_register` routines deleted (or +made no-ops); the host needs a new helper. + +**Benefit**: one source of truth for what constituents exist +(the host). Removes the auto-clone / scheme-register conceptual +overlap. Simplifies generator and runtime. + +**Limit**: changes the contract for the 4 scheme authors. Risk of +breaking yet-undiscovered downstream users of the scheme-side +registration model. + +### Comparison + +| Aspect | A | B | C | +|---|---|---|---| +| Lines changed | ~50 | ~200 | ~500+ | +| Coordination needed | framework only | framework + capgen-ng | framework + both generators + cam-sima | +| Fixes the crash | yes | yes | yes | +| Fixes `diag_name` portability | yes (host overrides) | yes | yes | +| Relaxes `is_match` | no | yes | yes | +| Removes scheme-side register | no | no | yes | +| Risk to existing CAM-SIMA workflows | none | low | medium | + +### Recommendation + +**Adopt A immediately (mostly done), aim for B over the next 4–6 +weeks, table C until the framework PR for B is in and we have a +clearer signal on whether the scheme-side register pattern is worth +keeping.** + +--- + +## 9. Appendix: framework setter inventory + +(For reference during the meeting. Reproduced from +`design_constituents_mutability.md`.) + +`ccpp_constituent_properties_t` methods (`src/ccpp_constituent_prop_mod.F90`): + +``` +Instantiation + procedure :: instantiate ! takes std_name, long_name, diag_name (REQUIRED), + ! units, vertical_dim, plus optional + ! advected, default_value, min_value, molar_mass, + ! water_species, mixing_ratio_type + procedure :: deallocate + +Getters (subset) + procedure :: standard_name + procedure :: long_name + procedure :: diagnostic_name + procedure :: units + procedure :: vertical_dimension + procedure :: is_advected + procedure :: is_thermo_active + procedure :: is_water_species + procedure :: is_mass_mixing_ratio + procedure :: is_volume_mixing_ratio + procedure :: is_number_concentration + procedure :: is_dry / is_moist / is_wet + procedure :: minimum + procedure :: molar_mass + procedure :: default_value + procedure :: has_default + procedure :: is_framework_owned ! NEW 2026-05-12 + +Setters (changes after instantiate) + procedure :: set_const_index + procedure :: set_thermo_active + procedure :: set_water_species + procedure :: set_minimum + procedure :: set_molar_mass + procedure :: set_framework_owned ! NEW 2026-05-12 + procedure :: set_advected ! GAP + procedure :: set_diagnostic_name ! GAP + procedure :: set_default_value ! GAP (or keep class A) + procedure :: set_mixing_ratio_type ! GAP (if class B) + +Identity / equality + procedure :: equivalent ! full equality + procedure :: is_match ! checks units + (class-B props ← too strict) +``` + +`ccpp_constituent_prop_ptr_t` is the pointer wrapper. Has parallel +setters that delegate to the underlying `ccpp_constituent_properties_t`. + +--- + +## Cross-references + +- `doc/constituents.md` — capgen-ng's user-facing constituents reference. +- `design_constituent_api.md` (memory) — capgen-ng's per-instance option-A design. +- `design_constituents_mutability.md` (memory) — extended design notes incl. class A/B classification. +- `project_implementation_status.md` (memory) — current implementation state and deferred items. +- `scripts/constituents.py` — original capgen's host-cap generator. +- `src/ccpp_constituent_prop_mod.F90` — framework. +- `capgen-ng/generator/host_constituents.py` — capgen-ng's host-side module emitter. +- `capgen-ng/generator/suite_resolver.py` (`_resolve_constituent_arg`) — capgen-ng's resolver routing. +- `EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` — CAM-SIMA's host-side wrappers around framework setters. + diff --git a/doc/migration.md b/doc/migration.md new file mode 100644 index 00000000..ce968aee --- /dev/null +++ b/doc/migration.md @@ -0,0 +1,545 @@ +# Migrating from ccpp-prebuild / ccpp-capgen to capgen-ng + +This document captures the **user-facing differences** a host model author +or scheme author needs to know when moving metadata, suite XML, and host +Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to +**capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and +`doc/redesign_analysis.md` (analysis of the old systems). + +Generated as of 2026-05-13. Current unit-test suite: 1113 passing. + +**Repository layout** (post-2026-05-13 cleanup): tooling lives under +`capgen-ng/` (top-level of this repo). Unit tests live at the top +level in `unit-tests/`; end-to-end tests in `end-to-end-tests/`. Run +the unit suite from the repo root with `python -m pytest unit-tests/`. + +## Table of contents + +1. [Metadata format changes](#1-metadata-format-changes) + 1. [1.8 `horizontal_loop_extent` → `horizontal_dimension`](#18-horizontal_loop_extent--horizontal_dimension) +2. [Suite definition file (SDF) changes](#2-suite-definition-file-sdf-changes) +3. [Host Fortran requirements](#3-host-fortran-requirements) +4. [Generator CLI and build integration](#4-generator-cli-and-build-integration) +5. [Generated cap layout — what's new and what changed](#5-generated-cap-layout--whats-new-and-what-changed) +6. [Framework changes (constituents)](#6-framework-changes-constituents) + 1. [6.3 Host metadata wins over auto-provisioning](#63-host-metadata-wins-over-auto-provisioning-2026-05-12) +7. [Validator (`ccpp_validator.py`)](#7-validator) +8. [Known gaps and deferred items](#8-known-gaps-and-deferred-items) + +--- + +## 1. Metadata format changes + +### 1.1 Table types + +Four `type =` values in `[ccpp-table-properties]`: + +| Type | Contents | +|---------|---------------------------------------------------------| +| `control` | Control variables passed as ``ccpp_physics_*`` args. | +| `host` | Host-model variables imported via `use`. | +| `ddt` | Derived-type definitions. | +| `scheme` | Scheme metadata. | + +The legacy `type = module` (capgen) becomes `type = host`. The legacy +`TYPEDEFS_NEW_METADATA` Python dict (prebuild) is replaced by `type = ddt` +tables. See `doc/redesign_prompt.md` §3.2. + +### 1.2 New table-property attributes + +All optional inside the `[ccpp-table-properties]` block: + +| Attribute | Applies to | Description | +|-----------------------|-----------------------|-------------| +| `module_name` | scheme, host, ddt | Fortran module name; overrides "module name = table name" when they differ. | +| `dependencies` | any | Comma-separated list of file paths to compile. **May appear multiple times** in one block (new this session); single occurrence still accepted. | +| `dependencies_path` | any | Relative base for `dependencies` entries. Single-valued. | +| `source_path` | any | Relative path to the Fortran source. Single-valued. | +| `kind_spec` | any | `:=>spec` (or shorthand). May appear multiple times. | + +Example with multi-line dependencies (real CCPP physics pattern): + +``` +[ccpp-table-properties] + name = GFS_rrtmg_setup + type = scheme + module_name = GFS_rrtmg_setup # optional when names match + dependencies_path = ../../ + dependencies = tools/mpiutil.F90 + dependencies = hooks/machine.F + dependencies = Radiation/RRTMG/radlw_main.F90,Radiation/RRTMG/radsw_main.F90 +``` + +### 1.3 New per-variable attributes + +Inside a `[ var_name ]` section. All optional. + +| Attribute | Type | Default | Notes | +|------------------|------|---------|-------| +| `top_at_one` | bool | `False` | When host and scheme disagree, generator emits a vertical-flip transform with reverse-stride subscript on the host side. Meaningless on variables without a vertical dimension. | +| `constituent` | bool | `False` | Scheme metadata only. Marks the var as a constituent reference. | +| `advected` | bool | `False` | Scheme metadata only. | +| `molar_mass` | float | `0.0` | Scheme metadata only. | +| `diagnostic_name` | str | (defaults to `local_name`) | Host-tooling hint; mutually exclusive with `diagnostic_name_fixed`. | + +### 1.4 Sliced local names with long subscript indices + +Local names with array slices may carry CCPP standard names as subscript +tokens: + +``` +[ dqdt(:,:,index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array) ] + standard_name = ... +``` + +The 63-char Fortran-identifier limit is enforced only on the base +identifier (`dqdt`), not on subscript tokens (which are CCPP standard +names resolved at codegen time and routinely exceed 63 chars). + +### 1.5 Unit strings: bare vs explicit positive exponent + +`m2` and `m+2` (or any `` vs `+` +combo) are normalised internally and treated as equivalent. Pre-existing +unit-conversion entries don't need to be duplicated; either spelling +matches. + +### 1.6 Improved error messages + +- **Duplicate standard name**: error message now lists both colliding + access paths and hints at the "sibling DDT instance" pattern (when + applicable). +- **Subcycle bound unresolved**: error names the std_name and points + at the control/host metadata as the fix. +- **Instance-dim used without `instance_number`**: error explains the + paired-opt-in requirement (see §1.7). + +### 1.7 Optional `instance_number` / `number_of_instances` pair + +These two control variables are now **paired optional**: + +- Declare **both** (`instance_number` in `type=control`, + `number_of_instances` in `type=host`) → multi-instance API. +- Declare **neither** → single-instance API. Public entry points drop + `instance_number`; internal per-instance arrays size to length 1. +- Declare exactly one → hard error from the validator. + +Hosts that don't need multi-instance bookkeeping can drop both declarations. + +### 1.8 `horizontal_loop_extent` → `horizontal_dimension` + +ccpp-prebuild / original ccpp-capgen used `horizontal_loop_extent` as +the horizontal-axis std name in scheme metadata. capgen-ng uses +`horizontal_dimension` uniformly — the run-vs-non-run distinction +isn't expressed in scheme metadata anymore (host passes +`horizontal_loop_begin`/`horizontal_loop_end` as control vars and the +generated cap slices accordingly). + +Migration paths: + +1. **Edit the metadata** (recommended) — search-and-replace + `horizontal_loop_extent` → `horizontal_dimension` in every scheme + `.meta` you maintain. +2. **Use `--legacy-mode`** (transient) — pass `--legacy-mode` to both + `ccpp_capgen_ng.py` and `ccpp_validator.py` and the rename happens + at parse time. A loud warning banner prints at startup so the + rewrite is never invisible. This shim *will be removed*; treat + it as a runway, not a destination. + +--- + +## 2. Suite definition file (SDF) changes + +### 2.1 Schema v2.0 with nested-suite expansion + +Capgen-ng parses v2.0 SDFs and expands `` references +recursively at parse time. See `doc/redesign_prompt.md` §3 and the +`suite_v2_0.xsd` schema. + +### 2.2 `` with CCPP standard-name loop bound + +```xml + + effr_pre + +``` + +The `loop=` attribute accepts: + +- **Integer literal** (`loop="3"`) — emitted verbatim. +- **CCPP standard name** (`loop="num_subcycles_for_effr"`) — resolved + against host/control metadata; supports DDT-component access paths + (e.g. `phys_state%num_subcycles`). +- **Absent / empty** — treated as `loop="1"`. + +The loop-bound standard name is automatically included in the +introspection inputs list (`ccpp_physics_suite_variables` and +`_suite_host_data`). + +### 2.3 Nested `` elements + +```xml + + + effr_calc + + +``` + +Nested subcycles produce nested `do` loops in the generated cap. Loop +counter variables follow the convention: + +- Outermost / single-level: `ccpp_loop_counter`. +- Each deeper level: `ccpp_loop_counter_2`, `ccpp_loop_counter_3`, ... + +Effective iteration count = product of every level's `loop=` value. +`effr_calc` in the example runs 3·2 = 6 times. + +### 2.4 Suite-level `` and `` schemes + +```xml + + my_init_scheme + ... + my_final_scheme + +``` + +- Each element contains a **single** scheme name as text content. + Multiple `` children inside ``/`` is a schema + violation. (Group-shaped lists belong inside ``.) +- The named scheme's `init` / `final` phase metadata is resolved like + any other scheme phase; missing-phase metadata is a generator error. +- The scheme call is emitted inside `_init` / `_final` + with USE for the scheme module + per-arg host modules, and the + standard errflg check. +- Call ordering: + - `_init`: after all group `state_alloc` and + `suite_data_init_fields`, **before** the `CCPP_SUITE_FRAMEWORK_INITIALIZED` + state transition. + - `_final`: before the `CCPP_SUITE_UNREGISTERED` transition. + +**Accepted spellings**: `` and `` only. Legacy spellings +**``** (typo), **``** (correct long form), and +**``** are rejected with a clear error pointing at the +canonical short form. + +To exercise: + +1. Declare a scheme with `init` and/or `final` phases in its metadata + (minimal sig — just `errmsg` + `errflg` — is fine). +2. Reference it in the SDF as shown above. +3. Add the scheme's `.F90` to your build's source list. + +--- + +## 3. Host Fortran requirements + +### 3.1 Required control variables + +Every host's `type=control` table must declare: + +| Standard name | Fortran type | Purpose | +|-----------------------------------|--------------|-----------------------------------| +| `suite_name` | character | Drives suite dispatch | +| `horizontal_loop_begin` | integer | Lower chunk-bound | +| `horizontal_loop_end` | integer | Upper chunk-bound | +| `thread_number` | integer | Current thread | +| `number_of_threads` | integer | Total threads | +| `number_of_physics_threads` | integer | Physics-internal budget | +| `ccpp_error_code` | integer | Error flag | +| `ccpp_error_message` | character | Error message | + +Optional (paired — see §1.7): + +| Standard name | Fortran type | Table type | Purpose | +|-------------------------|--------------|------------|--------------------------------| +| `instance_number` | integer | control | Current instance index | +| `number_of_instances` | integer | host | Total instance count | + +### 3.2 Required entry-point call sequence + +``` +ccpp_register(suite_name, errflg, errmsg, [instance_number]) + └── per scheme that declares a register phase +ccpp_init(suite_name, errflg, errmsg, [instance_number]) + └── per scheme that declares an init phase +ccpp_physics_init(...) + └── physics phase routines per group: + ccpp_physics_init + ccpp_physics_timestep_init + ccpp_physics_run ← run-loop phase + ccpp_physics_timestep_final + ccpp_physics_final +ccpp_final(suite_name, errflg, errmsg, [instance_number]) +``` + +`instance_number` appears in every signature only when the host +declares the `instance_number` / `number_of_instances` pair (§1.7). + +### 3.3 Host module convention + +The Fortran module that exports a host metadata table's variables is +typically named after the table. When that's not the case, use the +`module_name` table-property override (§1.2): + +``` +[ccpp-table-properties] + name = test_host_data + type = host + module_name = mod_test_host_data +``` + +--- + +## 4. Generator CLI and build integration + +### 4.1 `ccpp_capgen_ng.py` invocation + +``` +python ccpp_capgen_ng.py \ + --host-files [,,...] \ + --scheme-files [,,...] \ + --suites [,,...] \ + --host-name \ + --output-root /ccpp \ + [--kind-type =[:]] \ + [--legacy-mode] \ + [--verbose] [--verbose] +``` + +`--kind-type` syntax: `=[:]`. When `:` is +omitted, `` must be an ISO_FORTRAN_ENV constant (REAL32/REAL64/...) +and the module defaults to `iso_fortran_env`. `kind_phys` is +auto-defaulted to `iso_fortran_env:REAL64` when not supplied. + +`--legacy-mode` (transient migration shim, will be removed): silently +rewrites legacy CCPP standard names that ccpp-prebuild / original +ccpp-capgen used to their capgen-ng equivalents at parse time. +Currently translates `horizontal_loop_extent` → `horizontal_dimension`. +Prints a loud warning banner at startup so the rewrite is never +invisible. Available on both `ccpp_capgen_ng.py` and `ccpp_validator.py` +(keep the flag consistent between the two when both are invoked from +CMake). All translation logic is isolated in +`metadata/legacy_compat.py` and tagged with `# legacy-compat:` comments +at every touchpoint, so the shim can be cleanly removed when migration +is complete. + +### 4.2 `ccpp_datafile.py` query CLI + +Generated `datatable.xml` carries: + +- `` — generated outputs (utilities/host_files/suite_files). +- `` — `.meta` and expanded SDF. +- `` — per-scheme call lists. +- `` — host/api/suite/group dictionaries. + +Query via `ccpp_datafile.py -- `. Flags include +`--dependencies`, `--capgen-files`, `--host-files`, `--utility-files`, +`--suite-list`, `--required-variables `, `--input-variables `, +`--output-variables `, `--host-variables`, `--show`. + +### 4.3 CMake helpers + +`cmake/ccpp_capgen.cmake` and `cmake/ccpp_validator.cmake` provide the +`ccpp_capgen(...)` and `ccpp_validator(...)` macros. `ccpp_datafile(...)` +queries datatable.xml at configure time. + +--- + +## 5. Generated cap layout — what's new and what changed + +### 5.1 Output files + +Always generated: + +- `ccpp_kinds.F90` — kind parameters. Listed under ``. +- `ccpp_static_api.F90` — public host-facing entry points + introspection routines. +- `ccpp__cap.F90` — per-suite dispatcher. +- `ccpp___cap.F90` — per-group phase implementations. +- `ccpp__data.F90` — suite-owned interstitial DDT + module-level array. +- `ccpp__types.F90` — pointer-wrapper types for optional args. +- `ccpp_.meta` — inspection artifact; matches the generated cap. +- `datatable.xml` — build-system + host-introspection metadata. + +When any scheme registers constituents: + +- `ccpp_host_constituents.F90` — owns `ccpp_model_constituents_obj(:)` + and the host-facing constituent API. + +### 5.2 Per-suite data: TARGET on the instance array + +`ccpp_suite_data(:)` carries the `TARGET` attribute: + +```fortran +type(ccpp__data_t), allocatable, target, public :: ccpp_suite_data(:) +``` + +This makes every `ccpp_suite_data(i)%component(...)` subobject a valid +pointer-assignment target — needed for transformation temps and +optional-arg pointer wrappers. + +### 5.3 Variable transformations + +The generator emits three kinds of transform on a per-arg basis: + +| Transform | Trigger | +|------------------|--------------------------------------------------| +| Unit conversion | `host.units != scheme.units` with a registered conversion entry. | +| Kind conversion | `host.kind != scheme.kind` (different strings). | +| Vertical flip | `host.top_at_one != scheme.top_at_one` on a var with a vertical dim. | + +These compose. A scheme arg that needs unit + flip emits a single +combined assignment through a transformation temp: + +```fortran +temp_l = 1.0E-3_kind_phys*host_var(lb:ub, nlev:1:-1) ! unit conversion: kind_phys to kind_phys; vertical flip (top_at_one mismatch) +call scheme_run(temp=temp_l, ...) +host_var(lb:ub, nlev:1:-1) = 1.0E+3_kind_phys*temp_l ! ... reverse ... +``` + +Identity unit conversions (registered for dimensionally-equivalent +spellings like `J kg-1 ↔ m2 s-2`, formula `'{var}'`) are not labelled +"unit conversion" in the comment. + +### 5.4 Subcycle emission + +```fortran +integer :: ccpp_loop_counter +integer :: ccpp_loop_counter_2 +... +do ccpp_loop_counter = 1, phys_state%num_subcycles ! outer + call scheme_pre(...) + do ccpp_loop_counter_2 = 1, 2 ! inner + call scheme_calc(...) + end do +end do +``` + +### 5.5 State machine + +Per-instance integer state arrays: + +- `ccpp_suite_state(:)` — suite-level (UNREGISTERED / REGISTERED / + FRAMEWORK_INITIALIZED). +- `ccpp_group_state(:)` — group-level (UNINITIALIZED / INITIALIZED / + IN_TIMESTEP). + +Single-instance hosts get length-1 arrays indexed with literal `1`. +See `doc/redesign_prompt.md` §7. + +--- + +## 6. Framework changes (constituents) + +### 6.1 `ccpp_constituent_prop_mod` ownership flag + +(Framework PR — needs upstream merge.) Adds: + +- `framework_owns_me` private flag on `ccpp_constituent_properties_t`, + default `.false.`. +- `set_framework_owned(value)` setter (call before + `obj%new_field(const_prop, ...)` when transferring ownership). +- `is_framework_owned()` getter. +- `ccpt_deallocate` only frees when the flag is set; otherwise just + nullifies. + +Backward-compatible. Original capgen's auto-clone path in +`scripts/constituents.py` has been updated to call the setter. + +### 6.2 capgen-ng constituent API + +(See `doc/constituents.md` for the full reference.) Highlights: + +- One `ccpp_model_constituents_obj(:)` array per generator invocation, + sized to `number_of_instances`. +- Host-facing API: + - `ccpp_register_constituents(host_constituents, instance_number, ...)` + - `ccpp_initialize_constituents(ncols, num_layers, instance_number, ...)` + - `ccpp_const_get_index(stdname, const_index, instance_number, ...)` + - `ccpp_constituents_array(instance_number) → pointer` + - `ccpp_advected_constituents_array(instance_number) → pointer` + - `ccpp_model_const_properties(instance_number) → pointer` + - `ccpp_number_constituents(num_flds, advected, instance_number, ...)` + - `ccpp_gather_constituents`, `ccpp_update_constituents` + - `ccpp_is_scheme_constituent(var_name, ...)` (not per-instance) +- Scheme-side registration: four rules — register-phase + `ccpp_constituent_properties_t(:)` arg, consume base via + `advected=true intent=in/inout`, produce tendency via + `constituent=true intent=out` + `tendency_of_` std name, mismatches + are codegen errors. + +### 6.3 Host metadata wins over auto-provisioning (2026-05-12) + +If the host declares a framework-named standard name +(`ccpp_constituents` / `ccpp_constituent_tendencies` / +`ccpp_constituent_properties` / `number_of_ccpp_constituents` / +`index_of_`) as a regular host variable, the resolver uses the +host's declaration and skips capgen-ng auto-provisioning. Matters +most for legacy hosts (GFS / SCM) that own their own tracer +indices — e.g. `[ntcw]` with `standard_name = +index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array` +resolves to the host's short local name `ntcw`, not a parallel +module-level integer named after the full standard name (which +would also blow Fortran's 63-char identifier limit). See +`doc/constituents.md` §3. + +Active design review for the next constituents iteration: +`doc/constituents_overhaul.md` (Class A vs Class B property +classification, three reform proposals). + +--- + +## 7. Validator + +`capgen-ng/ccpp_validator.py` — standalone Fortran-vs-metadata checker. +Today validates **scheme** metadata against scheme Fortran files +(subroutine signatures, optional args, paren-aware decl splitting). + +Continuation-line handling covers both free-form (`&` at trailing end +of prior line only) and fixed-form / dual-form (`&` at both ends, with +the leading marker at column 6). Comment-only and blank lines +interleaved between continuation lines are skipped as Fortran 90+ +permits. + +When the signature parser finds a subroutine but extracts zero args +while metadata declares many, the "Argument count mismatch" error +appends a HINT pointing at the parser rather than masquerading as a +real mismatch — common cause is an unsupported signature feature. + +**Known gap**: host-metadata validation is not yet implemented. When +invoked with non-scheme `.meta` files, the validator silently filters +to zero schemes and reports "Validation passed." Slated for revisit +after the e2e test suite settles (`unit_conv` + `variable_transform` +complete). See `project_validator_host_check_deferred.md` (memory). + +--- + +## 8. Known gaps and deferred items + +| Item | Status | +|--------------------------------------------|-----------------------------------------------| +| `ccpp_loop_counter` standard name inside nested subcycles | Maps to OUTERMOST loop var. None of cam-sima uses this; revisit if a scheme needs the innermost value. | +| Validator host-metadata check | Deferred; revisit after e2e tests stabilise. | +| Constituents overhaul (Class A/B + setters) | Discussion doc at `doc/constituents_overhaul.md`. | +| Framework setters: `set_advected`, `set_diagnostic_name`, `set_default_value` | Deferred; depends on constituents-overhaul decision. | +| Codegen-time scheme-registration cross-check | Deferred; would require new `registers_std_names` metadata attr. | +| `_FRAMEWORK_CONST_DIM_INPUTS` cleanup | **Done 2026-05-13**: hand-curated frozenset gone; framework-constituent dim refs ride on a dedicated `used_const_dim_std_names` field on `ResolvedArg`. | +| Suppress `ccpp_host_constituents.F90` when unused | Deferred; currently emitted for every build even when no scheme/host actually exercises the constituent system. Now *correct* (empty) for SCM-style hosts thanks to the host-wins rule, but still dead code. See `design_constituent_host_wins.md`. | +| Python linter / formatter pass | Deferred; pick `ruff` and apply across `capgen-ng/`. | +| Generated Fortran ↔ Codee formatter idempotency | Deferred; emitted `.F90` must round-trip cleanly through the project's Codee Fortran formatter. | +| `fortran_to_metadata` developer utility | Deferred; bootstraps a `.meta` skeleton from an existing `.F90` subroutine. | +| `--legacy-mode` shim removal | Transient; remove `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and every `# legacy-compat:` touchpoint when scheme metadata has migrated. | +| Original capgen auto-clone path | Intentionally dropped in favour of explicit registration; kept in memory as "Option B" fallback. | + +--- + +## Cross-references + +- `doc/redesign_prompt.md` — original design specification (sections + marked "historic" where the implementation has evolved). +- `doc/redesign_analysis.md` — analysis of the legacy ccpp-prebuild + + ccpp-capgen toolchains. +- `doc/constituents.md` — full constituents reference for capgen-ng. +- `doc/constituents_overhaul.md` — architecture review and reform + proposals for the next iteration. + diff --git a/doc/redesign_analysis - original 202060505T2044.md b/doc/redesign_analysis - original 202060505T2044.md new file mode 100644 index 00000000..2248f1c0 --- /dev/null +++ b/doc/redesign_analysis - original 202060505T2044.md @@ -0,0 +1,2489 @@ +# CCPP Framework Code Generator — Technical Analysis for Redesign + +*Analysis date: 2026-05-04. Clarifications added: 2026-05-05.* + +This document is a deep-dive technical analysis of the two existing CCPP Framework code generators — +`ccpp-prebuild` and `ccpp-capgen` — produced as input to a planned complete redesign. +It covers execution flow, data structures, feature sets, build system integration, and +key architectural differences. + +--- + +## Table of Contents + +1. [Background and motivation](#1-background-and-motivation) +2. [ccpp-prebuild — detailed analysis](#2-ccpp-prebuild--detailed-analysis) +3. [ccpp-capgen — detailed analysis](#3-ccpp-capgen--detailed-analysis) +4. [Shared infrastructure](#4-shared-infrastructure) +5. [Feature comparison](#5-feature-comparison) +6. [Build system integration](#6-build-system-integration) +7. [Key architectural differences](#7-key-architectural-differences) +8. [Design considerations for the redesign](#8-design-considerations-for-the-redesign) +9. [Real-world example: CCPP Single Column Model (SCM)](#9-real-world-example-ccpp-single-column-model-scm) +10. [Real-world example: CAM-SIMA (capgen)](#10-real-world-example-cam-sima-capgen) +11. [Real-world example: UFS Weather Model (prebuild)](#11-real-world-example-ufs-weather-model-prebuild) +12. [Real-world example: Navy NEPTUNE (prebuild, restricted)](#12-real-world-example-navy-neptune-prebuild-restricted) +13. [Cross-cutting design decision: how host data enters the cap chain](#13-cross-cutting-design-decision-how-host-data-enters-the-cap-chain) + +--- + +## 1. Background and motivation + +The CCPP Framework is a code generator that analyzes metadata describing variables required +by physical parameterizations in numerical weather prediction (NWP) models, compares them +against metadata provided by a host model, and generates Fortran interface ("cap") code that +connects the two. + +There are two generations of the generator: + +**`ccpp-prebuild`** (`scripts/ccpp_prebuild.py`): +- Simple, mostly procedural Python +- Used in: NOAA UFS Weather Model, Navy NEPTUNE, CCPP-SCM +- Extremely reliable in research, development, and operations +- Fewer capabilities; simpler design + +**`ccpp-capgen`** (`scripts/ccpp_capgen.py`): +- Highly complex, object-oriented Python taken to the extreme +- Used in: NCAR CAM-SIMA (still mostly a research/development model) +- Many advanced features designed but never implemented (funding/priority gaps) +- Notoriously difficult to develop; no remaining team member fully understands it + +**The original plan** was to update `ccpp-capgen` with missing features from `ccpp-prebuild` +and transition all models to it. **This plan has been abandoned** in favor of a complete +redesign that draws the best lessons from both generations. + +The immediate trigger for abandoning capgen was the failure — after considerable effort by +three developers — to make capgen pass DDT arguments to group caps the way prebuild does. +This is the root cause of capgen's severe performance problem (seconds for prebuild, +10+ minutes for capgen on the same suite set) and of its broken handling of optional +variables under Fortran compiler debugging flags. + +--- + +## 2. ccpp-prebuild — detailed analysis + +### 2.1 Command-line arguments and configuration + +Entry point: `scripts/ccpp_prebuild.py`, `main()`. + +Arguments parsed by `argparse`: + +| Argument | Required | Purpose | +|---|---|---| +| `--config` | yes | Path to host-model Python config module | +| `--suites` | no | Comma-separated suite names (without `.xml`) | +| `--builddir` | no | Override build directory from config | +| `--namespace` | no | Appended to static API module name | +| `--debug` | no | Insert Fortran array-size checks in generated caps | +| `--clean` | no | Remove generated files and exit | +| `--verbose` | no | Set logging to DEBUG | + +The `--config` file is a plain Python module imported dynamically via `importlib`. +Key variables it must define: + +| Config variable | Purpose | +|---|---| +| `VARIABLE_DEFINITION_FILES` | List of host-model Fortran sources with metadata hooks | +| `SCHEME_FILES` | List of physics scheme Fortran sources | +| `CAPS_DIR` | Output directory for generated cap `.F90` files | +| `SUITES_DIR` | Directory containing suite definition XML files | +| `STATIC_API_DIR` | Output directory for `ccpp_static_api.F90` | +| `TYPEDEFS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for typedef build snippets | +| `SCHEMES_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for scheme build snippets | +| `CAPS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for cap build snippets | +| `HTML_VARTABLE_FILE`, `LATEX_VARTABLE_FILE` | Documentation output paths | +| `TYPEDEFS_NEW_METADATA` | Optional: dict enabling DDT member name translation bridge | + +The config file can contain arbitrary Python expressions — computed file lists, +conditional logic, environment-variable lookups — making it very flexible. + +### 2.2 Step-by-step execution pipeline + +``` +1. Import config module dynamically via importlib + +2. gather_variable_definitions() + for each file in VARIABLE_DEFINITION_FILES: + parse_variable_tables(file) [metadata_parser.py] + → metadata_define: OrderedDict[standard_name → [mkcap.Var]] + +3. collect_physics_subroutines() + for each file in SCHEME_FILES: + parse_scheme_tables(file) [metadata_parser.py] + → metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] + → arguments_request: OrderedDict[scheme → OrderedDict[subroutine → [std_names]]] + → dependencies_request: OrderedDict[scheme → [abs_paths]] + → schemes_in_files: OrderedDict[scheme → abs_path] + +4. compare_metadata() [batch matching] + for each std_name in metadata_request: + check exists in metadata_define + check type/kind/rank compatibility + register unit conversions in var.actions + copy local_name as var.target + → metadata: OrderedDict[std_name → [Var]] (targets and actions set) + +5. check_optional_arguments() [warnings only] + +6. For each requested suite XML: + Suite.parse(xml) [mkstatic.py] → Suite + Group objects + Group.write() → ccpp___cap.F90 + Suite.write() → ccpp__cap.F90 + +7. API.write() [mkstatic.py] + → ccpp_static_api[_].F90 + +8. Write build-system snippets [mkcap.py writers] + → CCPP_CAPS.cmake/mk/sh + → CCPP_SCHEMES.cmake/mk/sh + → CCPP_TYPEDEFS.cmake/mk/sh + → CCPP_API.cmake/sh + +9. mkdoc.metadata_to_html() → HTML variable table + mkdoc.metadata_to_latex() → LaTeX variable table +``` + +### 2.3 Data structures — the "flat dict" model + +Everything in prebuild lives in flat Python `OrderedDict` structures. There is no object +hierarchy; variables are simple Python objects with plain attributes. + +```python +# Top-level data containers +metadata_define: OrderedDict[standard_name → [mkcap.Var]] # 1 Var per std_name +metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] # N Vars (one per scheme×subroutine) +arguments_request: OrderedDict[scheme_name → OrderedDict[subroutine_name → [std_names]]] +dependencies_request: OrderedDict[scheme_name → [abs_paths]] +schemes_in_files: OrderedDict[scheme_name → abs_path] +``` + +`mkcap.Var` attributes: + +| Attribute | Type | Description | +|---|---|---| +| `standard_name` | str | CF-convention unique identifier | +| `long_name` | str | Human-readable description | +| `units` | str | Physical units | +| `local_name` | str | Fortran local name (may be DDT member reference) | +| `type` | str | Fortran type (real, integer, logical, or DDT name) | +| `kind` | str | Fortran kind parameter | +| `dimensions` | list[str] | Dimension standard names | +| `intent` | str | in / out / inout | +| `active` | str | `'T'`, `'F'`, or expression string | +| `optional` | str | `'T'` or `'F'` | +| `pointer` | bool | Whether Fortran POINTER attribute needed | +| `target` | str | Set during matching: the host model local_name | +| `actions` | dict | `{'in': fn, 'out': fn}` for unit conversions | +| `container` | str | Encoded provenance: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | + +**Performance note on `container` and `target`**: these two attributes act as a lookup +cache computed once during the `compare_metadata()` batch step. The `container` string +encodes where each variable lives in the host model (module and, if applicable, the +DDT member chain). The `target` records the resolved Fortran local name. Both are +computed once and then used directly during Fortran cap generation — no further dictionary +lookups are needed. This is a major contributor to prebuild's speed advantage. + +### 2.4 Metadata parsing and the bridge to capgen + +`metadata_parser.py` is a shared module that acts as a bridge. It detects whether a +metadata section in a Fortran source file uses the old pipe-delimited format (deprecated, +warning emitted) or the new `.meta` format (triggered by `!! \htmlinclude .html` +in the Fortran source comment hook). + +For `.meta` files, `read_new_metadata()` in `metadata_parser.py`: +1. Calls capgen's `metadata_table.parse_metadata_file()` → `[MetadataTable]` +2. Converts each `metavar.Var` to a `mkcap.Var` +3. Normalizes `active` to `'T'`/`'F'`/expression, `optional` to `'T'`/`'F'` + +The `TYPEDEFS_NEW_METADATA` config variable (when provided) triggers an additional +pass via `convert_local_name_from_new_metadata()` which translates flat +standard-name-style local names into DDT member references such as +`Atm(blk_no)%q(:,:,:,graupel_index)`. This is the bridge that makes the newer +`.meta` format work with the older DDT-heavy host model code. + +### 2.5 Variable matching — `compare_metadata()` + +A single batch function processes all matching. For each standard name in `metadata_request`: + +1. Check it exists in `metadata_define` — error if missing +2. Check there is exactly one definition — error if ambiguous +3. Call `var.compatible(other_var)` — checks equality of `standard_name`, `type`, `kind`, and rank +4. Register unit conversions: if units differ, `var.convert_from()` / `var.convert_to()` + stores a conversion function in `var.actions` +5. Check `active` attribute: if host variable is conditionally allocated and scheme variable + is not `optional`, issue a warning (not an error) +6. Copy `local_name` from the define side as `var.target` +7. Build module use list from container strings + +Result: `metadata` dict where each `Var` has `.target` set to the host model local name +and `.actions` populated with any needed unit conversion functions. + +### 2.6 Generated Fortran files + +#### Group cap: `ccpp___cap.F90` + +One module per group. For each CCPP stage (tsinit, init, run, tsfinal, finalize), a subroutine: + +```fortran +module ccpp_suite_A_physics_cap + use scheme_module, only: scheme_run + use host_module_A, only: ddt_A ! DDT, not flat fields + use host_module_B, only: ddt_B + implicit none + contains + + subroutine suite_A_physics_run_cap(ddt_A, ddt_B, im, iaend, ierr, ...) + type(ddt_A_type), intent(inout), target :: ddt_A ! entire DDT passed + type(ddt_B_type), intent(inout), target :: ddt_B + integer, intent(in) :: im, iaend ! loop bounds + integer, intent(out) :: ierr + logical, save :: initialized(200) = .false. + ! optional variable: local pointer, conditionally associated + real(kind_phys), pointer :: opt_var(:) => null() + if (ddt_A%active_flag) then + opt_var => ddt_A%opt_field + end if + ! unit conversion: local variable + real(kind_phys) :: converted_var(im) + converted_var(:) = ddt_B%field(:im) * conversion_factor + ! fixed-index extraction: local pointer for a specific tracer + real(kind_phys), pointer :: q_water_vapor(:,:) => null() + q_water_vapor => ddt_A%q(:,:,ntqv) ! ntqv = water vapor index in tracer array + ! call scheme with loop-bound application and extracted variables at the call site + call scheme_run( & + arg1 = ddt_A%field1(1:im), & ! horizontal loop-bound applied here + arg2 = ddt_A%field2(1:im,:), & ! loop-bound + all levels + qv = q_water_vapor(1:im,:), & ! specific tracer, loop-bound applied + arg3 = converted_var, & ! unit-converted local var + opt_arg = opt_var, & ! optional pointer + ...) + if (ierr /= 0) return + end subroutine +end module +``` + +Key points: +- **DDTs are passed as arguments, not flat fields.** Hundreds of variables arrive as + one or a small number of DDT arguments. This is the fundamental architectural choice + that makes prebuild fast and safe with compiler debugging flags. +- **Two distinct "subsetting" operations happen at the scheme call site:** + 1. *Loop-bound application*: horizontal range `1:im` (or `im` for scalar extents) + applied in the scheme call argument expressions. + 2. *Fixed-index extraction*: a specific element along one dimension is selected, + e.g. `q_water_vapor => ddt%q(:,:,ntqv)` extracts the water vapor tracer from the + full tracer array. A local pointer (or local variable for unit conversions) is + declared just before the scheme call and passed as the scheme argument. The group + cap always receives the full data; these extractions are local to the group cap. +- **Optional variables** are handled by declaring a local `pointer` variable and + conditionally associating it with the DDT field based on the `active` expression. + An unassociated pointer is passed to the scheme if the variable is inactive. This + avoids compiler exceptions when mandatory debugging flags are enabled, because the + unallocated field is never directly referenced — only the already-null pointer is. +- `logical :: initialized(200), save` — per-instance initialization tracking. The + 200 is the maximum number of complete model instances that can coexist in memory + simultaneously (used in ensemble approaches where multiple copies of the full model + state live in memory at once). Each instance has its own initialization flag. +- For the `run` phase, `im` and `iaend` (or similar) carry `horizontal_loop_begin` + and `horizontal_loop_end`, enabling OpenMP thread-level parallelism where each + thread processes a horizontal slice. +- Explicit keyword argument passing in scheme calls. +- Unit conversion: a local variable is declared and populated before the call; the + local variable is then passed to the scheme. +- Error check after each scheme call; returns immediately on error. +- `--debug` flag inserts Fortran array-size assertions. + +#### Suite cap: `ccpp__cap.F90` + +Imports all group cap functions and exposes one function per stage that chains group calls. + +#### Static API: `ccpp_static_api[_].F90` + +A single Fortran module `ccpp_static_api` with one subroutine per stage: + +```fortran +subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) + character(len=*), intent(in) :: suite_name, group_name + select case(trim(suite_name)) + case('suite_A') + select case(trim(group_name)) + case('physics') + call suite_A_physics_run_cap(cdata, ierr) + ... + end select + ... + end select +end subroutine +``` + +This is the **single entry point** the host model calls. The host model passes `suite_name` +and `group_name` at runtime; the static API dispatches to the appropriate cap function. + +### 2.7 Build system snippet files generated + +Six output files (Makefile, CMakefile, shell source) for three variable sets: + +| File | Content | +|---|---| +| `CCPP_CAPS.cmake` | `set(CAPS /abs/path/cap1.F90 /abs/path/cap2.F90 ...)` | +| `CCPP_SCHEMES.cmake` | `set(SCHEMES /abs/path/scheme1.F90 ...)` | +| `CCPP_TYPEDEFS.cmake` | `set(TYPEDEFS module1 module2 ...)` (module names, not paths) | +| `CCPP_API.cmake` | `set(API /abs/path/ccpp_static_api.F90)` | + +All files are written as `.tmp` first and compared against the existing version; they are +replaced only if the content changed, which avoids unnecessary recompilation of downstream +Fortran targets. + +### 2.8 What `mkcap.py`, `mkstatic.py`, and `mkdoc.py` each do + +**`mkcap.py`**: +- Defines the `mkcap.Var` class (prebuild's variable data class) +- Defines six file-writer classes: `CapsMakefile`, `CapsCMakefile`, `CapsSourcefile`, + `SchemesMakefile`, `SchemesCMakefile`, `SchemesSourcefile`, `TypedefsMakefile`, + `TypedefsCMakefile`, `TypedefsSourcefile` +- Each writer has a `write(file_list)` method that produces a formatted include file +- Does NOT generate any Fortran + +**`mkstatic.py`**: +- Defines `Suite`, `Group`, `Subcycle` classes that parse suite definition XML and + generate Fortran caps +- `Suite.parse()`: reads SDF XML via `xml.etree.ElementTree`, builds `Group` and + `Subcycle` objects +- `Suite.write()`: drives cap generation for all groups and the suite-level cap +- `Group.write()`: generates the group cap Fortran — argument list construction, + module `use` statements, unit conversion code, scheme calls, error handling +- Defines `API` class: generates the static API Fortran module (suite_name/group_name + dispatch switch) +- `CCPP_SUITE_VARIABLES` dict: mandatory variables always included (error message, + error code, loop counter, loop extent) +- Helper functions `extract_parents_and_indices_from_local_name()` and + `extract_dimensions_from_local_name()` handle complex DDT member access like + `Atm(blk_no)%q(:,:,:,graupel_index)` — these are critical for DDT-heavy host models + +**`mkdoc.py`**: +- `metadata_to_html()`: produces an HTML table of all host-model provided variables + (standard_name, long_name, units, rank, type, kind, source, local_name) +- `metadata_to_latex()`: produces a LaTeX table combining host-defined and scheme-requested + variables, annotating which schemes use each variable and whether unit conversion is needed +- Informational outputs only; do not affect the build + +--- + +## 3. ccpp-capgen — detailed analysis + +### 3.1 Command-line arguments + +Entry point: `scripts/ccpp_capgen.py`, `_main_func()`. +Arguments parsed via `framework_env.parse_command_line()` into a `CCPPFrameworkEnv` object: + +| Argument | Required | Purpose | +|---|---|---| +| `--host-files` | yes | `.meta` files or `.txt` indirect file lists | +| `--scheme-files` | yes | Same format | +| `--suites` | yes | `.xml` SDF files or `.txt` lists | +| `--output-root` | no | Directory for generated files | +| `--host-name` | no | If given, generates a host cap | +| `--ccpp-datafile` | no | Path for datatable XML (default: `datatable.xml`) | +| `--kind-type` | no (repeatable) | Fortran kind mappings, e.g. `kind_phys=REAL64` | +| `--preproc-directives` | no | Fortran preprocessor macros | +| `--use-error-obj` | no | Use error object instead of scalar error variables | +| `--force-overwrite` | no | Always regenerate output | +| `--clean` | no | Remove files listed in datatable and exit | +| `--verbose` | no (repeatable) | Increase log verbosity | + +`CCPPFrameworkEnv` (defined in `framework_env.py`) consolidates all settings into typed +properties and stores a `kind_dict` mapping CCPP kind names to `[kind_spec, module]` pairs. + +### 3.2 Step-by-step execution pipeline + +``` +1. create_file_list() + expand .txt indirect file lists, validate .meta extensions + +2. register_ddts(scheme_files) + pre-scan all scheme .meta files + register DDT type names via register_fortran_ddt_name() + (so the host parser can recognize them as non-intrinsic types) + +3. parse_host_model_files() + for each host .meta file: + metadata_table.parse_metadata_file() → [MetadataTable] + find_associated_fortran_file() → matching .F90 path + parse_fortran_file() → Fortran declarations (via fortran_tools) + check_fortran_against_metadata() → cross-validation (type, kind, rank, intent) + accumulate MetadataSection headers: DDT, module, host types + +4. HostModel(table_dict, host_name, run_env) + process DDT headers: → DDTLibrary + ddt_dict (VarDictionary) + process module/host headers: → main VarDictionary + __var_locations + add ConstituentVarDict synthetically for ccpp_model_constituents_t + +5. API(sdfs, host_model, scheme_headers, run_env) + for each SDF XML: + Suite construction: + auto-create 5 phase groups: register, initialize, timestep_initial, + timestep_final, finalize + parse elements → Group objects (RUN_PHASE_NAME) + parse / tags → Scheme objects in full-phase groups + Suite.analyze(host_model, scheme_library, ddt_library, run_env): + Group.analyze() → Scheme.analyze(): + for each scheme argument: + VarDictionary.find_variable() [scope chain search] + Var.compatible() [→ VarCompatObj with transformations] + loop dim substitution for _run phase + register constituent if constituent=True + variable promotion: group outputs → suite level if needed by later group + +6. ccpp_api.write(outdir, run_env) + suite cap .F90 per suite + group caps (embedded or separate) + host cap .F90 (if --host-name given) + ccpp_kinds.F90 + +7. generate_ccpp_datatable() → datatable.xml +``` + +### 3.3 Object hierarchy + +``` +API (ccpp_suite.py) + └── Suite (extends VarDictionary) [one per SDF XML] + parent → ConstituentVarDict (extends VarDictionary) + parent → API + ├── Group (suite_objects.py, extends VarDictionary) [one per ] + │ call_list: CallList (extends VarDictionary) + │ ├── Subcycle (suite_objects.py) + │ │ └── Scheme (suite_objects.py, extends SuiteObject) + │ └── Scheme (for full-phase groups: init, register, etc.) + └── (auto groups: register, initialize, timestep_initial, + timestep_final, finalize) + +HostModel (host_model.py, extends VarDictionary) + ├── ddt_lib: DDTLibrary + │ └── {ddt_name → MetadataSection} + ├── ddt_dict: VarDictionary (all DDT field variables, expanded) + └── loop_vars: VarDictionary (run-time dimension variables) + +VarDictionary (metavar.py) + ├── {standard_name → Var} + └── parent_dict → VarDictionary ← scope chain for find_variable() + +Var (metavar.py) + └── __prop_dict: {property_name → validated_value} + +VarDDT (ddt_library.py, extends Var) + └── __field: Var | VarDDT ← recursive DDT traversal chain +``` + +### 3.4 Variable matching — scope-chain and VarCompatObj + +Unlike prebuild's single batch `compare_metadata()`, capgen performs incremental, +scope-aware matching during the suite analysis phase. + +For each scheme argument in `Scheme.analyze()`: +1. `VarDictionary.find_variable(standard_name)` — searches scope chain: + local group dict → suite dict → ConstituentVarDict → host model dict +2. `Var.compatible(other, run_env)` returns a `VarCompatObj` — not a bool. + `VarCompatObj` carries: + - Whether the variables are equivalent (no transformation needed) + - Whether they are compatible with transformations (unit conversion, dimension + substitution, `top_at_one` flip) + - The reason for any incompatibility (for error messages) +3. For `_run` phase: `horizontal_dimension` is automatically substituted with + `horizontal_loop_begin:horizontal_loop_end` +4. For `constituent = True` variables: auto-registered in `ConstituentVarDict`; + allocation/management code is generated +5. Variable promotion: if a Group produces a variable needed by a later Group, it is + promoted to Suite-level scope + +`VarCompatObj` compatibility considers: +- Type equality +- Kind equality (with ISO kind aliases) +- Units compatibility (triggers unit conversion if compatible) +- Dimension substitutability (horizontal loop vs. full dimension, vertical extent) +- `top_at_one` orientation (triggers flip if needed) +- `protected` status (cannot be an output if protected) +- `CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` + from `var_props.py` + +### 3.5 `metavar.Var` properties + +`metavar.Var` stores all properties in a validated `__prop_dict`. Properties: + +**Specification properties** (all metadata contexts): + +| Property | Type | Notes | +|---|---|---| +| `local_name` | str | Valid Fortran identifier | +| `standard_name` | str | CF-convention, lowercase+underscores | +| `long_name` | str | Human-readable description | +| `units` | str | Physical units string | +| `dimensions` | list | Dimension standard names or `()` | +| `type` | str | Intrinsic or registered DDT name | +| `kind` | str | Fortran kind parameter | +| `active` | str | Conditional allocation expression | +| `optional` | bool | Whether scheme can handle missing var | +| `protected` | bool | Cannot be written by schemes | +| `allocatable` | bool | Has ALLOCATABLE attribute | +| `state_variable` | bool | Persists across timesteps | +| `persistence` | str | `timestep` or `run` | +| `default_value` | str | Fortran expression | +| `diagnostic_name` | str | Diagnostic output name | +| `target` | bool | Has TARGET attribute | +| `polymorphic` | bool | CLASS(*) type | +| `top_at_one` | bool | Vertical ordering: top at index 1 | + +**Scheme-only properties**: + +| Property | Type | Notes | +|---|---|---| +| `intent` | str | in / out / inout | + +**Constituent properties**: + +| Property | Type | Notes | +|---|---|---| +| `constituent` | bool | Is a CCPP-managed constituent (tracer) | +| `advected` | bool | Is advected by the dynamical core | +| `molar_mass` | float | Molecular weight (positive) | + +### 3.6 Capgen-only features + +**Fortran cross-validation** (`check_fortran_against_metadata()`): +- Parses the actual `.F90` file alongside the `.meta` file +- Checks that every metadata entry matches the real Fortran declaration: + variable count, local_name, type, kind, intent (for schemes), dimension rank/names +- Catches bugs where metadata was updated but the Fortran source was not (or vice versa) + +**State machine** (`ccpp_state_machine.py`, `state_machine.py`): +- `CCPP_STATE_MACH`: a `StateMachine` instance with 6 transitions +- Valid state sequence: `register → uninitialized → initialized → in_time_step` +- Suite caps include a `character(len=16) :: ccpp_suite_state` variable +- State-checking code at the start of each phase function enforces correct call ordering +- `CCPP_STATE_MACH.function_match()` uses compiled regex to identify which CCPP phase + a subroutine name belongs to + +**Constituent variable support** (`constituents.py`): +- `ConstituentVarDict` (extends `VarDictionary`) manages traceable species (tracers) +- When a scheme declares `constituent = True`, `find_variable()` auto-creates the variable +- Allocation code for the constituent array is auto-generated +- Constants: `CONST_DDT_NAME = "ccpp_model_constituents_t"`, + `CONST_PROP_TYPE = "ccpp_constituent_properties_t"` + +**DDT library** (`ddt_library.py`): +- `VarDDT(Var)`: represents a DDT field variable at any nesting level +- Traversal chain: `VarDDT → VarDDT → ... → Var` (innermost is the actual leaf field) +- `DDTLibrary`: dictionary of DDT `MetadataSection` objects +- `collect_ddt_fields()` expands DDT variables into component fields in `ddt_dict` + +**Host cap generation** (`host_cap.py`): +- Generated only when `--host-name` is given +- Produces `_ccpp_cap.F90` +- Subroutines: `_ccpp_physics_(api_vars)` + that call into suite cap functions + +**`ccpp_kinds.F90`**: +- Simple Fortran module `ccpp_kinds` containing `use` statements that import all kind + parameters specified via `--kind-type` +- Makes kind parameters available to both schemes and caps without circular dependencies + +**Datatable XML** (`ccpp_datafile.py`): +- Produced after generation; lists all generated files, scheme entries, variable properties, + suite configurations +- Queryable by the build system via `ccpp_datafile.py --suite-files` etc. +- Supports `--clean` workflow: reads the file list, removes all generated files, deletes itself +- `DatatableReport` class provides a programmatic query API + +**In-memory database** (`ccpp_database_obj.py`): +- `CCPPDatabaseObj`: wraps `HostModel` and `API` for programmatic access to capgen results +- Returned when `capgen()` is called with `return_db=True` +- Provides `host_model_dict()`, `suite_list()`, `constituent_dictionary(suite)` + +**Variable tracking tool** (`ccpp_track_variables.py`): +- Standalone diagnostic: traces a specific variable through a suite, showing which schemes + use it and with what intent +- Uses prebuild's `import_config` and capgen's `Suite`/`parse_metadata_file` together + +**Fortran-to-metadata tool** (`ccpp_fortran_to_metadata.py`): +- Standalone utility: parses annotated Fortran source files and generates skeleton `.meta` + files — used to bootstrap new scheme metadata + +--- + +## 4. Shared infrastructure + +### 4.1 Module sharing map + +| Module | Used by prebuild | Used by capgen | Notes | +|---|---|---|---| +| `metadata_parser.py` | yes | partial | **Bridge module**: calls capgen's parser, returns mkcap.Var | +| `metadata_table.py` | via bridge | yes (primary) | Native `.meta` format parser | +| `metavar.py` | no | yes | Primary `Var` class, `VarDictionary` | +| `var_props.py` | no | yes | `VariableProperty`, `VarCompatObj`, dimension constants | +| `mkcap.py` | yes | no | `mkcap.Var` class + build-snippet writers | +| `mkstatic.py` | yes | no | Suite/Group/API Fortran generators | +| `mkdoc.py` | yes | no | HTML/LaTeX documentation generators | +| `common.py` | yes | partial | `CCPP_STAGES`, container encoding | +| `framework_env.py` | dummy instance | yes (primary) | `CCPPFrameworkEnv` | +| `file_utils.py` | no | yes | `create_file_list`, `move_modified_files` | +| `code_block.py` | no | yes | Structured Fortran output | +| `ddt_library.py` | no | yes | `DDTLibrary`, `VarDDT` | +| `host_model.py` | no | yes | `HostModel` class | +| `host_cap.py` | no | yes | Host cap generation | +| `ccpp_suite.py` | no | yes | `Suite`, `API` classes | +| `suite_objects.py` | no | yes | `Scheme`, `Group`, `Subcycle`, `CallList` | +| `constituents.py` | no | yes | `ConstituentVarDict` | +| `ccpp_datafile.py` | no | yes | Datatable XML | +| `ccpp_database_obj.py` | no | yes | `CCPPDatabaseObj` | +| `ccpp_state_machine.py` | no | yes | `CCPP_STATE_MACH` | +| `state_machine.py` | no | yes | `StateMachine` base class | +| `ccpp_fortran_to_metadata.py` | no | yes | Fortran→metadata bootstrap tool | +| `ccpp_track_variables.py` | partial | partial | Uses both worlds | + +**The key architectural debt**: `metadata_parser.py` is a prebuild module that internally +calls capgen's `metadata_table.parse_metadata_file()` and converts results to `mkcap.Var` +objects. This creates a one-way dependency (prebuild → capgen's parser infrastructure) +while presenting a prebuild-style API to `ccpp_prebuild.py`. It exists only because +prebuild predates the `.meta` format. + +### 4.2 The `.meta` file format + +The `.meta` format is the native format for capgen and the expected format for all new +scheme development. The Fortran source file contains a comment hook pointing to the `.meta` +file: + +```fortran +!! \section arg_table_scheme_name_run Argument Table +!! \htmlinclude scheme_name_run.html +``` + +The `.meta` file itself uses an INI-style format: + +```ini +[ccpp-table-properties] + name = scheme_name + type = scheme + dependencies_path = ../some/path + dependencies = utility_module.F90, another.F90 + +[ccpp-arg-table] + name = scheme_name_run + type = scheme +[ im ] + standard_name = horizontal_loop_extent + long_name = horizontal loop extent + units = count + type = integer + dimensions = () + intent = in +[ dz ] + standard_name = layer_thickness + long_name = thickness of each model layer + units = m + type = real + kind = kind_phys + dimensions = (horizontal_loop_extent, vertical_layer_dimension) + intent = in +``` + +Multiple `[ccpp-arg-table]` sections are allowed in a scheme file (one per phase: +`_init`, `_run`, `_finalize`, `_timestep_init`, `_timestep_finalize`). +Singleton tables (DDT, module, host) allow only one section. + +### 4.3 Variable property validation (`var_props.py`) + +`VariableProperty` encapsulates a single metadata property with its name, Python type, +optionality, default, valid-value constraints, and a check function. Check functions used: + +| Checker | What it validates | +|---|---| +| `check_local_name` | Valid Fortran identifier | +| `check_cf_standard_name` | Lowercase, underscores, alphanumeric only | +| `check_fortran_type` | Intrinsic type or registered DDT name | +| `check_units` | Valid unit string (normalizes `+` in exponents) | +| `check_dimensions` | Valid dimension specification | +| `check_default_value` | Valid Fortran expression | +| `check_molar_mass` | Positive float (for constituents) | + +`CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` +in `var_props.py` define the recognized dimension forms and the run-time substitution +map (e.g., `horizontal_dimension → horizontal_loop_begin:horizontal_loop_end`). + +--- + +## 5. Feature comparison + +| Feature | prebuild | capgen | Notes | +|---|---|---|---| +| **Input formats** | | | | +| Native `.meta` format | via bridge | yes | | +| Old pipe-delimited format | deprecated warn | not supported | | +| **Parsing and validation** | | | | +| Fortran source cross-validation | no | yes | capgen parses actual .F90 to cross-check | +| Preprocessor directive support | no | yes | `--preproc-directives` | +| **Variable handling** | | | | +| Variable data class | `mkcap.Var` (flat attrs) | `metavar.Var` (validated prop dict) | | +| Scope-chain variable search | no | yes | group→suite→constituent→host | +| Variable promotion group→suite | no | yes | | +| Unit conversion | yes | yes | | +| Optional/active variables | yes (fully) | yes | both: local pointer + conditional ASSOCIATE | +| DDT library (first-class) | no | yes | `VarDDT` recursive chain | +| **Suite and cap generation** | | | | +| Suite definition (SDF XML) | yes | yes | Same XML format | +| Subcycle loops | yes | yes | | +| State machine in generated caps | no | yes | Runtime state enforcement | +| Static API module (dispatch switch) | yes | no | `ccpp_static_api.F90` | +| Host cap generation | no | yes | `_ccpp_cap.F90` | +| `ccpp_kinds.F90` | no | yes | | +| **Constituent/tracer support** | | | | +| Constituent variable management | no | yes | Auto-allocation, `ConstituentVarDict` | +| **Build system output** | | | | +| CMake/Makefile file-list snippets | yes | no | Six snippet files | +| Datatable XML (queryable) | no | yes | `ccpp_datafile.py` | +| Clean via datatable | no | yes | | +| **Documentation** | | | | +| HTML variable table | yes | no (stub, raises error) | `mkdoc.metadata_to_html` | +| LaTeX variable table | yes | no | `mkdoc.metadata_to_latex` | +| **Developer tools** | | | | +| Variable tracking diagnostic | yes | no | `ccpp_track_variables.py` | +| Fortran-to-metadata bootstrap | no | yes | `ccpp_fortran_to_metadata.py` | +| **Runtime API** | | | | +| In-memory database object | no | yes | `CCPPDatabaseObj` | +| **Debug / developer aids** | | | | +| Debug array-size checks in caps | yes (`--debug`) | no | | +| Namespace suffix for API name | yes (`--namespace`) | no | | +| **Configuration** | | | | +| Config mechanism | Python module (flexible) | CLI args only | | + +**Known gaps and corrections:** + +- Capgen's `--generate-docfiles` is declared in the CLI but raises + `CCPPError("not yet supported")` — documentation generation is unimplemented. +- Prebuild handles `TYPEDEFS_NEW_METADATA` for mixed old/new metadata deployments; + capgen has no equivalent because it only accepts the new format. +- Capgen validates Fortran source against metadata; prebuild trusts metadata and never + reads Fortran code. +- Capgen has no `--namespace` equivalent for the generated API module name. +- Capgen's `CCPPDatabaseObj` and datatable XML allow programmatic querying; prebuild + has no equivalent. +- Prebuild's static API pattern (single Fortran module with runtime dispatch) is absent + from capgen, which uses a different host-cap integration model. +- **Capgen cannot pass DDTs to group caps** — it passes everything as flat fields. + Despite considerable effort by multiple developers, this has not been fixed. This is + the primary reason capgen is being abandoned. +- **Capgen does not support multiple model instances in memory** (ensemble approach). + Prebuild's `initialized(200)` array handles this correctly. +- **Capgen does not own or allocate any data.** Wait — this is a prebuild characteristic. + Capgen *does* allocate data for physics-internal variables (variables used only within + the physics, not provided by the host model) at the suite level. Prebuild requires the + host model to provide and own all data, including any physics-internal scratch space. + +--- + +## 6. Build system integration + +### 6.1 How a host model invokes ccpp-prebuild + +Direct call (as in the test suite): +```bash +python ../../scripts/ccpp_prebuild.py \ + --config=ccpp_prebuild_config.py \ + --builddir=build \ + --suites=suite_A,suite_B \ + [--debug] [--namespace mymodel] +``` + +Typical CMake integration: +```cmake +# Run prebuild at configure time +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_prebuild.py + --config=${HOST_CCPP_PREBUILD_CONFIG} + --builddir=${CMAKE_CURRENT_BINARY_DIR} + --suites=${CCPP_SUITES} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + RESULT_VARIABLE PREBUILD_RESULT +) +if(NOT PREBUILD_RESULT EQUAL 0) + message(FATAL_ERROR "ccpp_prebuild.py failed") +endif() + +# Consume the generated snippet files +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) # → ${CAPS} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) # → ${SCHEMES} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) # → ${API} + +add_library(ccpp_physics OBJECT ${CAPS} ${SCHEMES} ${API}) +``` + +### 6.2 How a host model invokes ccpp-capgen + +Direct call: +```bash +python scripts/ccpp_capgen.py \ + --host-files host_data.meta,host_model.meta \ + --scheme-files scheme1.meta,scheme2.meta \ + --suites suite_A.xml,suite_B.xml \ + --output-root ${BUILD_DIR}/ccpp \ + --host-name my_host \ + --kind-type kind_phys=REAL64 \ + --ccpp-datafile ${BUILD_DIR}/ccpp/datatable.xml +``` + +Typical CMake integration: +```cmake +# Run capgen at configure time +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_capgen.py + --host-files ${HOST_META_FILES} + --scheme-files ${SCHEME_META_FILES} + --suites ${SUITE_SDFS} + --output-root ${CMAKE_CURRENT_BINARY_DIR}/ccpp + --host-name ${HOST_MODEL_NAME} + --ccpp-datafile ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + RESULT_VARIABLE CAPGEN_RESULT +) + +# Query the datatable for generated file lists +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py + ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + --suite-files + OUTPUT_VARIABLE SUITE_CAPS OUTPUT_STRIP_TRAILING_WHITESPACE +) +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py + ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + --host-files + OUTPUT_VARIABLE HOST_CAP OUTPUT_STRIP_TRAILING_WHITESPACE +) + +add_library(ccpp_physics OBJECT ${SUITE_CAPS} ${HOST_CAP}) +``` + +### 6.3 Available datatable query flags + +``` +--host-files → generated host cap .F90 files +--suite-files → generated suite cap .F90 files +--utility-files → generated utility .F90 files (e.g. ccpp_kinds.F90) +--ccpp-files → all generated .F90 files +--process-list → physics process types in the suite +--module-list → Fortran module names needed +--dependencies → scheme dependency files +--suite-list → configured suite names +--required-variables → variables required by all suites +--input-variables → input-only variables for a suite +--output-variables → output variables for a suite +--host-variables → variables provided by the host model +``` + +--- + +## 7. Key architectural differences + +### 7.1 Data model + +| Dimension | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| Variable representation | `mkcap.Var` with plain Python attributes | `metavar.Var` with validated `__prop_dict` | +| Variable storage | Two flat `OrderedDict`s | Scope-chain `VarDictionary` tree | +| Container encoding | Encoded string: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | Explicit class hierarchy | +| DDT handling | Encoded as string in `local_name`; helper regexes to extract | First-class `VarDDT` recursive chain | +| Variable matching | One batch `compare_metadata()` call | Incremental during suite analysis | +| Matching result | `bool` + side effects on `.target` / `.actions` | Rich `VarCompatObj` with transformation info | +| **Cap argument style** | **DDTs passed to group caps** | **Flat fields passed to group caps** | +| Subsetting location | At the scheme call site inside the group cap | Done at a higher level, before group cap | +| Data ownership | Host model owns all data including physics-internal | Capgen allocates physics-internal suite-level data | +| Multiple model instances | Yes — `initialized(200)` array, one flag per instance | No | +| Optional variable handling | Local pointer, conditionally associated | Same mechanism, but blocked by flat-field issue | + +### 7.2 Error handling + +| Aspect | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| Style | `(success, result)` tuples + `logging.error()` | `CCPPError` / `ParseInternalError` exceptions | +| Collection | Errors accumulate via `logging`; `main()` checks success | Raised immediately at point of detection | +| Location info | Filename from context; line numbers sometimes | `ParseContext` objects with file + line number | +| User errors vs bugs | Not distinguished | `CCPPError` (user) vs `ParseInternalError` (programmer) | + +### 7.3 Extensibility + +| Aspect | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| New metadata property | Add to `VALID_ITEMS` dict + `mkcap.Var` attribute | Add one `VariableProperty` entry + checker fn | +| New CCPP phase | Update `CCPP_STAGES` + regenerate static API template | Add one transition tuple to `CCPP_STATE_MACH` | +| New compatibility rule | Modify `var.compatible()` in `mkcap.py` | Extend `VarCompatObj` in `var_props.py` | +| New host model | Write a new Python config file | New `.meta` files + CLI invocation | + +### 7.4 Performance + +Prebuild generates caps for multiple suites in seconds. Capgen, on the same suite set +with the same physics, takes more than 10 minutes. Two independent causes: + +**Cause 1 — Repeated scope-chain traversal.** Every variable lookup in capgen traverses +a five-level `VarDictionary` parent chain (group → suite → constituent dict → host model +→ DDT dict) for every scheme argument in every group in every suite. Prebuild's +`compare_metadata()` does one flat dict lookup per standard name, once, and caches the +result in `var.container` and `var.target`. All subsequent use during Fortran generation +reads these cached attributes directly. + +**Cause 2 — Flat-field cap arguments.** This is likely the dominant cost. Capgen resolves +every scheme argument down to its individual flat field, generates a `use` statement and +an explicit argument for each one, and emits them in the generated Fortran. A DDT with +200 fields becomes 200 individual argument declarations, 200 `use` statements, and 200 +argument positions in the scheme call. Prebuild passes the DDT itself — one argument, +one `use` statement — and then subsets at the call site. + +**Consequence for correctness.** Passing flat fields in capgen also breaks optional +variable handling under Fortran compiler debugging flags. When a field inside a DDT is +conditionally allocated (optional), passing it as a flat field requires dereferencing the +DDT to extract the field — which the compiler will flag as an error if debugging is on +and the field happens to be unallocated. Prebuild avoids this entirely by passing the +DDT and using a local pointer at the scheme call site. + +### 7.5 Team comprehension and maintainability + +This is the critical real-world difference. `ccpp-prebuild` is understood by the whole +team because it is procedural Python: you can read `ccpp_prebuild.py` top-to-bottom and +follow what happens. The data structures are flat dicts; the control flow is linear. + +`ccpp-capgen` has a five-level class hierarchy, scope-chain dictionary lookups, +`VarCompatObj` carrying transformation state, `ConstituentVarDict` as a pluggable +scope-chain node, and a `StateMachine` with regex-based dispatch. No remaining team +member fully understands all of it. Development is extremely slow and risky. + +The failed effort to make capgen pass DDTs instead of flat fields is the concrete proof +point: three developers spent considerable time and could not fix it without fully +understanding the interplay between `VarDDT`, `DDTLibrary`, `VarDictionary` scope chains, +and the Fortran writer. This is the proximate reason for the redesign. + +--- + +## 8. Design considerations for the redesign + +The following observations from this analysis should inform the redesign: + +### 8.1 What to keep from prebuild +- Procedural, top-down control flow — easy to read and debug +- Config file as a Python module — extremely flexible without adding CLI arguments +- The static API pattern (`ccpp_static_api.F90` with runtime suite/group dispatch) — + proven, simple integration for the host model +- **DDT arguments in group caps** — pass DDTs, not flat fields; this is the core correctness + and performance requirement +- **Subsetting at the scheme call site** — group caps always receive full data; loop-bound + application and fixed-index extraction happen in the individual scheme call expressions + or via a local variable/pointer declared just before the call +- **Optional variable pattern** — local pointer declared in the group cap, conditionally + associated based on the `active` expression, then passed to the scheme; this is safe + under all compiler debugging modes +- The `initialized(N)` per-instance tracking — handles multiple simultaneous model + instances in memory (ensemble approach); `N` is the max number of instances +- **Framework-owned data needs a simpler design** — capgen's variable promotion and + `ConstituentVarDict` scope-chain approach is too complex; a cleaner mechanism for + framework-allocated physics-internal data is needed (to be designed) +- HTML and LaTeX documentation generation +- The six CMake/Makefile/shell snippet output files — simple and direct (can be revisited) + +### 8.2 What to keep from capgen +- Native `.meta` file parsing (eliminate the `metadata_parser.py` bridge entirely) +- Fortran source cross-validation (`check_fortran_against_metadata()`) — catches real bugs +- Rich compatibility reporting (`VarCompatObj`-style) — better error messages +- `ccpp_kinds.F90` generation — important for portability +- Datatable XML as output accounting (strictly better than six include files) +- `--preproc-directives` support +- Constituent variable support (needed for CAM-SIMA) +- State machine enforcement (optional feature, but architecturally clean) + +### 8.3 What to eliminate +- The `mkcap.Var` / `metavar.Var` duality — one variable class, natively reading `.meta` +- The `metadata_parser.py` bridge module — it exists only because of the old format +- The scope-chain `VarDictionary` hierarchy — replace with flat, explicit lookup: + one host dict, one scheme dict; no parent-chain traversal +- The five-level class inheritance (Suite → VarDictionary → ParseSource → ...) +- `ConstituentVarDict` as a scope-chain node — a simple explicit constituent registry suffices +- Capgen's variable promotion (group → suite level) — this complexity exists only because + capgen allocates physics-internal data; if the host always owns all data, promotion + is unnecessary +- Capgen's flat-field cap generation — DDT arguments must be the foundation + +### 8.4 Framework-owned data — open design question + +Capgen's variable promotion mechanism (promoting a variable from group scope to suite scope +when a later group needs it) and the `ConstituentVarDict` complexity exist because capgen +allocates and manages physics-internal data — variables used only within the physics, +not visible to the host model. This capability is **wanted** in the redesign: the host +model should not have to declare and own scratch variables that are purely internal to the +physics. + +The problem is not the concept but the implementation. Capgen's approach — weaving +framework-allocated variables into the `VarDictionary` scope chain and promoting them +upward — produces the complexity that made capgen unmaintainable. + +**Open question for the redesign:** What is a simpler mechanism for the framework to +allocate, own, and pass physics-internal variables? Candidate approaches (to be evaluated +with real-world examples): + +- A completely separate, flat "framework data" dictionary, distinct from the host variable + lookup, populated during analysis and passed explicitly to the caps as a dedicated + argument (e.g., a framework-managed DDT or allocatable array container). +- A simplified promotion concept: variables are statically promoted to the widest scope + that needs them during the analysis phase, but stored in a simple flat dict rather than + via a scope-chain lookup. +- Constituent variables (tracers) as a special sub-case with their own well-defined + allocation interface, separate from generic physics-internal data. + +This question will be revisited once real-world examples clarify how many and what kind of +physics-internal variables actually need to be managed. + +### 8.5 Critical design decisions for the redesign prompt + +1. **DDT cap arguments are non-negotiable.** Group caps must receive DDTs. The entire + subsetting, optional-variable, and performance story depends on this. + +2. **Data ownership**: host-owns-all (prebuild model) vs. generator-allocates-internals + (capgen model). This single decision determines whether variable promotion and + suite-level allocation are needed. + +3. **Integration pattern**: static API (prebuild style, `suite_name` + `group_name` dispatch) + vs. host cap (capgen style, separate host-side Fortran glue). Models currently using + each pattern depend on it. + +4. **Config mechanism**: Python module (prebuild style, flexible) vs. pure CLI + file lists + (capgen style, scriptable). The Python module config is very powerful for complex models. + +5. **DDT member access parsing**: `extract_parents_and_indices_from_local_name()` and + `extract_dimensions_from_local_name()` in `mkstatic.py` handle expressions like + `Atm(blk_no)%q(:,:,:,graupel_index)`. The redesign needs a clean, explicit design for + parsing and emitting these — not an afterthought regex patch. + +6. **Output accounting**: datatable XML (capgen) is the right answer. The six CMake snippet + files (prebuild) are redundant and harder to extend. + +7. **Multiple model instances**: the redesign must preserve the `initialized(N)` pattern + or an equivalent. The value of `N` may need to be configurable. + +8. **Backward compatibility of generated Fortran interfaces**: real-world model examples + will define exactly which naming conventions, argument orders, and module structures the + host models depend on. + +--- + +## 9. Real-world example: CCPP Single Column Model (SCM) + +*Source:* `EXT/ccpp-scm/` — uses `ccpp-prebuild`. + +The SCM is a horizontally degenerate model (always `im = 1`, no OpenMP threading) but +it compiles the largest set of suites in the CCPP ecosystem, making it the most complete +real-world picture of what prebuild must handle. + +**Scale:** 63 suites, 257 scheme files (137 scheme entries in config, many containing +multiple modules), 300 generated cap files, ~1,200+ host model variables, ~550 optional +(conditionally active) variables. + +--- + +### 9.1 The `TYPEDEFS_NEW_METADATA` bridge — the DDT accessor map + +This is the most important SCM-specific configuration. It maps each DDT type name to the +Fortran expression used to access an instance of that type from the host model's top-level +scope. It is what allows the code generator to convert a `local_name` like `tgrs` (declared +inside `GFS_statein_type`) into the cap argument expression +`physics%Statein%tgrs(...)`. + +```python +TYPEDEFS_NEW_METADATA = { + 'GFS_typedefs': { + 'GFS_diag_type' : 'physics%Diag', + 'GFS_control_type' : 'physics%Model', + 'GFS_cldprop_type' : 'physics%Cldprop', + 'GFS_tbd_type' : 'physics%Tbd', + 'GFS_sfcprop_type' : 'physics%Sfcprop', + 'GFS_coupling_type': 'physics%Coupling', + 'GFS_statein_type' : 'physics%Statein', + 'GFS_radtend_type' : 'physics%Radtend', + 'GFS_grid_type' : 'physics%Grid', + 'GFS_stateout_type': 'physics%Stateout', + 'GFS_typedefs' : '', + }, + 'CCPP_typedefs': { + 'GFS_interstitial_type': 'physics%Interstitial(cdata%thrd_no)', + 'CCPP_typedefs' : '', + }, + 'scm_type_defs': { + 'physics_type': 'physics', + 'scm_type_defs': '', + }, + 'ccpp_types': { + 'ccpp_t' : 'cdata', + 'ccpp_types': '', + 'MPI_Comm': '', + }, + # ... plus 8 more entries for physics-side modules (machine, radsw_param, etc.) +} +``` + +**How it works:** For a variable with `local_name = tgrs` declared in `GFS_statein_type`, +the generator looks up `'GFS_statein_type'` in the map, finds `'physics%Statein'`, and +constructs the target as `physics%Statein%tgrs`. For the thread-indexed interstitial DDT, +`physics%Interstitial(cdata%thrd_no)%` is produced automatically. + +This dictionary is the **entire** mechanism by which the prebuild bridge converts flat +metadata into correct DDT-member accessor expressions. It is a hand-maintained workaround +that the redesigned generator must **eliminate**: all information needed to derive these +accessor expressions is already present in the CCPP metadata, provided the metadata storage +model is designed correctly to capture the DDT hierarchy and instance/thread indexing. + +--- + +### 9.2 Host model DDT structure + +``` +! Module-level variables accessible globally: +physics (type physics_type, from module scm_type_defs) +cdata (type ccpp_t, from module ccpp_types) +one (integer parameter = 1, from module ccpp_types) + +! physics_type contains: +physics%Model → GFS_control_type (control parameters: integers, logicals, 1D arrays) +physics%Statein → GFS_statein_type (input atmospheric state: 2D/3D real arrays) +physics%Stateout → GFS_stateout_type (output tendencies) +physics%Sfcprop → GFS_sfcprop_type (surface properties: 2D real arrays) +physics%Coupling → GFS_coupling_type (coupling fields) +physics%Grid → GFS_grid_type (grid geometry) +physics%Tbd → GFS_tbd_type (to-be-determined / miscellaneous) +physics%Cldprop → GFS_cldprop_type (cloud microphysics properties) +physics%Radtend → GFS_radtend_type (radiation tendencies) +physics%Diag → GFS_diag_type (diagnostic output arrays) +physics%Interstitial(1:thrd_cnt) → GFS_interstitial_type (per-thread scratch space) +``` + +The interstitial DDT is an array indexed by thread number. Even though the SCM is +single-threaded, all caps use `physics%Interstitial(cdata%thrd_no)` (i.e., index 1). +This is the pattern that enables OpenMP parallelism in the full UFS models. + +--- + +### 9.3 The horizontal dimension in the SCM + +The SCM uses a **chunked** horizontal loop even though `im = 1`. The chunk mechanism is: + +```fortran +chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) +chunk_end = physics%Model%chunk_end(cdata%chunk_no) +``` + +All 2D and 3D array slice expressions in caps use this pattern: +```fortran +physics%Statein%tgrs(chunk_begin:chunk_end, one:levs) +``` + +In the SCM, `chunk_begin = chunk_end = 1` always, but the pattern is general enough for +multi-column models. The `one` lower bound (a named integer constant = 1) is a framework +convention used consistently throughout all caps. + +--- + +### 9.4 Four categories of local variables in group caps + +Every group cap generates four categories of local variable declarations before its scheme +calls: + +**Category 1 — Loop bounds and scalars (always present):** +```fortran +integer :: chunk_begin, chunk_end +integer :: levs +chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) +chunk_end = physics%Model%chunk_end(cdata%chunk_no) +levs = physics%Model%levs +``` + +**Category 2 — Fixed-index extractions (tracer indices, surface-level slices):** + +For a tracer `qgrs(:,:,ntqv)`: +```fortran +! No local variable declared — the expression is used inline at the call site: +call scheme_run(qv = physics%Statein%qgrs(chunk_begin:chunk_end, one:levs, physics%Model%ntqv), ...) +``` + +For a surface-level slice `prsi(:,1)`: +```fortran +call scheme_run(prsi_sfc = physics%Statein%prsi(chunk_begin:chunk_end, 1), ...) +``` + +The fixed index may be a literal integer (`1`) or a runtime scalar variable from a DDT +field (`physics%Model%ntqv`). Both are inlined at the call site. + +**Category 3 — Optional variable pointer arrays:** + +One pointer-array type and one pointer-array variable are declared for each optional +variable. They are dimensioned by thread count: +```fortran +type :: real_kind_phys_rank2_ptr_arr_type + real(kind_phys), dimension(:,:), pointer :: p => null() +end type real_kind_phys_rank2_ptr_arr_type +type(real_kind_phys_rank2_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array +``` + +Before each scheme call that uses the variable, the condition is evaluated and the pointer +either associated or left null: +```fortran +if (physics%Model%lndp_type /= 0) then + sfc_wts_1_ptr_array(cdata%thrd_no)%p => & + physics%Coupling%sfc_wts(chunk_begin:chunk_end, one:physics%Model%n_var_lndp) +end if +``` + +Passed to the scheme as a keyword argument: +```fortran +call gfs_surface_generic_pre_run(..., sfc_wts=sfc_wts_1_ptr_array(cdata%thrd_no)%p, ...) +``` + +After the call, the pointer is nullified: +```fortran +if (physics%Model%lndp_type /= 0) then + nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) +end if +``` + +**Category 4 — Unit conversion local variables:** + +Not present in the SCM (GFS uses consistent SI units throughout). When present in other +models, a local array is declared, populated before the call, and passed as the argument: +```fortran +real(kind_phys) :: converted_var(chunk_begin:chunk_end) +converted_var(:) = physics%Statein%source_field(chunk_begin:chunk_end) * conversion_factor +call scheme_run(..., target_arg=converted_var, ...) +``` + +--- + +### 9.5 Array size checks + +Every array argument — mandatory or optional — has a size check immediately before the +scheme call. The check uses `size()` and computes the expected size from dimension variables: + +```fortran +! Mandatory variable — outer condition is always .true. +if (.true.) then + if (size(physics%Statein%tgrs(chunk_begin:chunk_end, one:levs)) /= & + (chunk_end-chunk_begin+1)*(levs-one+1)) then + write(cdata%errmsg, '(a,i8,a,i8)') & + 'Detected size mismatch for variable tgrs: expected ', expected, ' but got ', actual + ierr = 1 + return + end if +end if + +! Optional variable — outer condition mirrors the active= expression +if (physics%Model%lndp_type /= 0) then + if (associated(sfc_wts_1_ptr_array(cdata%thrd_no)%p)) then + if (size(sfc_wts_1_ptr_array(cdata%thrd_no)%p) /= expected_size) then + ...error... + end if + end if +end if +``` + +--- + +### 9.6 The `initialized(200)` array and instance management + +```fortran +logical, dimension(200), save :: initialized = .false. +``` + +`cdata%ccpp_instance` is a 1-based integer assigned to each independent CCPP state object. +In an ensemble, each ensemble member gets a different instance number (1–200). The `init_cap` +sets `initialized(cdata%ccpp_instance) = .true.` at the end of successful init. The +`run_cap` checks `if (.not. initialized(cdata%ccpp_instance))` and aborts with an error +if init was never called for that instance. The `final_cap` resets the flag to `.false.`. + +The value 200 is hardcoded — it is the maximum supported number of simultaneous model +instances. This could be made configurable. + +--- + +### 9.7 Suite and group cap hierarchy + +Three-level cap hierarchy: + +``` +ccpp_static_api.F90 (module ccpp_static_api) + → dispatches by suite_name + optional group_name + → owns physics, cdata, constants via module use + → calls suite-level caps: + +ccpp_scm_gfs_v16_cap.F90 (module ccpp_scm_gfs_v16_cap) + → aggregates arguments from all groups + → calls group caps in order per phase: + +ccpp_scm_gfs_v16_time_vary_cap.F90 (module ccpp_scm_gfs_v16_time_vary_cap) +ccpp_scm_gfs_v16_radiation_cap.F90 (module ccpp_scm_gfs_v16_radiation_cap) +ccpp_scm_gfs_v16_phys_ps_cap.F90 (module ccpp_scm_gfs_v16_phys_ps_cap) +ccpp_scm_gfs_v16_phys_ts_cap.F90 (module ccpp_scm_gfs_v16_phys_ts_cap) +``` + +Each level is a pure Fortran module. Argument passing is explicit keyword-argument style +at every level; no implicit global data (except in the static API, which uses `use`). + +--- + +### 9.8 Static API: module-level variable ownership + +The static API module uses all host-model modules and accesses their variables at module +scope. It does **not** take host data as subroutine arguments — instead it fills the +group cap arguments from its own module-use-associated variables: + +```fortran +module ccpp_static_api + use scm_type_defs, only: physics + use ccpp_types, only: cdata, one + use scm_physical_constants, only: con_g, con_pi, con_t0c, ... + use gfs_typedefs, only: ltp + use ccpp_scm_gfs_v16_cap, only: scm_gfs_v16_run_cap, ... + ... +contains + subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) + ! cdata passed in, others accessed from module scope + select case (to_lower(trim(suite_name))) + case ('scm_gfs_v16') + if (present(group_name)) then + select case (to_lower(trim(group_name))) + case ('phys_ps') + ierr = scm_gfs_v16_phys_ps_run_cap(one=one, physics=physics, cdata=cdata, ...) + ... + end select + else + ierr = scm_gfs_v16_run_cap(one=one, physics=physics, cdata=cdata, ...) + end if + case ('scm_gfs_v17_p8') + ... + end select + end subroutine +end module +``` + +This design means the static API file must be recompiled whenever any host-model module +changes (because it `use`s them), and it must be regenerated whenever suites change. +Its location in the **source tree** (not build tree) is a deliberate SCM design choice: +the file is committed to the repository as a generated artifact. + +--- + +### 9.9 Build system + +Prebuild runs at **cmake configure time** via `execute_process()`, before any compilation +starts. This is unusual but simplifies the cmake dependency graph. + +```cmake +execute_process( + COMMAND ccpp/framework/scripts/ccpp_prebuild.py + --config=ccpp/config/ccpp_prebuild_config.py + --suites=${CCPP_SUITES} + --builddir=${CMAKE_CURRENT_BINARY_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../.. + OUTPUT_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.out + ERROR_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.err +) +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_CAPS.cmake) # → ${CAPS} +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_SCHEMES.cmake) # → ${SCHEMES} +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} +include(scm/src/CCPP_STATIC_API.cmake) # → ${API} +``` + +**Suite selection:** If `CCPP_SUITES` is not set by the user, a helper script +`suite_info.py` selects a compiler-appropriate subset. The full set of 63 suites is +used for production; subsets speed up development builds. + +--- + +### 9.10 Observations relevant to the redesign + +1. **`TYPEDEFS_NEW_METADATA` is a workaround that the redesign must eliminate.** The + DDT accessor information (which type lives at which accessor path) can be fully derived + from the CCPP metadata itself, given a well-designed metadata storage model. The + redesign must derive DDT accessor expressions automatically from the metadata rather + than requiring a separate hand-maintained dictionary. This is one of the primary + motivations for the new metadata storage design. + +2. **Three-level cap hierarchy (group → suite → static API) should be preserved.** + It provides clean separation: group caps are independently testable, suite caps + aggregate phases, the static API is the single host-callable entry point. + +3. **The static API's module-level `use` of host data is model-specific.** In models + where host data is not module-level (e.g., passed as subroutine arguments), the + static API pattern changes. The SCM is the simplest case because `physics` and `cdata` + are global module variables. + +4. **Instance and thread indexing are two orthogonal dimensions of host data access.** + Host model data uses two distinct indexing patterns that must be handled correctly: + + - **Regular state data** (Statein, Stateout, Sfcprop, etc.): dimensioned by instance + number — `physics%Statein(ccpp_instance_number)%array(1:horizontal_dimension, 1:vertical_dimension, ...)`. + In models supporting multiple in-memory model instances (ensemble), the top-level + DDT is an array indexed by `cdata%ccpp_instance`. + + - **Interstitial (per-thread scratch) data**: dimensioned by both instance and thread — + `physics%Interstitial(ccpp_instance_number, ccpp_thread_number)%array(1:horizontal_loop_extent, ...)`. + Critically, the horizontal dimension of interstitial arrays is sized to + `horizontal_loop_extent` (one OpenMP thread's chunk), not `horizontal_dimension` + (the full column count). `max_number_of_threads` instances are allocated per model + instance. Interstitial data can only be used during the **run phase** — this is a + known limitation of ccpp-prebuild that the redesign should address or at minimum + preserve explicitly. + +5. **Optional variable pointer arrays dimensioned by thread count** are the current + solution to thread-safe optional variable handling. This pattern is verbose (one + derived type + one array per optional variable per cap function) but correct. + The redesign could simplify this. + +6. **~550 optional variables in this model.** Optional/conditional variables are not + a corner case — they are a first-class feature. The redesign must handle them + efficiently and correctly. + +7. **Array size checks are debug-only and should not appear in the redesign by default.** + In prebuild they are only generated when the `--debug` flag is passed. The redesigned + generator should not produce them in normal mode — out-of-bounds access is caught at + runtime by compiler flags (e.g., `-fcheck=bounds` with gfortran, `-check bounds` with + ifort). The 12,991-line group cap is partly a consequence of generating these checks + unconditionally in the debug mode artifact examined here. + +8. **No unit conversions appear in GFS/SCM.** Unit conversion infrastructure must be + present in the redesign but the GFS physics package is self-consistent in units. + Unit conversions are more relevant for other host models. + +9. **The `one` constant** (integer parameter = 1) is passed as an explicit argument + everywhere and used as the lower bound in all array slices. This is a framework + convention. The redesign should decide whether this convention is preserved or + whether array lower bounds are handled differently. + +10. **Subcycles produce actual Fortran `do` loops inside the generated group cap.** + The loop from `1` to `cdata%loop_max` is generated directly in the cap function, + not left to the host model: + ```fortran + cdata%loop_max = 2 + do cdata%loop_cnt = 1, cdata%loop_max + call scheme_A_run(...) + if (ierr /= 0) return + call scheme_B_run(...) + if (ierr /= 0) return + end do + ``` + `cdata%loop_max` is set at the start of the subcycle block (from the `loop=` attribute + in the SDF XML) and `cdata%loop_cnt` is the current iteration counter, both visible + to schemes via the `ccpp_t` DDT. + +--- + +## 10. Real-world example: CAM-SIMA (capgen) + +*Source:* `EXT/cam-sima/` — uses `ccpp-capgen`. + +CAM-SIMA is the only model currently using capgen. It is still primarily a research model. +Unlike the SCM it uses a full 3D grid with OpenMP parallelism, but exposes host model +data as flat module variables rather than DDTs in the metadata layer. This example reveals +both what capgen can do and where it fundamentally fails. + +**Scale:** 1 suite (`cam7`), 2 run groups (`physics_before_coupler`, `physics_after_coupler`), +~75 scheme calls, 18 host `.meta` files, 893-line host cap, 2865-line suite cap. + +--- + +### 10.1 Suite structure + +`suite_cam7.xml` has two groups and no subcycles: + +| Group | Schemes (approx.) | Purpose | +|---|---|---| +| `physics_before_coupler` | 52 scheme calls | Cloud fraction, energy checks, dry adiabatic adjustment, Zhang-McFarlane deep convection full cycle, constituent tendency application | +| `physics_after_coupler` | ~20 scheme calls | Tropopause diagnostics, gravity wave drag (7 parameterizations + diagnostics), tendency application, energy consistency | + +CCPP phases in use: register, initialize, timestep_initial, run (per group), timestep_final, finalize. + +--- + +### 10.2 Host model variable structure + +**18 host `.meta` files, all of type `module`.** There are no `host` or `ddt` table types +anywhere. All host variables are flat scalars or arrays in Fortran modules. + +CAM-SIMA does **not** expose its physics DDTs (`phys_state`, `phys_tend`, `cam_in`, etc.) +through metadata. These types exist in `physics_types.F90` but have no `.meta` file. +The generated host cap accesses them directly via `use physics_types, only: phys_state, ...` +and passes individual DDT members as flat keyword arguments: +```fortran +! In cam_ccpp_cap.F90 — direct access to non-metadataized DDT members: +call cam7_physics_before_coupler(..., pint=phys_state%pint, t=phys_state%t, & + dtdt_total=phys_tend%dtdt_total, landfrac=cam_in%landfrac, ...) +``` + +This means capgen has no knowledge of how host data is structured. The host cap is +partly machine-generated and partly depends on manually wiring non-metadataized sources. +**This is a fundamental architectural gap** — changes to `physics_types` are invisible +to the framework. + +**Key host variables by module:** + +| Module | Key variables | +|---|---| +| `physics_grid` | `columns_on_task` (horizontal_dimension), `col_start`, `col_end`, lat, lon, area | +| `vert_coord` | `pver` (vertical_layer_dimension), `pverp` (vertical_interface_dimension) | +| `physconst` | ~35 physical constants, all `protected = True` | +| `cam_constituents` | `num_advected` (count of advected tracers) | +| `spmd_utils` | `mpicom`, `masterproc`, `npes`, `iam` | + +No instance indexing (`physics(1)`) and no thread-indexed DDTs appear — CAM-SIMA uses +a fundamentally different data model from the GFS/SCM stack. + +--- + +### 10.3 The two-cap architecture + +Capgen generates two distinct Fortran files: + +**`cam_ccpp_cap.F90` — the host cap (893 lines)** +- Module `cam_ccpp_cap` +- Imports non-metadataized host variables directly via `use physics_types`, `use physconst`, etc. +- Manages the constituent object (`ccpp_model_constituents_t`) — registration, initialization, gather/scatter, index lookup +- Public subroutines: `cam_ccpp_physics_run`, `cam_ccpp_physics_initialize`, etc. — the entry points the host model calls +- Dispatches to the suite cap, passing ~61–76 flat keyword arguments + +**`ccpp_cam7_cap.F90` — the suite cap (2865 lines)** +- Module `ccpp_cam7_cap` +- No host-specific imports — knows nothing about `physics_types`, `phys_state`, etc. +- All arguments are flat scalars and arrays, fully matched to metadata standard_names +- Contains all scheme calls, suite-level persistent variables, local temporaries, state machine +- The suite cap could in principle be used with any host model that provides the same standard names + +This two-cap split is **architecturally correct**: it separates host-specific binding +from physics-neutral dispatch. The redesign should preserve this separation. + +--- + +### 10.4 The flat-field argument problem — concrete evidence + +The run-phase subroutines expose the core problem with capgen's approach directly: + +```fortran +subroutine cam7_physics_before_coupler(errflg, errmsg, col_start, col_end, pver, dtime, & + gravit, pint, te_ini_dyn, teout, amiroot, iulog, ptend_s, temp, dtdt_total, cpair, & + lagrang, layer_surf, layer_toa, interface_surf, interface_toa, ncnst, piln, pmid, pdel, & + rpdel, qv, carr, cprops, rair, zvir, zi, zm, cp_or_cv_dycore, u, v, pintdry, phis, & + te_cur_phys, te_cur_dyn, tw_cur, latice, latvap, energy_formula_physics, & + energy_formula_dycore, cappa, q_tend, const_tend, qmin, pverp, cpwv, cpliq, rh2o, lat, & + long, pblh, mcon, tpert, dlf, rprd, ql, rliq, landfrac, cpair3, ttend_dp, tmelt, & + top_lev, ke, ke_lnd, cldfrc, domomtran, momcu, momcd, il1g, nstep, & + dudt_total, dvdt_total, fracis, dpdry, ps) +``` + +**61 dummy arguments for one group cap.** `physics_after_coupler` has 76. These are +individual flat arrays and scalars — no DDT in sight. This is exactly the problem that +three developers failed to fix: in the GFS/UFS context, this would be 1,200+ arguments. +The GFS physics stack simply cannot be connected to capgen in its current form. + +In contrast, the prebuild equivalent for the same data would pass `physics` (one DDT argument) +and `cdata` — two arguments covering hundreds of variables. + +--- + +### 10.5 Suite-level persistent variables — the framework-owned data pattern + +The suite cap allocates and owns arrays that persist across group calls within a timestep. +These are allocated in `cam7_initialize` and deallocated in `cam7_finalize`: + +```fortran +! Suite-level persistent (allocated in initialize, freed in finalize): +real(kind_phys), allocatable :: windu_tend(:,:) ! GW drag u-tendency accumulator +real(kind_phys), allocatable :: windv_tend(:,:) ! GW drag v-tendency accumulator +real(kind_phys), allocatable :: scaling_dycore(:,:) ! energy scaling factor +real(kind_phys), allocatable :: tend_te_tnd(:) ! energy tendency accumulator +real(kind_phys), allocatable :: tend_tw_tnd(:) ! water tendency accumulator +real(kind_phys), allocatable :: temp_ini(:,:) ! temperature saved at timestep start +real(kind_phys), allocatable :: z_ini(:,:) ! height saved at timestep start +real(kind_phys), allocatable :: flx_vap(:), flx_cnd(:), flx_ice(:), flx_sen(:) +logical, allocatable :: doconvtran(:) ! per-constituent convection flag +type(coords1d) :: p ! pressure coordinate DDT for GW drag +``` + +These are physics-internal variables — the host model does not know about them, does not +own them, and does not need to. This is the capgen "data ownership" model: the suite cap +is the data owner for variables that only matter within the physics. + +**This pattern is correct and desirable.** The complexity in capgen comes not from the +concept but from how these variables are discovered during analysis (scope-chain promotion) +and passed around (via VarDictionary). The redesign needs a simpler mechanism to achieve +the same result: statically enumerate physics-internal variables during analysis and have +the suite cap own them as named allocatables. + +During the run phase, suite-level persistent arrays are subsetted when passed to schemes: +```fortran +call gw_common_run(..., windu_tend=windu_tend(col_start:col_end, 1:pver), ...) +``` + +Run-phase local temporaries (e.g., `cape`, `cme`, `mu`, `md`) are allocated at function +entry and deallocated at exit: +```fortran +allocate(cape(col_start:col_end)) +... +call zm_convr_run(..., cape=cape, ...) +... +deallocate(cape) +``` + +These temporaries use `col_start` as the lower bound so that assumed-shape dummy arguments +in schemes see a 1-based array — a subtle but important detail. + +--- + +### 10.6 Horizontal chunking model + +CAM-SIMA uses `col_start`/`col_end` (passed as arguments to every run subroutine) to +define the current horizontal chunk: + +```fortran +ncol = col_end - col_start + 1 +``` + +Schemes declare `horizontal_loop_extent` and receive `ncol`. The horizontal dimension +in the host (storage dimension) is `columns_on_task`. The subsetting from storage to +loop extent happens at the boundary between host cap and suite cap — the host cap +passes the right subsections: + +```fortran +! In cam_ccpp_cap.F90: +call cam7_physics_before_coupler(..., col_start=col_start, col_end=col_end, & + pmid=phys_state%pmid, ...) ! full arrays passed; suite cap subsets internally +``` + +Inside the suite cap, persistent arrays are subsetted explicitly when passed to schemes: +```fortran +windu_tend(col_start:col_end, 1:pver) +``` +Local temporaries allocated as `allocate(cape(col_start:col_end))` are already +correctly sized and passed as assumed-shape `(:)`. + +--- + +### 10.7 State machine + +The suite cap has a character module variable tracking lifecycle state: + +```fortran +character(len=16) :: ccpp_suite_state = 'uninitialized' +``` + +Transitions: `uninitialized` → register → `uninitialized` → initialize → `initialized` +→ timestep_initial → `in_time_step` → (run, no state change) → timestep_final → +`initialized` → finalize → `uninitialized`. + +Each phase entry point checks the expected prior state: +```fortran +if (trim(ccpp_suite_state) /= 'in_time_step') then + errflg = 1 + write(errmsg, '(3a)') "Invalid initial CCPP state, '", trim(ccpp_suite_state), & + "' in cam7_physics_before_coupler" + return +end if +``` + +Non-run phases also include an OpenMP thread guard: +```fortran +#ifdef _OPENMP + if (omp_get_thread_num() > 1) then + errflg = 1 + errmsg = "Cannot call initialize routine from a threaded region" + return + end if +#endif +``` + +The state machine is simple, complete, and useful. The redesign should preserve it. + +--- + +### 10.8 Constituent variable handling + +CAM-SIMA demonstrates the full constituent lifecycle: + +```fortran +! In cam_ccpp_cap.F90: +type(ccpp_model_constituents_t), target :: cam_constituents_obj + +! Registration (scheme-declared constituents): +call suite_cam7_constituents_num_consts(num_consts) +call suite_cam7_constituents_const_name(iconst, const_name) +call cam_constituents_obj%new_field(const_name, ...) + +! Initialization (host-declared constituents like water vapor): +cam_model_const_stdnames(1) = "water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water" +call cam_constituents_obj%new_field(cam_model_const_stdnames(1), ...) + +! Per-timestep gather from host: +call cam_ccpp_gather_constituents(phys_state%q, ...) + +! Passing to suite cap: +call cam7_physics_before_coupler(..., + qv = cam_constituents_obj%vars_layer(:, :, cam_model_const_indices(1)), + carr = cam_constituents_obj%vars_layer, + cprops = cam_constituents_obj%const_metadata, ...) + +! Per-timestep scatter back to host: +call cam_ccpp_update_constituents(phys_state%q, ...) +``` + +The suite cap sees constituents as: +- `carr(:,:,:)` — the full rank-3 constituent array (ncol, nlev, ncnst) +- `qv(:,:)` — water vapor slice extracted in the host cap: `cam_constituents_obj%vars_layer(:,:,cam_model_const_indices(1))` +- `cprops(:)` — array of `ccpp_constituent_prop_ptr_t` metadata objects +- `doconvtran(1:ncnst)` — suite-level logical array set by scheme init indicating which constituents are convected + +This constituent API is sophisticated and worth preserving or improving in the redesign. + +--- + +### 10.9 Known defects in the capgen output + +**Repeated scheme init/final calls.** Capgen generates one init call per occurrence of +a scheme name in the XML, without deduplication: +- `qneg_init` called 5 times (once per `qneg` entry in the suite XML) +- `qneg_timestep_final` called 5 times +- `check_energy_chng_init` called twice +- `save_ttend_from_convect_deep_timestep_init` called 3 times + +If these routines have internal state, allocations, or side effects, this is a correctness +defect. The redesign must deduplicate init/final calls by unique scheme name. + +**Unit conversion embedded silently in the cap.** Before `zm_conv_convtran_run`: +```fortran +dpdry_local(:,1:pver) = 1.0E-2_kind_phys * dpdry(:,1:pver) ! Pa → hPa +``` +This is generated from the metadata units mismatch but appears as an opaque transform +in the cap. The redesign should make this visible (e.g., a comment naming the standard +name, the source units, and the target units). + +--- + +### 10.10 Build system — capgen invocation + +Capgen is invoked from Python (`cam_autogen.py`), not from cmake: + +```python +from ccpp_capgen import capgen +capgen_db = capgen(run_env, return_db=True) +``` + +This is a programmatic API call, not a subprocess. The `CCPPDatabaseObj` returned +(`capgen_db`) is then used directly in Python to query scheme lists, constituent names, +and file paths — avoiding the datatable XML query step that cmake-based invocations need. + +Output files consumed by the build: +- `cam_ccpp_cap.F90` — compiled into the atmosphere component +- `ccpp_cam7_cap.F90` — compiled into the atmosphere component +- `ccpp_kinds.F90` — compiled into the atmosphere component +- Utility files from `ccpp_framework/src/` (copied to build dir) +- `ccpp_datatable.xml` — queried by the build system for file lists + +--- + +### 10.11 Observations relevant to the redesign + +1. **The two-cap split (host cap + suite cap) is the right architecture.** It cleanly + separates host-specific binding from physics-neutral dispatch. The redesign must + preserve this. + +2. **Flat-field arguments in the suite cap are the critical failure.** 61–76 dummy + arguments per run subroutine is already large for a research model; for UFS/GFS + with 1,200+ variables it is completely infeasible. The redesign must pass DDTs. + +3. **The CAM-SIMA host does not use DDTs in metadata.** All host variables are flat + module variables. This is a fundamentally different host model architecture from + GFS/SCM. The redesign must support both styles: flat-module hosts (CAM-SIMA) and + deep-DDT hosts (GFS/SCM). + +4. **Non-metadataized variables hardwired into the host cap is a serious gap.** + `phys_state`, `phys_tend`, `cam_in` from `physics_types` have no `.meta` files. + The host cap accesses them directly. This means the framework cannot verify or + track these variables. The redesign should either require full metadata coverage + or have an explicit mechanism for declaring non-metadataized pass-through variables. + +5. **Suite-level persistent variables (framework-owned data) work well in practice.** + `windu_tend`, `scaling_dycore`, `temp_ini`, etc. are owned by the suite cap, invisible + to the host, and persist across group calls. This is the right pattern for + physics-internal state. The redesign needs this but with a simpler discovery mechanism + than capgen's scope-chain promotion. + +6. **Deduplicate init/final calls.** The redesign must deduplicate `_init`, `_finalize`, + `_timestep_init`, and `_timestep_final` calls by unique scheme name (not by occurrence + in the XML). + +7. **The constituent API in the host cap is comprehensive.** The `ccpp_model_constituents_t` + object with its register/init/gather/scatter/index API is sophisticated and should be + preserved or improved. + +8. **The suite-variables introspection subroutine** (`ccpp_physics_suite_variables`, + enumerating 83 standard names as inputs/outputs) is a useful capability for build + system integration and should be in the redesign. + +9. **The programmatic Python API** (`capgen(run_env, return_db=True)`) is valuable + for hosts like CAM-SIMA that invoke the generator from Python. The redesign should + support both CLI and programmatic invocation. + +10. **Unit conversions must be annotated in the generated cap**, not silently embedded + as magic-number multiplications. A comment with source units, target units, and the + standard name involved is the minimum. + +11. **The horizontal chunking model** (`col_start`/`col_end` as explicit arguments, + `ncol = col_end - col_start + 1` computed at entry) works and is clean. Suite-level + persistent arrays are allocated full-size and subsetted at call sites. + +12. **No optional variables in this model.** CAM-SIMA does not exercise optional/active + variable handling. This feature must be in the redesign but is not demonstrated here. + +--- + +## 11. Real-world example: UFS Weather Model (prebuild) + +The UFS Weather Model is the most complex and production-critical of the three examples. +It is a fully-coupled, 3-D operational NWP model. The CCPP physics is used in the +atmospheric component (`UFSATM`). Unlike the SCM (column model, process-split only) and +CAM-SIMA (capgen, flat-field arguments), UFS uses prebuild in a 3-D blocked/threaded +configuration that is architecturally distinct from both prior examples. + +The two suites analyzed here are: +- `FV3_GFS_v17_coupled_p8` — the primary operational GFS suite +- `FV3_GFS_v17_coupled_p8_ugwpv1` — a variant replacing `unified_ugwp` with `ugwpv1` + +The ugwpv1 suite is structurally identical to the base suite except for the `phys_ps` +group (4 extra scheme calls), so all observations below apply to both. + +--- + +### 11.1 Suite structure + +The primary suite has 5 groups: + +| Group | Subcycles | Scheme calls | Phase called | +|-------|-----------|-------------|--------------| +| `time_vary` | 1 | 4 | timestep_init (domain-level, no blocking) | +| `radiation` | 1 | 8 | run (block/thread loop) | +| `phys_ps` | 3 (loop=1, loop=2, loop=1) | 21 | run (block/thread loop) | +| `phys_ts` | 3 (loop=1, loop=1, loop=1) | 12 | run (block/thread loop) | +| `stochastics` | 1 | 2 | run (block/thread loop) | + +The `time_vary` group is the only one called at timestep_init/finalize. All other groups +are called from the run phase via the OpenMP blocked loop. This is a fundamentally +different usage pattern from SCM (which runs everything sequentially) and CAM-SIMA +(which has no run phase at all for the groups analyzed). + +The `phys_ps` group has a surface iteration subcycle with `loop="2"`, which generates an +actual Fortran `do` loop in the cap body: +```fortran +! Start of next subcycle +cdata%loop_max = 2 +do cdata%loop_cnt = 1, cdata%loop_max + ! ... sfc_diff, sfc_nst, noahmpdrv, sfc_land, sfc_cice, sfc_sice ... +end do +``` + +--- + +### 11.2 Cap hierarchy and scale + +The three-level hierarchy is preserved from prebuild: + +``` +ccpp_static_api.F90 (627 lines) ← suite+group name dispatch + ↓ +ccpp_fv3_gfs_v17_coupled_p8_cap.F90 (363 lines) ← calls all group caps in order + ↓ +ccpp_fv3_gfs_v17_coupled_p8_time_vary_cap.F90 (1404 lines) +ccpp_fv3_gfs_v17_coupled_p8_radiation_cap.F90 (967 lines) +ccpp_fv3_gfs_v17_coupled_p8_phys_ps_cap.F90 (4226 lines) ← 200 optional ptr arrays +ccpp_fv3_gfs_v17_coupled_p8_phys_ts_cap.F90 (1953 lines) +ccpp_fv3_gfs_v17_coupled_p8_stochastics_cap.F90 (443 lines) +``` + +The ugwpv1 variant generates another 10,220 lines of largely redundant code (identical +caps with one suite-name prefix change and minor scheme-list differences). Total for both +suites: 18,333 lines of generated Fortran. + +This redundancy is a key motivation for the redesign: suite variants that share groups +should not regenerate identical cap code. The redesign should support group-level cap +sharing across suite variants. + +--- + +### 11.3 Host model DDT structure + +All host data lives in `CCPP_data.F90` as module-level `save, target` variables: + +```fortran +type(GFS_control_type) :: GFS_control ! config/control +type(GFS_statein_type) :: GFS_statein ! atmospheric state in +type(GFS_stateout_type) :: GFS_stateout ! atmospheric state out +type(GFS_grid_type) :: GFS_grid ! grid geometry +type(GFS_tbd_type) :: GFS_tbd ! temporal interp data +type(GFS_cldprop_type) :: GFS_cldprop ! cloud properties +type(GFS_sfcprop_type) :: GFS_sfcprop ! surface properties +type(GFS_radtend_type) :: GFS_radtend ! radiation tendencies +type(GFS_coupling_type) :: GFS_coupling ! coupling fields +type(GFS_diag_type) :: GFS_intdiag ! diagnostics +type(GFS_interstitial_type), allocatable (:) :: GFS_interstitial ! scratch, per thread +``` + +Plus three `ccpp_t` instances for different levels of parallelism (see §11.5). + +This is structurally similar to the SCM's `physics` DDT hierarchy, but with one key +difference: all DDTs are at the same flat level rather than nested (no `physics%Statein`, +only `GFS_statein`). Each DDT maps to a distinct functional role. + +The `GFS_typedefs.F90` file (not auto-generated) defines all DDT types along with ~30 +physical constants (`con_pi`, `con_g`, `con_rd`, etc.) that also appear in the metadata. + +--- + +### 11.4 DDT arguments in the cap chain + +The static API imports all DDTs and physical constants from `CCPP_data` and `GFS_typedefs` +via `use` statements, then passes them as named arguments to group cap functions. This is +the full DDT-argument pattern that prebuild implements: + +```fortran +! In ccpp_static_api.F90: +use ccpp_data, only: gfs_control, gfs_statein, gfs_sfcprop, ... +use gfs_typedefs, only: con_pi, con_g, con_rd, ... + +ierr = fv3_gfs_v17_coupled_p8_phys_ps_run_cap( & + one=one, gfs_control=gfs_control, cdata=cdata, & + gfs_statein=gfs_statein, gfs_sfcprop=gfs_sfcprop, & + con_g=con_g, con_pi=con_pi, ... & + gfs_interstitial=gfs_interstitial) +``` + +The group cap receives these as typed `intent(*), target` dummy arguments and uses them +directly to construct call-site subsections. This means **the group cap is fully portable +— it does not use any host module directly**, only what it receives as arguments. + +The `target` attribute is required because the cap creates pointer sections of these DDTs +(array subsections via pointer assignment) when handling optional variables. + +--- + +### 11.5 The dual cdata architecture + +UFS uses two distinct sets of `ccpp_t` handles with different scopes: + +**Domain-level (`cdata_domain`)**: Used for non-run phases (init, finalize, time_vary +timestep_init/finalize). Called once per step, no blocking: +```fortran +cdata_domain%blk_no = 1; cdata_domain%chunk_no = 1 +cdata_domain%thrd_no = 1; cdata_domain%thrd_cnt = 1 +``` + +**Block/thread-level (`cdata_block(nb, nt)`)**: Used for run phase (radiation, phys_ps, +phys_ts, stochastics). Allocated as a 2-D array `(1:nblks, 1:nthrdsX)` where `nthrdsX` +accounts for non-uniform last-block sizing: +```fortran +cdata_block(nb,nt)%blk_no = nb +cdata_block(nb,nt)%chunk_no = nb ! block number = chunk number +cdata_block(nb,nt)%thrd_no = nt +cdata_block(nb,nt)%thrd_cnt = nthrdsX +``` + +The redesign must support this dual cdata usage: a single `cdata` handle for domain-level +phases and a 2-D array of handles for blocked run phases. + +--- + +### 11.6 OpenMP threading model + +Non-run phases allow internal threading in physics schemes: +```fortran +GFS_control%nthreads = nthrds ! all N threads available to physics +call ccpp_physics_timestep_init(cdata_domain, ...) +``` + +Run phase uses all threads for blocking, so physics must not spawn additional threads: +```fortran +GFS_control%nthreads = 1 ! no internal threading allowed +!$OMP parallel num_threads(nthrds) ... +!$OMP do schedule(dynamic,1) +do nb = 1, nblks + call GFS_Interstitial(nt)%create(ixs=chunk_begin(nb), ixe=chunk_end(nb), model=GFS_control) + call ccpp_physics_run(cdata_block(nb,nt), group_name="phys_ps", ...) + call GFS_Interstitial(nt)%destroy(GFS_control) +end do +!$OMP end do +!$OMP end parallel +``` + +The `nt = omp_get_thread_num()+1` pattern (1-based thread index) is used throughout. +Each thread owns one `GFS_Interstitial(nt)` and one `cdata_block(nb,nt)` per block +iteration. The dynamic schedule means different threads process different blocks at +different times, which is why the interstitial must be created/destroyed per-iteration +rather than pre-allocated per-thread. + +--- + +### 11.7 Horizontal dimension: the chunk_begin/chunk_end pattern + +For non-run phases, the full horizontal dimension is used at every call site: +```fortran +tgrs(one:gfs_control%ncols, one:gfs_control%levs) +``` + +For run phases, the chunk range is looked up from the control DDT using the block number: +```fortran +tgrs(gfs_control%chunk_begin(cdata%chunk_no) : gfs_control%chunk_end(cdata%chunk_no), & + one:gfs_control%levs) +``` + +The chunk size (horizontal extent `im`) is retrieved as: +```fortran +im = gfs_control%blksz(cdata%blk_no) +``` + +`blksz(nb)` handles **non-uniform block sizes**: the last block may be smaller than the +others if the domain size is not divisible by the number of blocks. The `chunk_begin`/ +`chunk_end` arrays (indexed by chunk number = block number) give the global offset range. + +This is a cleaner pattern than SCM's `chunk_begin`/`chunk_end` as explicit dummy +arguments, because UFS looks them up from the already-passed `gfs_control` DDT. + +**Critical implication for the redesign**: The subsetting pattern `(chunk_begin:chunk_end)` +appears at every single array call site in the run phase — literally hundreds of times in +the phys_ps cap alone. This boilerplate is generated by prebuild from the metadata. In +the redesign, this subsetting must remain at the call site (not higher up) to allow each +thread to process its own chunk independently. + +--- + +### 11.8 The GFS_interstitial — pointer-based scratch DDT + +`GFS_interstitial_type` (defined in `CCPP_typedefs.F90`) is a DDT where **every field is +a pointer**, initialized to null: +```fortran +type GFS_interstitial_type + real(kind_phys), pointer :: adjsfculw_land(:) => null() + real(kind_phys), pointer :: del(:,:) => null() + ! ... ~200+ pointer fields +end type +``` + +This is dramatically different from the SCM's interstitial (which is a regular allocatable +DDT allocated once per thread at startup). The UFS interstitial is: +1. **Created** (`GFS_Interstitial(nt)%create(ixs, ixe, model)`) before each block — this + allocates all required fields to the chunk size `ixe-ixs+1` +2. **Reset** (`GFS_Interstitial(nt)%reset(model)`) to zero before radiation and phys_ps +3. **Destroyed** (`GFS_Interstitial(nt)%destroy(model)`) after each block — deallocates + +This design exists because different blocks (especially the last block) can have different +sizes. Pre-allocating to the maximum size wastes memory at scale; per-block allocation +ensures exact sizing. The pointer-based design also allows the `create()` method to +selectively allocate only the fields needed for the current physics configuration. + +In the caps, the interstitial is accessed as: +```fortran +gfs_interstitial(cdata%thrd_no)%del(chunk_begin:chunk_end, one:levs) +``` + +The interstitial array is 1-D (indexed by thread, not by `(instance, thread)` as in SCM). +This works because UFS has only one model instance at runtime — no ensemble-in-memory. + +--- + +### 11.9 Optional variables — the pointer array pattern at scale + +The phys_ps run cap has **200 optional pointer arrays** in its local variable section. +Each looks like: +```fortran +type :: real_kind_phys_rank1_ptr_arr_type + real(kind_phys), dimension(:), pointer :: p => null() +end type real_kind_phys_rank1_ptr_arr_type +type(real_kind_phys_rank1_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array +``` + +Usage pattern (consistent with SCM but with threading dimension): +```fortran +if (gfs_control%lndp_type /= 0) then + sfc_wts_1_ptr_array(cdata%thrd_no)%p => & + gfs_coupling%sfc_wts(chunk_begin:chunk_end, one:gfs_control%n_var_lndp) +end if +! ... scheme call ... +if (gfs_control%lndp_type /= 0) then + nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) +end if +``` + +The array is dimensioned by `cdata%thrd_cnt` (total thread count) and indexed by +`cdata%thrd_no` (current thread number). This handles the threaded run phase where +multiple threads are simultaneously executing the same run cap function with different +chunk ranges. Each thread independently associates and nullifies its own pointer slot. + +200 optional variables in `phys_ps` alone. This is the regime for which the SCM had ~550 +total optional vars — confirming that operational 3-D GFS physics is heavily optional-var +driven. The design is sound but generates enormous boilerplate. + +A key observation: the type definition for each pointer wrapper (`integer_..._ptr_arr_type`, +`real_kind_phys_rank1_ptr_arr_type`, etc.) is **re-declared inside every single function +that needs it**. This results in duplicate type definitions across all group caps. The +redesign should define these wrapper types once in a shared module. + +--- + +### 11.10 Physical constants as metadata variables + +The UFS static API has an extensive USE list of physical constants from `gfs_typedefs`: +``` +con_pi, con_g, con_t0c, con_hfus, con_solr_2008, con_solr_2002, con_c, con_plnk, +con_boltz, con_rd, ltp, con_zero, con_rerth, con_p0, con_rv, con_cp, con_rgas, +con_amd, con_amw, con_avgd, con_hvap, con_eps, con_omega, con_fvirt, con_ttp, +con_thgni, con_epsm1, con_rog, con_rocp, con_tice, con_sbc, con_jcal, con_rhw0, +rlapse, rhowater, karman, con_1ovg, con_cliq, con_cvap, rainmin, con_epsm1 (30+ total) +``` + +These travel through the full chain: static API USE → suite cap argument → group cap +argument → scheme call argument. Each constant is declared as a separate scalar dummy +argument (`real(kind_phys), intent(in), target :: con_pi`) in every group cap that needs +it. + +This is correct but verbose. The redesign should consider whether constants should be +gathered into a dedicated DDT (e.g., `gfs_constants_type`) so the cap chain carries one +argument instead of 30. This would also eliminate the need to explicitly enumerate which +constants each group needs — they could all come along in the constants DDT. + +--- + +### 11.11 The `one` lower-bound anchor + +The integer constant `one = 1` (from `ccpp_types`) is passed as an explicit argument +throughout the UFS cap chain for the same reason as in SCM: it anchors lower array bounds +without triggering association-status issues: +```fortran +type(gfs_interstitial_type), intent(inout), target :: gfs_interstitial(one:) +tgrs(one:gfs_control%ncols, one:gfs_control%levs) +``` + +This pattern is ubiquitous and is a known prebuild idiom. + +--- + +### 11.12 No framework-owned persistent variables + +Unlike CAM-SIMA (which allocates scheme-persistent variables in the suite cap), the UFS +has no framework-owned persistent state in any cap. All persistent state lives in the host +DDTs (`GFS_tbd`, `GFS_sfcprop`, etc.). The interstitial DDT (`GFS_interstitial`) is purely +transient — created and destroyed each block. + +This is consistent with UFS's prebuild-based architecture. Whether framework-owned +persistent variables would be beneficial for UFS is an open question for the redesign. + +--- + +### 11.13 Build system and driver + +Prebuild is invoked from CMake (not programmatically) and generates: +- Group cap files (one per group × suites) +- Suite cap files (one per suite) +- `ccpp_static_api.F90` +- `CCPP_CAPS.cmake`, `CCPP_SCHEMES.cmake`, `CCPP_TYPEDEFS.cmake` — consumed by CMake to + enumerate files to compile + +The host driver (`CCPP_driver.F90`) is **hand-written**, not auto-generated. It owns the +OpenMP loop, the cdata allocation/setup, the interstitial create/destroy, and the +diagnostic bucket zeroing. This is a significant difference from CAM-SIMA where the +equivalent driver code is partially generated. In the redesign, this host driver code +should remain hand-written — it encodes model-specific threading and blocking decisions +that cannot be derived from metadata alone. + +--- + +### 11.14 Observations relevant to the redesign + +1. **The DDT-argument cap chain is fully validated at UFS scale.** Passing 10+ DDTs plus + 30+ scalar constants as named arguments through three cap levels works correctly in + production. The redesign must replicate this exactly. + +2. **The chunk_begin/chunk_end subsetting at call sites is non-negotiable.** Hundreds of + array sections per group cap. The generator must produce this from the metadata + `horizontal_dimension` standard name and the `active` flag for optional variables. + This is prebuild's core value at 3-D scale. + + *Design direction*: Rather than carrying `chunk_no` in cdata and having the cap look + up `gfs_control%chunk_begin(chunk_no)`, the redesign should pass + `horizontal_loop_begin` and `horizontal_loop_end` as explicit arguments directly to + `ccpp_physics_run()` (and analogous calls). This decouples the cap from knowing about + the host's internal chunk-lookup arrays. The host driver sets these for each block + iteration and passes them in; the cap uses them directly. + +3. **The domain-vs-block execution contexts must be supported, but the cdata object is + not necessarily the right mechanism.** The key information is: instance number, thread + number, horizontal_loop_begin, horizontal_loop_end, error flag/message. If all of + these are explicit named arguments to `ccpp_physics_*`, the cdata object becomes + redundant scaffolding. This is an open design question to be discussed separately, but + the UFS analysis shows that cdata carries exactly these values — the object is a + transport container, not a framework abstraction. + +4. **The `blksz` non-uniform block size is a first-class concern.** The generator must + produce `im = gfs_control%blksz(cdata%blk_no)` (or an equivalent `horizontal_loop_extent` + computed from the explicit begin/end) for the horizontal extent argument in run phases. + +5. **GFS_interstitial as a pointer-DDT is the correct design for 3-D models.** Creating + and destroying per block avoids memory waste from over-allocation to the maximum chunk + size. The pointer-based field design enables selective allocation. The redesign should + document this pattern and support it. (Whether the generator should emit the + `type(X_interstitial_type)` DDT definition itself or only the caps is TBD.) + +6. **200 optional pointer arrays in one group cap is manageable but the wrapper type + proliferation is not.** The 4 wrapper types (`integer_r1_ptr_arr_type`, + `real_r1_ptr_arr_type`, `real_r2_ptr_arr_type`, `character_len3_r1_ptr_arr_type`) + should be defined once in a shared module (e.g., `ccpp_types.F90`) and reused across + all caps, eliminating thousands of duplicate lines. + +7. **Physical constants as metadata variables must be gathered into a constants DDT.** + The redesign will collect all physics constants into a single `constants_type` DDT + (or equivalent), reducing 30+ individual scalar arguments in the cap chain to one + argument. This requires a metadata declaration mechanism for compound read-only + objects (i.e., constants do not need intent tracking the way state variables do). + +8. **No framework-owned persistent variables in UFS** confirms that this feature is + optional and model-specific. The redesign needs to support it (for CAM-SIMA-like + models) but should not force it on models that do not need it. + +9. **The host driver is correctly hand-written.** The OpenMP blocking, interstitial + lifecycle, diagnostic bucket management — these are model-specific decisions that + belong in the host driver, not in generated code. The redesign should not try to + generate the driver. + +10. **Suite variant cap redundancy is not a concern.** For research/development, multiple + suites are active simultaneously and generated code size doesn't matter. For + production, only one suite is compiled and used at a time. The redesign need not + prioritize eliminating redundant group cap code across suite variants. + +--- + +## 12. Real-world example: Navy NEPTUNE (prebuild, restricted) + +The NEPTUNE source code cannot be shared. The following is based on architectural +description provided by the lead developer. + +NEPTUNE uses `ccpp-prebuild` with the same GFS physics as UFS and nearly identical suites. +Its unique distinguishing feature is **multiple coexisting CCPP physics instances** — it +is the only model among the four examples that exercises this capability at runtime. + +--- + +### 12.1 Multiple instances — the N-dimensioned DDT array mechanism + +In NEPTUNE, the host model allocates N copies of all GFS DDTs as 1-D arrays indexed by +instance number: + +```fortran +type(GFS_sfcprop_type), allocatable :: gfs_sfcprop(1:N) +type(GFS_statein_type), allocatable :: gfs_statein(1:N) +type(GFS_stateout_type), allocatable :: gfs_stateout(1:N) +! ... all GFS DDTs dimensioned 1:N +type(GFS_control_type), allocatable :: gfs_control(1:N) +``` + +The static API imports these module-level arrays via `use` statements (same as UFS). +The instance selection happens at the call site inside the group cap, using +`cdata%ccpp_instance` as the array index: + +```fortran +call foo_run( & + tair = gfs_statein(cdata%ccpp_instance)%tair( & + gfs_control(cdata%ccpp_instance)%chunk_begin(cdata%chunk_no) : & + gfs_control(cdata%ccpp_instance)%chunk_end(cdata%chunk_no), & + 1:nvertical), & + ...) +``` + +Three things are happening simultaneously at each call-site array section: +1. **Instance selection**: `gfs_statein(cdata%ccpp_instance)` picks the correct DDT from + the N-element array +2. **Chunk subsetting**: `chunk_begin(chunk_no):chunk_end(chunk_no)` applies the run-phase + horizontal slice +3. **Vertical bound**: explicit `1:nvertical` + +This is the same pattern as UFS except the DDTs are 1-D arrays rather than scalars. +The generator must produce this instance-indexed subsetting when the host declares its +DDTs as arrays. + +--- + +### 12.2 What NEPTUNE tells us about `cdata%ccpp_instance` + +The `initialized(200)` array in every group cap (confirmed in both SCM and UFS caps) now +has its full motivation: it handles up to 200 simultaneous instances without requiring +per-instance cap code. The `cdata%ccpp_instance` value (1-based) is the runtime selector +into both the host DDT arrays and the `initialized` guard array. + +NEPTUNE is the reason `200` is not `1`. In single-instance models (UFS, SCM, CAM-SIMA) +`cdata%ccpp_instance` is always 1 and the N-dimensioned DDT arrays have `N=1`. + +--- + +### 12.3 Observations relevant to the redesign + +1. **Multiple instances require only one change at the call site**: inserting the instance + index at the correct dimension position. Everything else (chunking, optional variables, + threading) composes with this unchanged. + +2. **The instance dimension can appear anywhere in any host variable — not just as an + index into an array of DDTs.** A flat array `flat_field(1:ninstance, 1:nhoriz, 1:nvert)` + is equally valid; its call site becomes: + ```fortran + flat_field(instance_number, horiz_begin:horiz_end, 1:nvert) + ``` + The generator handles this by classifying each dimension by its declared standard name. + `instance_dimension` is a registered standard name (like `horizontal_dimension` and + `vertical_dimension`) — the generator knows its semantics regardless of where it + appears in the dimension list or whether the variable is a DDT array element or a + plain array. See §13.4 for the full dimension classification model. + +3. **No new cap-level mechanism is needed for multi-instance.** The instance number + (from the control layer, see §13) is sufficient. The cap code shape is the same; + only the call-site indexing expression differs based on the declared dimension roles. + +--- + +## 13. Cross-cutting design decision: how host data enters the cap chain + +Across all four models, two mechanisms are used for getting host model data into the +generated caps: + +| Mechanism | Models using it | Description | +|-----------|----------------|-------------| +| **Module USE** | UFS, SCM, CAM-SIMA, NEPTUNE | Static API has `use ccpp_data, only: gfs_statein, ...`. Data module name is known at generation time. | +| **Command-line arguments** | capgen (optional) | Generator accepts host variable access paths as CLI flags; generated caps receive data as explicit dummy arguments. | + +### 13.1 The capgen dual-mechanism problem + +Capgen supports both mechanisms, and this is a direct source of its complexity. The +variable-matching logic, VarDictionary scope chains, and `CCPPDatabaseObj` all exist +partly to handle the routing of variables that may arrive via either path. Maintaining +two entry points to the data layer doubles the surface area that must be tested and +reasoned about. + +### 13.2 The proposed single-mechanism approach + +The redesign will use **module USE exclusively** for all host data. The reasoning: + +- All four production models already use module USE, including CAM-SIMA (the capgen + model), which does not use capgen's CLI-argument path in practice. +- Module names are stable, known at generation time, and make the generated code + self-documenting (`use ccpp_data, only: gfs_statein` is unambiguous). +- Eliminating the CLI-argument entry path eliminates an entire class of generator + complexity. + +### 13.3 Runtime control variables — the thin explicit layer + +While all *data* enters via module USE, a set of *control* variables must be passed at +runtime because they change from call to call. These are not physics data; they tell the +cap *how* to index into the data it already has access to: + +| Variable | Purpose | When it matters | +|----------|---------|----------------| +| `ccpp_instance` | Select the instance dimension in host variables | NEPTUNE (N>1); others use 1 | +| `ccpp_thread_no` | Index optional pointer arrays per thread | Run phase with OpenMP | +| `horizontal_loop_begin` | Start of horizontal chunk to process | Run phase | +| `horizontal_loop_end` | End of horizontal chunk to process | Run phase | +| `ccpp_nthreads` | Max threads available for internal physics use | Non-run phases (currently `gfs_control%nthreads`) | +| `errmsg` / `errflg` | Error reporting return path | All phases | + +These are exactly the values that `cdata` carries in the current implementation. +Whether they are packaged as a `ccpp_t` struct or passed as individual named arguments to +`ccpp_physics_*` is an open design question for implementation. Either way, the generator +only needs to know about these variables and their standard names — it does not need to +accept host data paths on the command line. + +### 13.4 The dimension classification model + +A host variable's metadata declares the **standard name of each of its dimensions** in +order. The generator classifies every dimension into one of three categories and +constructs the call-site expression accordingly. + +**Category 1 — Registered dimensions.** The generator knows the semantics of these +standard names and generates special call-site expressions for them: + +| Standard name | Call-site expression | Notes | +|--------------|---------------------|-------| +| `instance_dimension` | `instance_number` (scalar index) | Omitted if variable has no instance dimension | +| `horizontal_dimension` | `1:horizontal_dimension` (non-run) or `horiz_begin:horiz_end` (run) | Phase-dependent | +| `vertical_dimension` | `1:vertical_dimension` | Fixed range | + +`instance_dimension` has the same registered status as `horizontal_dimension` and +`vertical_dimension`. Single-instance models simply do not declare any variables with +an `instance_dimension`, and the generator omits that index entirely. + +**Category 2 — Arbitrary host-declared dimensions.** Any dimension whose standard name +is not in the registered set. These are declared in host metadata pointing to a Fortran +expression accessible via module USE — either a flat module variable or a DDT member +(e.g. `gfs_control%ntrac`, `gfs_control%kice`). The generator emits `1:expression` +at the call site, resolved at generation time from the metadata. Fixed-index extractions +(e.g. `gfs_statein%qgrs(..., gfs_control%ntqv)`) are a special case: the dimension +value is a scalar index rather than a range upper bound, and the metadata must declare +which case applies. + +**Category 3 — Optional selector.** Not a dimension per se, but a boolean `active` +condition declared in variable metadata. Generates a pointer-association guard around +the call site (the pattern described in §9 and §11). + +This three-category model works uniformly regardless of host layout: +- `gfs_statein(instance)%tair(horiz, vert)` — registered instance + registered horizontal + registered vertical +- `flat_field(instance, horiz, vert, ntrac)` — registered + registered + registered + arbitrary +- `flat_field(horiz, vert)` — no instance dimension, single-instance model + +No special-casing per host model is needed in the generator. + +### 13.5 `type = control` — metadata declaration for runtime control variables + +The registered dimensions (§13.4 Category 1) are *dimension names* that appear in a +variable's `dimensions = (...)` list. Their actual *runtime values* are supplied by a +separate set of variables declared with `type = control` in host metadata. + +| `type = control` standard name | Fills in registered dimension / purpose | +|-------------------------------|----------------------------------------| +| `ccpp_instance` | `instance_dimension` — scalar index selecting the active instance | +| `ccpp_thread_no` | Not a dimension; indexes optional pointer arrays per thread | +| `horizontal_loop_begin` | Lower bound of `horizontal_dimension` in run phase | +| `horizontal_loop_end` | Upper bound of `horizontal_dimension` in run phase | +| `ccpp_nthreads` | Not a dimension; max threads available for internal physics use | +| `errmsg` / `errflg` | Error reporting return path | + +Variables declared `type = control` are: +- **Passed explicitly as runtime arguments** to `ccpp_physics_*` by the host driver + (not accessed via module USE, because their values change per call) +- **Used by the generator** to construct call-site indexing expressions for registered + dimensions, and to generate the `ccpp_nthreads` assignment before non-run scheme calls +- **Available to physics schemes** by standard name like any other variable — if a scheme + declares a variable with a matching standard name (e.g. `ccpp_nthreads`, + `horizontal_loop_begin`), the framework passes it as a scheme argument in the normal way + +This is similar in concept to capgen's `type = host` annotation but with a narrower, +well-defined scope. The name `control` is intentional: these variables *control* how +the cap indexes into the data, not what the data is. + +The set of recognized standard names for `type = control` variables is fixed and small. +Declaring them explicitly in metadata — rather than having the generator recognize magic +names — keeps the mechanism open and self-documenting. + +### 13.6 Consequences for the generator + +1. The generator reads host metadata to learn: + - Module names for all host data variables (emitted as `use` statements in the static API) + - The dimension standard names of each variable (for call-site expression construction) + - Which variables are `type = control` (for the runtime argument layer) +2. At cap generation time, the static API's `use` statements are emitted from the module + names — no runtime flexibility, no CLI data routing. +3. Call-site subsetting for every variable is constructed purely from its declared + dimension standard names: registered dimensions use the Category 1 rules; arbitrary + dimensions are resolved to Fortran expressions via the host metadata. +4. The only runtime inputs to the cap are the `type = control` variables. Their values + are supplied by the host driver for each `ccpp_physics_*` call. diff --git a/doc/redesign_analysis.md b/doc/redesign_analysis.md new file mode 100644 index 00000000..67252a5f --- /dev/null +++ b/doc/redesign_analysis.md @@ -0,0 +1,2639 @@ +# CCPP Framework Code Generator — Technical Analysis for Redesign + +*Analysis date: 2026-05-04. Clarifications added: 2026-05-05.* + +This document is a deep-dive technical analysis of the two existing CCPP Framework code generators — +`ccpp-prebuild` and `ccpp-capgen` — produced as input to a planned complete redesign. +It covers execution flow, data structures, feature sets, build system integration, and +key architectural differences. + +--- + +## Table of Contents + +1. [Background and motivation](#1-background-and-motivation) +2. [ccpp-prebuild — detailed analysis](#2-ccpp-prebuild--detailed-analysis) +3. [ccpp-capgen — detailed analysis](#3-ccpp-capgen--detailed-analysis) +4. [Shared infrastructure](#4-shared-infrastructure) +5. [Feature comparison](#5-feature-comparison) +6. [Build system integration](#6-build-system-integration) +7. [Key architectural differences](#7-key-architectural-differences) +8. [Design considerations for the redesign](#8-design-considerations-for-the-redesign) +9. [Real-world example: CCPP Single Column Model (SCM)](#9-real-world-example-ccpp-single-column-model-scm) +10. [Real-world example: CAM-SIMA (capgen)](#10-real-world-example-cam-sima-capgen) +11. [Real-world example: UFS Weather Model (prebuild)](#11-real-world-example-ufs-weather-model-prebuild) +12. [Real-world example: Navy NEPTUNE (prebuild, restricted)](#12-real-world-example-navy-neptune-prebuild-restricted) +13. [Cross-cutting design decision: how host data enters the cap chain](#13-cross-cutting-design-decision-how-host-data-enters-the-cap-chain) + +--- + +## 1. Background and motivation + +The CCPP Framework is a code generator that analyzes metadata describing variables required +by physical parameterizations in numerical weather prediction (NWP) models, compares them +against metadata provided by a host model, and generates Fortran interface ("cap") code that +connects the two. + +There are two generations of the generator: + +**`ccpp-prebuild`** (`scripts/ccpp_prebuild.py`): +- Simple, mostly procedural Python +- Used in: NOAA UFS Weather Model, Navy NEPTUNE, CCPP-SCM +- Extremely reliable in research, development, and operations +- Fewer capabilities; simpler design + +**`ccpp-capgen`** (`scripts/ccpp_capgen.py`): +- Highly complex, object-oriented Python taken to the extreme +- Used in: NCAR CAM-SIMA (still mostly a research/development model) +- Many advanced features designed but never implemented (funding/priority gaps) +- Notoriously difficult to develop; no remaining team member fully understands it + +**The original plan** was to update `ccpp-capgen` with missing features from `ccpp-prebuild` +and transition all models to it. **This plan has been abandoned** in favor of a complete +redesign that draws the best lessons from both generations. + +The immediate trigger for abandoning capgen was the failure — after considerable effort by +three developers — to make capgen pass DDT arguments to group caps the way prebuild does. +This is the root cause of capgen's severe performance problem (seconds for prebuild, +10+ minutes for capgen on the same suite set) and of its broken handling of optional +variables under Fortran compiler debugging flags. + +--- + +## 2. ccpp-prebuild — detailed analysis + +### 2.1 Command-line arguments and configuration + +Entry point: `scripts/ccpp_prebuild.py`, `main()`. + +Arguments parsed by `argparse`: + +| Argument | Required | Purpose | +|---|---|---| +| `--config` | yes | Path to host-model Python config module | +| `--suites` | no | Comma-separated suite names (without `.xml`) | +| `--builddir` | no | Override build directory from config | +| `--namespace` | no | Appended to static API module name | +| `--debug` | no | Insert Fortran array-size checks in generated caps | +| `--clean` | no | Remove generated files and exit | +| `--verbose` | no | Set logging to DEBUG | + +The `--config` file is a plain Python module imported dynamically via `importlib`. +Key variables it must define: + +| Config variable | Purpose | +|---|---| +| `VARIABLE_DEFINITION_FILES` | List of host-model Fortran sources with metadata hooks | +| `SCHEME_FILES` | List of physics scheme Fortran sources | +| `CAPS_DIR` | Output directory for generated cap `.F90` files | +| `SUITES_DIR` | Directory containing suite definition XML files | +| `STATIC_API_DIR` | Output directory for `ccpp_static_api.F90` | +| `TYPEDEFS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for typedef build snippets | +| `SCHEMES_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for scheme build snippets | +| `CAPS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for cap build snippets | +| `HTML_VARTABLE_FILE`, `LATEX_VARTABLE_FILE` | Documentation output paths | +| `TYPEDEFS_NEW_METADATA` | Optional: dict enabling DDT member name translation bridge | + +The config file can contain arbitrary Python expressions — computed file lists, +conditional logic, environment-variable lookups — making it very flexible. + +### 2.2 Step-by-step execution pipeline + +``` +1. Import config module dynamically via importlib + +2. gather_variable_definitions() + for each file in VARIABLE_DEFINITION_FILES: + parse_variable_tables(file) [metadata_parser.py] + → metadata_define: OrderedDict[standard_name → [mkcap.Var]] + +3. collect_physics_subroutines() + for each file in SCHEME_FILES: + parse_scheme_tables(file) [metadata_parser.py] + → metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] + → arguments_request: OrderedDict[scheme → OrderedDict[subroutine → [std_names]]] + → dependencies_request: OrderedDict[scheme → [abs_paths]] + → schemes_in_files: OrderedDict[scheme → abs_path] + +4. compare_metadata() [batch matching] + for each std_name in metadata_request: + check exists in metadata_define + check type/kind/rank compatibility + register unit conversions in var.actions + copy local_name as var.target + → metadata: OrderedDict[std_name → [Var]] (targets and actions set) + +5. check_optional_arguments() [warnings only] + +6. For each requested suite XML: + Suite.parse(xml) [mkstatic.py] → Suite + Group objects + Group.write() → ccpp___cap.F90 + Suite.write() → ccpp__cap.F90 + +7. API.write() [mkstatic.py] + → ccpp_static_api[_].F90 + +8. Write build-system snippets [mkcap.py writers] + → CCPP_CAPS.cmake/mk/sh + → CCPP_SCHEMES.cmake/mk/sh + → CCPP_TYPEDEFS.cmake/mk/sh + → CCPP_API.cmake/sh + +9. mkdoc.metadata_to_html() → HTML variable table + mkdoc.metadata_to_latex() → LaTeX variable table +``` + +### 2.3 Data structures — the "flat dict" model + +Everything in prebuild lives in flat Python `OrderedDict` structures. There is no object +hierarchy; variables are simple Python objects with plain attributes. + +```python +# Top-level data containers +metadata_define: OrderedDict[standard_name → [mkcap.Var]] # 1 Var per std_name +metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] # N Vars (one per scheme×subroutine) +arguments_request: OrderedDict[scheme_name → OrderedDict[subroutine_name → [std_names]]] +dependencies_request: OrderedDict[scheme_name → [abs_paths]] +schemes_in_files: OrderedDict[scheme_name → abs_path] +``` + +`mkcap.Var` attributes: + +| Attribute | Type | Description | +|---|---|---| +| `standard_name` | str | CF-convention unique identifier | +| `long_name` | str | Human-readable description | +| `units` | str | Physical units | +| `local_name` | str | Fortran local name (may be DDT member reference) | +| `type` | str | Fortran type (real, integer, logical, or DDT name) | +| `kind` | str | Fortran kind parameter | +| `dimensions` | list[str] | Dimension standard names | +| `intent` | str | in / out / inout | +| `active` | str | `'T'`, `'F'`, or expression string | +| `optional` | str | `'T'` or `'F'` | +| `pointer` | bool | Whether Fortran POINTER attribute needed | +| `target` | str | Set during matching: the host model local_name | +| `actions` | dict | `{'in': fn, 'out': fn}` for unit conversions | +| `container` | str | Encoded provenance: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | + +**Performance note on `container` and `target`**: these two attributes act as a lookup +cache computed once during the `compare_metadata()` batch step. The `container` string +encodes where each variable lives in the host model (module and, if applicable, the +DDT member chain). The `target` records the resolved Fortran local name. Both are +computed once and then used directly during Fortran cap generation — no further dictionary +lookups are needed. This is a major contributor to prebuild's speed advantage. + +### 2.4 Metadata parsing and the bridge to capgen + +`metadata_parser.py` is a shared module that acts as a bridge. It detects whether a +metadata section in a Fortran source file uses the old pipe-delimited format (deprecated, +warning emitted) or the new `.meta` format (triggered by `!! \htmlinclude .html` +in the Fortran source comment hook). + +For `.meta` files, `read_new_metadata()` in `metadata_parser.py`: +1. Calls capgen's `metadata_table.parse_metadata_file()` → `[MetadataTable]` +2. Converts each `metavar.Var` to a `mkcap.Var` +3. Normalizes `active` to `'T'`/`'F'`/expression, `optional` to `'T'`/`'F'` + +The `TYPEDEFS_NEW_METADATA` config variable (when provided) triggers an additional +pass via `convert_local_name_from_new_metadata()` which translates flat +standard-name-style local names into DDT member references such as +`Atm(blk_no)%q(:,:,:,graupel_index)`. This is the bridge that makes the newer +`.meta` format work with the older DDT-heavy host model code. + +### 2.5 Variable matching — `compare_metadata()` + +A single batch function processes all matching. For each standard name in `metadata_request`: + +1. Check it exists in `metadata_define` — error if missing +2. Check there is exactly one definition — error if ambiguous +3. Call `var.compatible(other_var)` — checks equality of `standard_name`, `type`, `kind`, and rank +4. Register unit conversions: if units differ, `var.convert_from()` / `var.convert_to()` + stores a conversion function in `var.actions` +5. Check `active` attribute: if host variable is conditionally allocated and scheme variable + is not `optional`, issue a warning (not an error) +6. Copy `local_name` from the define side as `var.target` +7. Build module use list from container strings + +Result: `metadata` dict where each `Var` has `.target` set to the host model local name +and `.actions` populated with any needed unit conversion functions. + +### 2.6 Generated Fortran files + +#### Group cap: `ccpp___cap.F90` + +One module per group. For each CCPP stage (tsinit, init, run, tsfinal, finalize), a subroutine: + +```fortran +module ccpp_suite_A_physics_cap + use scheme_module, only: scheme_run + use host_module_A, only: ddt_A ! DDT, not flat fields + use host_module_B, only: ddt_B + implicit none + contains + + subroutine suite_A_physics_run_cap(ddt_A, ddt_B, im, iaend, ierr, ...) + type(ddt_A_type), intent(inout), target :: ddt_A ! entire DDT passed + type(ddt_B_type), intent(inout), target :: ddt_B + integer, intent(in) :: im, iaend ! loop bounds + integer, intent(out) :: ierr + logical, save :: initialized(200) = .false. + ! optional variable: local pointer, conditionally associated + real(kind_phys), pointer :: opt_var(:) => null() + if (ddt_A%active_flag) then + opt_var => ddt_A%opt_field + end if + ! unit conversion: local variable + real(kind_phys) :: converted_var(im) + converted_var(:) = ddt_B%field(:im) * conversion_factor + ! fixed-index extraction: local pointer for a specific tracer + real(kind_phys), pointer :: q_water_vapor(:,:) => null() + q_water_vapor => ddt_A%q(:,:,ntqv) ! ntqv = water vapor index in tracer array + ! call scheme with loop-bound application and extracted variables at the call site + call scheme_run( & + arg1 = ddt_A%field1(1:im), & ! horizontal loop-bound applied here + arg2 = ddt_A%field2(1:im,:), & ! loop-bound + all levels + qv = q_water_vapor(1:im,:), & ! specific tracer, loop-bound applied + arg3 = converted_var, & ! unit-converted local var + opt_arg = opt_var, & ! optional pointer + ...) + if (ierr /= 0) return + end subroutine +end module +``` + +Key points: +- **DDTs are passed as arguments, not flat fields.** Hundreds of variables arrive as + one or a small number of DDT arguments. This is the fundamental architectural choice + that makes prebuild fast and safe with compiler debugging flags. +- **Two distinct "subsetting" operations happen at the scheme call site:** + 1. *Loop-bound application*: horizontal range `1:im` (or `im` for scalar extents) + applied in the scheme call argument expressions. + 2. *Fixed-index extraction*: a specific element along one dimension is selected, + e.g. `q_water_vapor => ddt%q(:,:,ntqv)` extracts the water vapor tracer from the + full tracer array. A local pointer (or local variable for unit conversions) is + declared just before the scheme call and passed as the scheme argument. The group + cap always receives the full data; these extractions are local to the group cap. +- **Optional variables** are handled by declaring a local `pointer` variable and + conditionally associating it with the DDT field based on the `active` expression. + An unassociated pointer is passed to the scheme if the variable is inactive. This + avoids compiler exceptions when mandatory debugging flags are enabled, because the + unallocated field is never directly referenced — only the already-null pointer is. +- `logical :: initialized(200), save` — per-instance initialization tracking. The + 200 is the maximum number of complete model instances that can coexist in memory + simultaneously (used in ensemble approaches where multiple copies of the full model + state live in memory at once). Each instance has its own initialization flag. +- For the `run` phase, `im` and `iaend` (or similar) carry `horizontal_loop_begin` + and `horizontal_loop_end`, enabling OpenMP thread-level parallelism where each + thread processes a horizontal slice. +- Explicit keyword argument passing in scheme calls. +- Unit conversion: a local variable is declared and populated before the call; the + local variable is then passed to the scheme. +- Error check after each scheme call; returns immediately on error. +- `--debug` flag inserts Fortran array-size assertions. + +#### Suite cap: `ccpp__cap.F90` + +Imports all group cap functions and exposes one function per stage that chains group calls. + +#### Static API: `ccpp_static_api[_].F90` + +A single Fortran module `ccpp_static_api` with one subroutine per stage: + +```fortran +subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) + character(len=*), intent(in) :: suite_name, group_name + select case(trim(suite_name)) + case('suite_A') + select case(trim(group_name)) + case('physics') + call suite_A_physics_run_cap(cdata, ierr) + ... + end select + ... + end select +end subroutine +``` + +This is the **single entry point** the host model calls. The host model passes `suite_name` +and `group_name` at runtime; the static API dispatches to the appropriate cap function. + +### 2.7 Build system snippet files generated + +Six output files (Makefile, CMakefile, shell source) for three variable sets: + +| File | Content | +|---|---| +| `CCPP_CAPS.cmake` | `set(CAPS /abs/path/cap1.F90 /abs/path/cap2.F90 ...)` | +| `CCPP_SCHEMES.cmake` | `set(SCHEMES /abs/path/scheme1.F90 ...)` | +| `CCPP_TYPEDEFS.cmake` | `set(TYPEDEFS module1 module2 ...)` (module names, not paths) | +| `CCPP_API.cmake` | `set(API /abs/path/ccpp_static_api.F90)` | + +All files are written as `.tmp` first and compared against the existing version; they are +replaced only if the content changed, which avoids unnecessary recompilation of downstream +Fortran targets. + +### 2.8 What `mkcap.py`, `mkstatic.py`, and `mkdoc.py` each do + +**`mkcap.py`**: +- Defines the `mkcap.Var` class (prebuild's variable data class) +- Defines six file-writer classes: `CapsMakefile`, `CapsCMakefile`, `CapsSourcefile`, + `SchemesMakefile`, `SchemesCMakefile`, `SchemesSourcefile`, `TypedefsMakefile`, + `TypedefsCMakefile`, `TypedefsSourcefile` +- Each writer has a `write(file_list)` method that produces a formatted include file +- Does NOT generate any Fortran + +**`mkstatic.py`**: +- Defines `Suite`, `Group`, `Subcycle` classes that parse suite definition XML and + generate Fortran caps +- `Suite.parse()`: reads SDF XML via `xml.etree.ElementTree`, builds `Group` and + `Subcycle` objects +- `Suite.write()`: drives cap generation for all groups and the suite-level cap +- `Group.write()`: generates the group cap Fortran — argument list construction, + module `use` statements, unit conversion code, scheme calls, error handling +- Defines `API` class: generates the static API Fortran module (suite_name/group_name + dispatch switch) +- `CCPP_SUITE_VARIABLES` dict: mandatory variables always included (error message, + error code, loop counter, loop extent) +- Helper functions `extract_parents_and_indices_from_local_name()` and + `extract_dimensions_from_local_name()` handle complex DDT member access like + `Atm(blk_no)%q(:,:,:,graupel_index)` — these are critical for DDT-heavy host models + +**`mkdoc.py`**: +- `metadata_to_html()`: produces an HTML table of all host-model provided variables + (standard_name, long_name, units, rank, type, kind, source, local_name) +- `metadata_to_latex()`: produces a LaTeX table combining host-defined and scheme-requested + variables, annotating which schemes use each variable and whether unit conversion is needed +- Informational outputs only; do not affect the build + +--- + +## 3. ccpp-capgen — detailed analysis + +### 3.1 Command-line arguments + +Entry point: `scripts/ccpp_capgen.py`, `_main_func()`. +Arguments parsed via `framework_env.parse_command_line()` into a `CCPPFrameworkEnv` object: + +| Argument | Required | Purpose | +|---|---|---| +| `--host-files` | yes | `.meta` files or `.txt` indirect file lists | +| `--scheme-files` | yes | Same format | +| `--suites` | yes | `.xml` SDF files or `.txt` lists | +| `--output-root` | no | Directory for generated files | +| `--host-name` | no | If given, generates a host cap | +| `--ccpp-datafile` | no | Path for datatable XML (default: `datatable.xml`) | +| `--kind-type` | no (repeatable) | Fortran kind mappings, syntax `=[:]`. Module defaults to `iso_fortran_env` for ISO_FORTRAN_ENV specs. Examples: `kind_phys=REAL64`, `kind_phys=my_host_kinds:kind_r8`. If omitted, `kind_phys=iso_fortran_env:REAL64` is injected. | +| `--preproc-directives` | no | Fortran preprocessor macros | +| `--use-error-obj` | no | Use error object instead of scalar error variables | +| `--force-overwrite` | no | Always regenerate output | +| `--clean` | no | Remove files listed in datatable and exit | +| `--verbose` | no (repeatable) | Increase log verbosity | + +`CCPPFrameworkEnv` (defined in `framework_env.py`) consolidates all settings into typed +properties and stores a `kind_dict` mapping CCPP kind names to `[kind_spec, module]` pairs. + +### 3.2 Step-by-step execution pipeline + +``` +1. create_file_list() + expand .txt indirect file lists, validate .meta extensions + +2. register_ddts(scheme_files) + pre-scan all scheme .meta files + register DDT type names via register_fortran_ddt_name() + (so the host parser can recognize them as non-intrinsic types) + +3. parse_host_model_files() + for each host .meta file: + metadata_table.parse_metadata_file() → [MetadataTable] + find_associated_fortran_file() → matching .F90 path + parse_fortran_file() → Fortran declarations (via fortran_tools) + check_fortran_against_metadata() → cross-validation (type, kind, rank, intent) + accumulate MetadataSection headers: DDT, module, host types + +4. HostModel(table_dict, host_name, run_env) + process DDT headers: → DDTLibrary + ddt_dict (VarDictionary) + process module/host headers: → main VarDictionary + __var_locations + add ConstituentVarDict synthetically for ccpp_model_constituents_t + +5. API(sdfs, host_model, scheme_headers, run_env) + for each SDF XML: + Suite construction: + auto-create 5 phase groups: register, initialize, timestep_initial, + timestep_final, finalize + parse elements → Group objects (RUN_PHASE_NAME) + parse / tags → Scheme objects in full-phase groups + Suite.analyze(host_model, scheme_library, ddt_library, run_env): + Group.analyze() → Scheme.analyze(): + for each scheme argument: + VarDictionary.find_variable() [scope chain search] + Var.compatible() [→ VarCompatObj with transformations] + loop dim substitution for _run phase + register constituent if constituent=True + variable promotion: group outputs → suite level if needed by later group + +6. ccpp_api.write(outdir, run_env) + suite cap .F90 per suite + group caps (embedded or separate) + host cap .F90 (if --host-name given) + ccpp_kinds.F90 + +7. generate_ccpp_datatable() → datatable.xml +``` + +### 3.3 Object hierarchy + +``` +API (ccpp_suite.py) + └── Suite (extends VarDictionary) [one per SDF XML] + parent → ConstituentVarDict (extends VarDictionary) + parent → API + ├── Group (suite_objects.py, extends VarDictionary) [one per ] + │ call_list: CallList (extends VarDictionary) + │ ├── Subcycle (suite_objects.py) + │ │ └── Scheme (suite_objects.py, extends SuiteObject) + │ └── Scheme (for full-phase groups: init, register, etc.) + └── (auto groups: register, initialize, timestep_initial, + timestep_final, finalize) + +HostModel (host_model.py, extends VarDictionary) + ├── ddt_lib: DDTLibrary + │ └── {ddt_name → MetadataSection} + ├── ddt_dict: VarDictionary (all DDT field variables, expanded) + └── loop_vars: VarDictionary (run-time dimension variables) + +VarDictionary (metavar.py) + ├── {standard_name → Var} + └── parent_dict → VarDictionary ← scope chain for find_variable() + +Var (metavar.py) + └── __prop_dict: {property_name → validated_value} + +VarDDT (ddt_library.py, extends Var) + └── __field: Var | VarDDT ← recursive DDT traversal chain +``` + +### 3.4 Variable matching — scope-chain and VarCompatObj + +Unlike prebuild's single batch `compare_metadata()`, capgen performs incremental, +scope-aware matching during the suite analysis phase. + +For each scheme argument in `Scheme.analyze()`: +1. `VarDictionary.find_variable(standard_name)` — searches scope chain: + local group dict → suite dict → ConstituentVarDict → host model dict +2. `Var.compatible(other, run_env)` returns a `VarCompatObj` — not a bool. + `VarCompatObj` carries: + - Whether the variables are equivalent (no transformation needed) + - Whether they are compatible with transformations (unit conversion, dimension + substitution, `top_at_one` flip) + - The reason for any incompatibility (for error messages) +3. For `_run` phase: `horizontal_dimension` is automatically substituted with + `horizontal_loop_begin:horizontal_loop_end` +4. For `constituent = True` variables: auto-registered in `ConstituentVarDict`; + allocation/management code is generated +5. Variable promotion: if a Group produces a variable needed by a later Group, it is + promoted to Suite-level scope + +`VarCompatObj` compatibility considers: +- Type equality +- Kind equality (with ISO kind aliases) +- Units compatibility (triggers unit conversion if compatible) +- Dimension substitutability (horizontal loop vs. full dimension, vertical extent) +- `top_at_one` orientation (triggers flip if needed) +- `protected` status (cannot be an output if protected) +- `CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` + from `var_props.py` + +### 3.5 `metavar.Var` properties + +`metavar.Var` stores all properties in a validated `__prop_dict`. Properties: + +**Specification properties** (all metadata contexts): + +| Property | Type | Notes | +|---|---|---| +| `local_name` | str | Valid Fortran identifier | +| `standard_name` | str | CF-convention, lowercase+underscores | +| `long_name` | str | Human-readable description | +| `units` | str | Physical units string | +| `dimensions` | list | Dimension standard names or `()` | +| `type` | str | Intrinsic or registered DDT name | +| `kind` | str | Fortran kind parameter | +| `active` | str | Conditional allocation expression | +| `optional` | bool | Whether scheme can handle missing var | +| `protected` | bool | Cannot be written by schemes | +| `allocatable` | bool | Has ALLOCATABLE attribute | +| `state_variable` | bool | Persists across timesteps | +| `persistence` | str | `timestep` or `run` | +| `default_value` | str | Fortran expression | +| `diagnostic_name` | str | Diagnostic output name | +| `target` | bool | Has TARGET attribute | +| `polymorphic` | bool | CLASS(*) type | +| `top_at_one` | bool | Vertical ordering: top at index 1 | + +**Scheme-only properties**: + +| Property | Type | Notes | +|---|---|---| +| `intent` | str | in / out / inout | + +**Constituent properties**: + +| Property | Type | Notes | +|---|---|---| +| `constituent` | bool | Is a CCPP-managed constituent (tracer) | +| `advected` | bool | Is advected by the dynamical core | +| `molar_mass` | float | Molecular weight (positive) | + +### 3.6 Capgen-only features + +**Fortran cross-validation** (`check_fortran_against_metadata()`): +- Parses the actual `.F90` file alongside the `.meta` file +- Checks that every metadata entry matches the real Fortran declaration: + variable count, local_name, type, kind, intent (for schemes), dimension rank/names +- Catches bugs where metadata was updated but the Fortran source was not (or vice versa) + +**State machine** (`ccpp_state_machine.py`, `state_machine.py`): +- `CCPP_STATE_MACH`: a `StateMachine` instance with 6 transitions +- Valid state sequence: `register → uninitialized → initialized → in_time_step` +- Suite caps include a `character(len=16) :: ccpp_suite_state` variable +- State-checking code at the start of each phase function enforces correct call ordering +- `CCPP_STATE_MACH.function_match()` uses compiled regex to identify which CCPP phase + a subroutine name belongs to + +**Constituent variable support** (`constituents.py`): +- `ConstituentVarDict` (extends `VarDictionary`) manages traceable species (tracers) +- When a scheme declares `constituent = True`, `find_variable()` auto-creates the variable +- Allocation code for the constituent array is auto-generated +- Constants: `CONST_DDT_NAME = "ccpp_model_constituents_t"`, + `CONST_PROP_TYPE = "ccpp_constituent_properties_t"` + +**DDT library** (`ddt_library.py`): +- `VarDDT(Var)`: represents a DDT field variable at any nesting level +- Traversal chain: `VarDDT → VarDDT → ... → Var` (innermost is the actual leaf field) +- `DDTLibrary`: dictionary of DDT `MetadataSection` objects +- `collect_ddt_fields()` expands DDT variables into component fields in `ddt_dict` + +**Host cap generation** (`host_cap.py`): +- Generated only when `--host-name` is given +- Produces `_ccpp_cap.F90` +- Subroutines: `_ccpp_physics_(api_vars)` + that call into suite cap functions + +**`ccpp_kinds.F90`**: +- Simple Fortran module `ccpp_kinds`. **Always generated**, even when no `--kind-type` + is supplied (in that case `kind_phys=iso_fortran_env:REAL64` is injected + automatically and an INFO log line is emitted). +- One `use , only: ` line per module (modules sorted; specs deduped per + module). Each kind is then re-exported as + `integer, parameter, public :: = `. +- Supports host-supplied kind modules: `--kind-type kind_phys=my_host_kinds:kind_r8` + emits `use my_host_kinds, only: kind_r8` and + `integer, parameter, public :: kind_phys = kind_r8`. +- Listed in `` of `datatable.xml` (matches original capgen) so + the build system picks it up via `ccpp_datafile.py --ccpp-files`. +- USEd by all generated Fortran files that declare kind-typed variables — the group + cap, the suite types module, and the suite data module. The static API and suite + cap have no kind references and do not USE it. + +**Datatable XML** (`ccpp_datafile.py`): +- Produced after generation; lists all generated files, scheme entries, variable properties, + suite configurations +- Queryable by the build system via `ccpp_datafile.py --suite-files` etc. +- Supports `--clean` workflow: reads the file list, removes all generated files, deletes itself +- `DatatableReport` class provides a programmatic query API + +**In-memory database** (`ccpp_database_obj.py`): +- `CCPPDatabaseObj`: wraps `HostModel` and `API` for programmatic access to capgen results +- Returned when `capgen()` is called with `return_db=True` +- Provides `host_model_dict()`, `suite_list()`, `constituent_dictionary(suite)` + +**Variable tracking tool** (`ccpp_track_variables.py`): +- Standalone diagnostic: traces a specific variable through a suite, showing which schemes + use it and with what intent +- Uses prebuild's `import_config` and capgen's `Suite`/`parse_metadata_file` together + +**Fortran-to-metadata tool** (`ccpp_fortran_to_metadata.py`): +- Standalone utility: parses annotated Fortran source files and generates skeleton `.meta` + files — used to bootstrap new scheme metadata + +--- + +## 4. Shared infrastructure + +### 4.1 Module sharing map + +| Module | Used by prebuild | Used by capgen | Notes | +|---|---|---|---| +| `metadata_parser.py` | yes | partial | **Bridge module**: calls capgen's parser, returns mkcap.Var | +| `metadata_table.py` | via bridge | yes (primary) | Native `.meta` format parser | +| `metavar.py` | no | yes | Primary `Var` class, `VarDictionary` | +| `var_props.py` | no | yes | `VariableProperty`, `VarCompatObj`, dimension constants | +| `mkcap.py` | yes | no | `mkcap.Var` class + build-snippet writers | +| `mkstatic.py` | yes | no | Suite/Group/API Fortran generators | +| `mkdoc.py` | yes | no | HTML/LaTeX documentation generators | +| `common.py` | yes | partial | `CCPP_STAGES`, container encoding | +| `framework_env.py` | dummy instance | yes (primary) | `CCPPFrameworkEnv` | +| `file_utils.py` | no | yes | `create_file_list`, `move_modified_files` | +| `code_block.py` | no | yes | Structured Fortran output | +| `ddt_library.py` | no | yes | `DDTLibrary`, `VarDDT` | +| `host_model.py` | no | yes | `HostModel` class | +| `host_cap.py` | no | yes | Host cap generation | +| `ccpp_suite.py` | no | yes | `Suite`, `API` classes | +| `suite_objects.py` | no | yes | `Scheme`, `Group`, `Subcycle`, `CallList` | +| `constituents.py` | no | yes | `ConstituentVarDict` | +| `ccpp_datafile.py` | no | yes | Datatable XML | +| `ccpp_database_obj.py` | no | yes | `CCPPDatabaseObj` | +| `ccpp_state_machine.py` | no | yes | `CCPP_STATE_MACH` | +| `state_machine.py` | no | yes | `StateMachine` base class | +| `ccpp_fortran_to_metadata.py` | no | yes | Fortran→metadata bootstrap tool | +| `ccpp_track_variables.py` | partial | partial | Uses both worlds | + +**The key architectural debt**: `metadata_parser.py` is a prebuild module that internally +calls capgen's `metadata_table.parse_metadata_file()` and converts results to `mkcap.Var` +objects. This creates a one-way dependency (prebuild → capgen's parser infrastructure) +while presenting a prebuild-style API to `ccpp_prebuild.py`. It exists only because +prebuild predates the `.meta` format. + +### 4.2 The `.meta` file format + +The `.meta` format is the native format for capgen and the expected format for all new +scheme development. The Fortran source file contains a comment hook pointing to the `.meta` +file: + +```fortran +!! \section arg_table_scheme_name_run Argument Table +!! \htmlinclude scheme_name_run.html +``` + +The `.meta` file itself uses an INI-style format: + +```ini +[ccpp-table-properties] + name = scheme_name + type = scheme + source_path = ../src + dependencies_path = ../some/path + dependencies = utility_module.F90, another.F90 + +[ccpp-arg-table] + name = scheme_name_run + type = scheme +[ im ] + standard_name = horizontal_loop_extent + long_name = horizontal loop extent + units = count + type = integer + dimensions = () + intent = in +[ dz ] + standard_name = layer_thickness + long_name = thickness of each model layer + units = m + type = real + kind = kind_phys + dimensions = (horizontal_loop_extent, vertical_layer_dimension) + intent = in +``` + +Multiple `[ccpp-arg-table]` sections are allowed in a scheme file (one per phase: +`_init`, `_run`, `_finalize`, `_timestep_init`, `_timestep_finalize`). +Singleton tables (DDT, module, host) allow only one section. + +The three table-level properties in `[ccpp-table-properties]` that carry path information: + +| Property | Purpose | Resolution | +|---|---|---| +| `source_path` | Relative path from the `.meta` file's directory to the directory containing the corresponding `.F90` Fortran source file | `os.path.normpath(os.path.join(meta_dir, source_path))`. Defaults to `meta_dir` when absent. | +| `dependencies_path` | Optional subdirectory relative to `meta_dir`; used as the base directory for resolving entries in `dependencies` | `os.path.normpath(os.path.join(meta_dir, dependencies_path))`. Defaults to `meta_dir` when absent. | +| `dependencies` | Comma-separated list of dependency file names or relative paths | Each entry resolved via `os.path.normpath(os.path.join(dep_base, entry))` where `dep_base` is the resolved `dependencies_path`. The value `none` is ignored. | + +**Implementation note — `flush_table_props` pattern:** The INI parser processes the +`[ccpp-table-properties]` and `[ccpp-arg-table]` headers in one streaming pass. Extra +table-level keys (`source_path`, `dependencies_path`, `dependencies`) are collected in a +`pending_props` dict alongside `name` and `type`. The parser must apply these properties +to the `MetadataTable` object — via a `flush_table_props()` call — at every +state-transition point (first `[ccpp-arg-table]` header, next `[ccpp-table-properties]` +header, and end-of-file) **before** resetting `pending_props`. Without this, the extra +properties are silently discarded. + +### 4.3 Variable property validation (`var_props.py`) + +`VariableProperty` encapsulates a single metadata property with its name, Python type, +optionality, default, valid-value constraints, and a check function. Check functions used: + +| Checker | What it validates | +|---|---| +| `check_local_name` | Valid Fortran identifier | +| `check_cf_standard_name` | Lowercase, underscores, alphanumeric only | +| `check_fortran_type` | Intrinsic type or registered DDT name | +| `check_units` | Valid unit string (normalizes `+` in exponents) | +| `check_dimensions` | Valid dimension specification | +| `check_default_value` | Valid Fortran expression | +| `check_molar_mass` | Positive float (for constituents) | + +`CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` +in `var_props.py` define the recognized dimension forms and the run-time substitution +map (e.g., `horizontal_dimension → horizontal_loop_begin:horizontal_loop_end`). + +--- + +## 5. Feature comparison + +| Feature | prebuild | capgen | Notes | +|---|---|---|---| +| **Input formats** | | | | +| Native `.meta` format | via bridge | yes | | +| Old pipe-delimited format | deprecated warn | not supported | | +| **Parsing and validation** | | | | +| Fortran source cross-validation | no | yes | capgen parses actual .F90 to cross-check | +| Preprocessor directive support | no | yes | `--preproc-directives` | +| **Variable handling** | | | | +| Variable data class | `mkcap.Var` (flat attrs) | `metavar.Var` (validated prop dict) | | +| Scope-chain variable search | no | yes | group→suite→constituent→host | +| Variable promotion group→suite | no | yes | | +| Unit conversion | yes | yes | | +| Optional/active variables | yes (fully) | yes | both: local pointer + conditional ASSOCIATE | +| DDT library (first-class) | no | yes | `VarDDT` recursive chain | +| **Suite and cap generation** | | | | +| Suite definition (SDF XML) | yes | yes | Same XML format | +| Subcycle loops | yes | yes | | +| State machine in generated caps | no | yes | Runtime state enforcement | +| Static API module (dispatch switch) | yes | no | `ccpp_static_api.F90` | +| Host cap generation | no | yes | `_ccpp_cap.F90` | +| `ccpp_kinds.F90` | no | yes | | +| **Constituent/tracer support** | | | | +| Constituent variable management | no | yes | Auto-allocation, `ConstituentVarDict` | +| **Build system output** | | | | +| CMake/Makefile file-list snippets | yes | no | Six snippet files | +| Datatable XML (queryable) | no | yes | `ccpp_datafile.py` | +| Clean via datatable | no | yes | | +| **Documentation** | | | | +| HTML variable table | yes | no (stub, raises error) | `mkdoc.metadata_to_html` | +| LaTeX variable table | yes | no | `mkdoc.metadata_to_latex` | +| **Developer tools** | | | | +| Variable tracking diagnostic | yes | no | `ccpp_track_variables.py` | +| Fortran-to-metadata bootstrap | no | yes | `ccpp_fortran_to_metadata.py` | +| **Runtime API** | | | | +| In-memory database object | no | yes | `CCPPDatabaseObj` | +| **Debug / developer aids** | | | | +| Debug array-size checks in caps | yes (`--debug`) | no | | +| Namespace suffix for API name | yes (`--namespace`) | no | | +| **Configuration** | | | | +| Config mechanism | Python module (flexible) | CLI args only | | + +**Known gaps and corrections:** + +- Capgen's `--generate-docfiles` is declared in the CLI but raises + `CCPPError("not yet supported")` — documentation generation is unimplemented. +- Prebuild handles `TYPEDEFS_NEW_METADATA` for mixed old/new metadata deployments; + capgen has no equivalent because it only accepts the new format. +- Capgen validates Fortran source against metadata; prebuild trusts metadata and never + reads Fortran code. +- Capgen has no `--namespace` equivalent for the generated API module name. +- Capgen's `CCPPDatabaseObj` and datatable XML allow programmatic querying; prebuild + has no equivalent. +- Prebuild's static API pattern (single Fortran module with runtime dispatch) is absent + from capgen, which uses a different host-cap integration model. +- **Capgen cannot pass DDTs to group caps** — it passes everything as flat fields. + Despite considerable effort by multiple developers, this has not been fixed. This is + the primary reason capgen is being abandoned. +- **Capgen does not support multiple model instances in memory** (ensemble approach). + Prebuild's `initialized(200)` array handles this correctly. +- **Capgen does not own or allocate any data.** Wait — this is a prebuild characteristic. + Capgen *does* allocate data for physics-internal variables (variables used only within + the physics, not provided by the host model) at the suite level. Prebuild requires the + host model to provide and own all data, including any physics-internal scratch space. + +--- + +## 6. Build system integration + +### 6.1 How a host model invokes ccpp-prebuild + +Direct call (as in the test suite): +```bash +python ../../scripts/ccpp_prebuild.py \ + --config=ccpp_prebuild_config.py \ + --builddir=build \ + --suites=suite_A,suite_B \ + [--debug] [--namespace mymodel] +``` + +Typical CMake integration: +```cmake +# Run prebuild at configure time +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_prebuild.py + --config=${HOST_CCPP_PREBUILD_CONFIG} + --builddir=${CMAKE_CURRENT_BINARY_DIR} + --suites=${CCPP_SUITES} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + RESULT_VARIABLE PREBUILD_RESULT +) +if(NOT PREBUILD_RESULT EQUAL 0) + message(FATAL_ERROR "ccpp_prebuild.py failed") +endif() + +# Consume the generated snippet files +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) # → ${CAPS} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) # → ${SCHEMES} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) # → ${API} + +add_library(ccpp_physics OBJECT ${CAPS} ${SCHEMES} ${API}) +``` + +### 6.2 How a host model invokes ccpp-capgen + +Direct call: +```bash +python scripts/ccpp_capgen.py \ + --host-files host_data.meta,host_model.meta \ + --scheme-files scheme1.meta,scheme2.meta \ + --suites suite_A.xml,suite_B.xml \ + --output-root ${BUILD_DIR}/ccpp \ + --host-name my_host \ + --kind-type kind_phys=REAL64 \ + --ccpp-datafile ${BUILD_DIR}/ccpp/datatable.xml +``` + +Typical CMake integration: +```cmake +# Run capgen at configure time +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_capgen.py + --host-files ${HOST_META_FILES} + --scheme-files ${SCHEME_META_FILES} + --suites ${SUITE_SDFS} + --output-root ${CMAKE_CURRENT_BINARY_DIR}/ccpp + --host-name ${HOST_MODEL_NAME} + --ccpp-datafile ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + RESULT_VARIABLE CAPGEN_RESULT +) + +# Query the datatable for generated file lists +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py + ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + --suite-files + OUTPUT_VARIABLE SUITE_CAPS OUTPUT_STRIP_TRAILING_WHITESPACE +) +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py + ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + --host-files + OUTPUT_VARIABLE HOST_CAP OUTPUT_STRIP_TRAILING_WHITESPACE +) + +add_library(ccpp_physics OBJECT ${SUITE_CAPS} ${HOST_CAP}) +``` + +### 6.3 Available datatable query flags + +``` +--host-files → generated host cap .F90 files +--suite-files → generated suite cap .F90 files +--utility-files → generated utility .F90 files (e.g. ccpp_kinds.F90) +--ccpp-files → all generated .F90 files +--process-list → physics process types in the suite +--module-list → Fortran module names needed +--dependencies → scheme dependency files +--suite-list → configured suite names +--required-variables → variables required by all suites +--input-variables → input-only variables for a suite +--output-variables → output variables for a suite +--host-variables → variables provided by the host model +``` + +--- + +## 7. Key architectural differences + +### 7.1 Data model + +| Dimension | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| Variable representation | `mkcap.Var` with plain Python attributes | `metavar.Var` with validated `__prop_dict` | +| Variable storage | Two flat `OrderedDict`s | Scope-chain `VarDictionary` tree | +| Container encoding | Encoded string: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | Explicit class hierarchy | +| DDT handling | Encoded as string in `local_name`; helper regexes to extract | First-class `VarDDT` recursive chain | +| Variable matching | One batch `compare_metadata()` call | Incremental during suite analysis | +| Matching result | `bool` + side effects on `.target` / `.actions` | Rich `VarCompatObj` with transformation info | +| **Cap argument style** | **DDTs passed to group caps** | **Flat fields passed to group caps** | +| Subsetting location | At the scheme call site inside the group cap | Done at a higher level, before group cap | +| Data ownership | Host model owns all data including physics-internal | Capgen allocates physics-internal suite-level data | +| Multiple model instances | Yes — `initialized(200)` array, one flag per instance | No | +| Optional variable handling | Local pointer, conditionally associated | Same mechanism, but blocked by flat-field issue | + +### 7.2 Error handling + +| Aspect | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| Style | `(success, result)` tuples + `logging.error()` | `CCPPError` / `ParseInternalError` exceptions | +| Collection | Errors accumulate via `logging`; `main()` checks success | Raised immediately at point of detection | +| Location info | Filename from context; line numbers sometimes | `ParseContext` objects with file + line number | +| User errors vs bugs | Not distinguished | `CCPPError` (user) vs `ParseInternalError` (programmer) | + +### 7.3 Extensibility + +| Aspect | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| New metadata property | Add to `VALID_ITEMS` dict + `mkcap.Var` attribute | Add one `VariableProperty` entry + checker fn | +| New CCPP phase | Update `CCPP_STAGES` + regenerate static API template | Add one transition tuple to `CCPP_STATE_MACH` | +| New compatibility rule | Modify `var.compatible()` in `mkcap.py` | Extend `VarCompatObj` in `var_props.py` | +| New host model | Write a new Python config file | New `.meta` files + CLI invocation | + +### 7.4 Performance + +Prebuild generates caps for multiple suites in seconds. Capgen, on the same suite set +with the same physics, takes more than 10 minutes. Two independent causes: + +**Cause 1 — Repeated scope-chain traversal.** Every variable lookup in capgen traverses +a five-level `VarDictionary` parent chain (group → suite → constituent dict → host model +→ DDT dict) for every scheme argument in every group in every suite. Prebuild's +`compare_metadata()` does one flat dict lookup per standard name, once, and caches the +result in `var.container` and `var.target`. All subsequent use during Fortran generation +reads these cached attributes directly. + +**Cause 2 — Flat-field cap arguments.** This is likely the dominant cost. Capgen resolves +every scheme argument down to its individual flat field, generates a `use` statement and +an explicit argument for each one, and emits them in the generated Fortran. A DDT with +200 fields becomes 200 individual argument declarations, 200 `use` statements, and 200 +argument positions in the scheme call. Prebuild passes the DDT itself — one argument, +one `use` statement — and then subsets at the call site. + +**Consequence for correctness.** Passing flat fields in capgen also breaks optional +variable handling under Fortran compiler debugging flags. When a field inside a DDT is +conditionally allocated (optional), passing it as a flat field requires dereferencing the +DDT to extract the field — which the compiler will flag as an error if debugging is on +and the field happens to be unallocated. Prebuild avoids this entirely by passing the +DDT and using a local pointer at the scheme call site. + +### 7.5 Team comprehension and maintainability + +This is the critical real-world difference. `ccpp-prebuild` is understood by the whole +team because it is procedural Python: you can read `ccpp_prebuild.py` top-to-bottom and +follow what happens. The data structures are flat dicts; the control flow is linear. + +`ccpp-capgen` has a five-level class hierarchy, scope-chain dictionary lookups, +`VarCompatObj` carrying transformation state, `ConstituentVarDict` as a pluggable +scope-chain node, and a `StateMachine` with regex-based dispatch. No remaining team +member fully understands all of it. Development is extremely slow and risky. + +The failed effort to make capgen pass DDTs instead of flat fields is the concrete proof +point: three developers spent considerable time and could not fix it without fully +understanding the interplay between `VarDDT`, `DDTLibrary`, `VarDictionary` scope chains, +and the Fortran writer. This is the proximate reason for the redesign. + +--- + +## 8. Design considerations for the redesign + +The following observations from this analysis should inform the redesign: + +### 8.1 What to keep from prebuild +- Procedural, top-down control flow — easy to read and debug +- Config file as a Python module — extremely flexible without adding CLI arguments +- The static API pattern (`ccpp_static_api.F90` with runtime suite/group dispatch) — + proven, simple integration for the host model +- **DDT arguments in group caps** — pass DDTs, not flat fields; this is the core correctness + and performance requirement +- **Subsetting at the scheme call site** — group caps always receive full data; loop-bound + application and fixed-index extraction happen in the individual scheme call expressions + or via a local variable/pointer declared just before the call +- **Optional variable pattern** — local pointer declared in the group cap, conditionally + associated based on the `active` expression, then passed to the scheme; this is safe + under all compiler debugging modes +- The `initialized(N)` per-instance tracking — handles multiple simultaneous model + instances in memory (ensemble approach); `N` is the max number of instances +- **Framework-owned data needs a simpler design** — capgen's variable promotion and + `ConstituentVarDict` scope-chain approach is too complex; a cleaner mechanism for + framework-allocated physics-internal data is needed (to be designed) +- HTML and LaTeX documentation generation +- The six CMake/Makefile/shell snippet output files — simple and direct (can be revisited) + +### 8.2 What to keep from capgen +- Native `.meta` file parsing (eliminate the `metadata_parser.py` bridge entirely) +- Fortran source cross-validation (`check_fortran_against_metadata()`) — catches real bugs +- Rich compatibility reporting (`VarCompatObj`-style) — better error messages +- `ccpp_kinds.F90` generation — important for portability +- Datatable XML as output accounting (strictly better than six include files) +- `--preproc-directives` support +- Constituent variable support (needed for CAM-SIMA) +- State machine enforcement (optional feature, but architecturally clean) + +### 8.3 What to eliminate +- The `mkcap.Var` / `metavar.Var` duality — one variable class, natively reading `.meta` +- The `metadata_parser.py` bridge module — it exists only because of the old format +- The scope-chain `VarDictionary` hierarchy — replace with flat, explicit lookup: + one host dict, one scheme dict; no parent-chain traversal +- The five-level class inheritance (Suite → VarDictionary → ParseSource → ...) +- `ConstituentVarDict` as a scope-chain node — a simple explicit constituent registry suffices +- Capgen's variable promotion (group → suite level) — this complexity exists only because + capgen allocates physics-internal data; if the host always owns all data, promotion + is unnecessary +- Capgen's flat-field cap generation — DDT arguments must be the foundation + +### 8.4 Framework-owned data — open design question + +Capgen's variable promotion mechanism (promoting a variable from group scope to suite scope +when a later group needs it) and the `ConstituentVarDict` complexity exist because capgen +allocates and manages physics-internal data — variables used only within the physics, +not visible to the host model. This capability is **wanted** in the redesign: the host +model should not have to declare and own scratch variables that are purely internal to the +physics. + +The problem is not the concept but the implementation. Capgen's approach — weaving +framework-allocated variables into the `VarDictionary` scope chain and promoting them +upward — produces the complexity that made capgen unmaintainable. + +**Open question for the redesign:** What is a simpler mechanism for the framework to +allocate, own, and pass physics-internal variables? Candidate approaches (to be evaluated +with real-world examples): + +- A completely separate, flat "framework data" dictionary, distinct from the host variable + lookup, populated during analysis and passed explicitly to the caps as a dedicated + argument (e.g., a framework-managed DDT or allocatable array container). +- A simplified promotion concept: variables are statically promoted to the widest scope + that needs them during the analysis phase, but stored in a simple flat dict rather than + via a scope-chain lookup. +- Constituent variables (tracers) as a special sub-case with their own well-defined + allocation interface, separate from generic physics-internal data. + +This question will be revisited once real-world examples clarify how many and what kind of +physics-internal variables actually need to be managed. + +### 8.5 Critical design decisions for the redesign prompt + +1. **DDT cap arguments are non-negotiable.** Group caps must receive DDTs. The entire + subsetting, optional-variable, and performance story depends on this. + +2. **Data ownership**: host-owns-all (prebuild model) vs. generator-allocates-internals + (capgen model). This single decision determines whether variable promotion and + suite-level allocation are needed. + +3. **Integration pattern**: static API (prebuild style, `suite_name` + `group_name` dispatch) + vs. host cap (capgen style, separate host-side Fortran glue). Models currently using + each pattern depend on it. + +4. **Config mechanism**: Python module (prebuild style, flexible) vs. pure CLI + file lists + (capgen style, scriptable). The Python module config is very powerful for complex models. + +5. **DDT member access parsing**: `extract_parents_and_indices_from_local_name()` and + `extract_dimensions_from_local_name()` in `mkstatic.py` handle expressions like + `Atm(blk_no)%q(:,:,:,graupel_index)`. The redesign needs a clean, explicit design for + parsing and emitting these — not an afterthought regex patch. + +6. **Output accounting**: datatable XML (capgen) is the right answer. The six CMake snippet + files (prebuild) are redundant and harder to extend. + +7. **Multiple model instances**: the redesign must preserve the `initialized(N)` pattern + or an equivalent. The value of `N` may need to be configurable. + +8. **Backward compatibility of generated Fortran interfaces**: real-world model examples + will define exactly which naming conventions, argument orders, and module structures the + host models depend on. + +### 8.6 Implementation decisions made during redesign + +The following decisions were made during implementation of `capgen-ng` and are recorded +here as amendments to the analysis above. + +**State machine parameters are local to each generated group cap module.** +The original redesign prompt described the integer state constants as coming from a +shared framework library module. In practice they are generated as `private` named +parameters directly inside each group cap module: + +```fortran +integer, parameter, private :: CCPP_GROUP_UNINITIALIZED = 0 +integer, parameter, private :: CCPP_GROUP_INITIALIZED = 1 +integer, parameter, private :: CCPP_GROUP_IN_TIMESTEP = 2 +``` + +This keeps generated files self-contained — no implicit dependency on a framework +runtime library at the caps level. The values are replicated across all generated group +cap files, but the names are the contract. + +**`source_path` is used by the validator, not the generator.** +The generator trusts metadata and never opens Fortran source files. `source_path` is +meaningful only to the standalone validator tool, which uses it to auto-discover the +`.F90` file paired with each `.meta` file (same base name, different directory). + +**`dependencies` paths are written to `datatable.xml`.** +The resolved absolute paths from each scheme's `dependencies` table-level property are +collected and written to the `` section of `datatable.xml`, sorted and +deduplicated. The CMake build system reads these via `ccpp_datafile.py` to add external +dependency files to the build graph. + +**Optional variable (pointer wrapper) implementation decisions.** +Optional arguments (Case 2 and Case 4) use per-suite Fortran derived types for pointer +wrappers. All unique `(type, kind, rank)` combinations needed by any optional arg across +all groups in a suite are collected and written to `ccpp__types.F90`. Each type +name is generated as `{type}_{kind}_rank{N}_ptr_type` (e.g. `real_kind_phys_rank1_ptr_type`). +Group cap modules `USE` this file. The types file is omitted entirely when no optional +args exist in the suite. The active condition for a pointer assignment is inherited from +the **host variable's** `active` attribute when the scheme itself specifies no `active`. + +**Character length (`len=N` / `len=*`) rules.** +Character kind declarations follow specific compatibility rules enforced by the resolver: + +- `len=*` in a **scheme** is always compatible with any host `len=` — assumed-length + dummy arguments accept any host-declared length. No transform is generated. +- Matching specific `len=N` in both host and scheme requires no transform (naturally equal). +- Mismatched specific lengths (`len=512` host vs `len=128` scheme) are a **metadata error**; + the scheme must declare `len=*` or match the defining metadata exactly. +- `len=*` in the **host** with a specific `len=N` in the scheme is also an error. + +The resolver raises `CCPPError` for the illegal cases. No kind transform is ever generated +for character variables — lengths are a Fortran compatibility constraint, not a unit conversion. + +**`source_path` is used by the validator, not the generator.** +The group cap's `state_alloc` subroutine always takes `number_of_instances` as an +explicit `intent(in)` integer argument — it never USEs any host module to obtain it. +The call chain is: `ccpp_init` → `_init` → each group's `state_alloc`. At each +level the argument is conditional: + +- **Multi-instance host** (`number_of_instances` declared in host metadata with local + name e.g. `ninstances`): + - `ccpp_init(suite_name, ninstances, errmsg, errflg)` — static API receives it + - `_init(ninstances, errmsg, errflg)` — suite cap threads it through + - `state_alloc(ninstances, errmsg, errflg)` — group cap allocates array of that size +- **Single-instance host** (no `number_of_instances` in host metadata): + - All three signatures omit the argument + - `state_alloc(1, errmsg, errflg)` — the literal `1` is passed + +State array **indexing** uses `instance_number`'s local name (e.g. `inst_num`) from +the control metadata. For single-instance hosts the literal `1` is used. `instance_number` +is injected into the group cap's `_init` and `_final` subroutine signatures even when no +scheme in those phases uses it — the state guard and state transition require it: + +```fortran +subroutine ccpp___init(inst_num, ...) + if (ccpp_group_state(inst_num) >= CCPP_GROUP_INITIALIZED) return + ! ... scheme _init calls ... + ccpp_group_state(inst_num) = CCPP_GROUP_INITIALIZED +``` + +This injection does **not** happen for `_run`, `_timestep_init`, or `_timestep_final` +unless a scheme in those phases explicitly requests `instance_number`. The suite cap's +`_physics_init` and `_physics_final` dispatch subroutines similarly pass +`instance_number` to the group cap calls when the host provides it. + +**Control variable validation — flat unconditional required set.** +All required control variables (`suite_name`, `horizontal_loop_begin`, `horizontal_loop_end`, +`thread_number`, `number_of_threads`, `number_of_physics_threads`, `ccpp_error_code`, +`ccpp_error_message`, `instance_number`) are unconditional — every host must declare all +of them. Single-threaded or single-instance models pass `1` or `''` for any they don't +actively use. The generator validates the complete set after parsing host metadata, +collects all missing-variable errors together, and halts before emitting any code. +`instance_number` in particular is NOT conditional on `instance_dimension` usage — +it is always required. + +**`group_name` is conditionally included, not in the required set.** +`group_name` is included in the static API signature only if the host declares it in +their `type=control` table. When absent, the cap calls all groups unconditionally. The +generator warns (not errors) if `group_name` is absent and any suite has multiple groups. +When present: a required (non-optional) character argument; `''` or `'all'` calls all +groups in order; any other value dispatches to the named group only. + +**`horizontal_loop_extent` eliminated; schemes always use `horizontal_dimension`.** +Scheme metadata always declares `horizontal_dimension` as the horizontal extent +dimension, regardless of phase. There is no `horizontal_loop_extent` standard name in +the new design. The distinction between run-phase chunked processing and full-domain +init/final processing is handled entirely at the host level — the host passes actual +chunk bounds to `ccpp_physics_run` and `1`/`ncols` to all other phases. The cap always +generates `(horizontal_loop_begin:horizontal_loop_end)` for scheme call-site array +slices. For suite-owned array allocation sizing, `horizontal_dimension` from the host +`type=host` table (module USE) is used directly. This separation means allocation +correctness does not depend on the host passing any particular control variable values. + +**Uniform signature across all `ccpp_physics_*` entry points.** +All five physics entry points (`ccpp_physics_init`, `ccpp_physics_timestep_init`, +`ccpp_physics_run`, `ccpp_physics_timestep_final`, `ccpp_physics_final`) share the +same control argument set. No per-phase signature variations. `horizontal_loop_begin` +and `horizontal_loop_end` are in scope for all phases — a `_init` scheme that declares +`horizontal_dimension` correctly receives `(lb:ub)` slicing just as a `_run` scheme +would, with the host responsible for passing the right values. + +--- + +## 9. Real-world example: CCPP Single Column Model (SCM) + +*Source:* `EXT/ccpp-scm/` — uses `ccpp-prebuild`. + +The SCM is a horizontally degenerate model (always `im = 1`, no OpenMP threading) but +it compiles the largest set of suites in the CCPP ecosystem, making it the most complete +real-world picture of what prebuild must handle. + +**Scale:** 63 suites, 257 scheme files (137 scheme entries in config, many containing +multiple modules), 300 generated cap files, ~1,200+ host model variables, ~550 optional +(conditionally active) variables. + +--- + +### 9.1 The `TYPEDEFS_NEW_METADATA` bridge — the DDT accessor map + +This is the most important SCM-specific configuration. It maps each DDT type name to the +Fortran expression used to access an instance of that type from the host model's top-level +scope. It is what allows the code generator to convert a `local_name` like `tgrs` (declared +inside `GFS_statein_type`) into the cap argument expression +`physics%Statein%tgrs(...)`. + +```python +TYPEDEFS_NEW_METADATA = { + 'GFS_typedefs': { + 'GFS_diag_type' : 'physics%Diag', + 'GFS_control_type' : 'physics%Model', + 'GFS_cldprop_type' : 'physics%Cldprop', + 'GFS_tbd_type' : 'physics%Tbd', + 'GFS_sfcprop_type' : 'physics%Sfcprop', + 'GFS_coupling_type': 'physics%Coupling', + 'GFS_statein_type' : 'physics%Statein', + 'GFS_radtend_type' : 'physics%Radtend', + 'GFS_grid_type' : 'physics%Grid', + 'GFS_stateout_type': 'physics%Stateout', + 'GFS_typedefs' : '', + }, + 'CCPP_typedefs': { + 'GFS_interstitial_type': 'physics%Interstitial(cdata%thrd_no)', + 'CCPP_typedefs' : '', + }, + 'scm_type_defs': { + 'physics_type': 'physics', + 'scm_type_defs': '', + }, + 'ccpp_types': { + 'ccpp_t' : 'cdata', + 'ccpp_types': '', + 'MPI_Comm': '', + }, + # ... plus 8 more entries for physics-side modules (machine, radsw_param, etc.) +} +``` + +**How it works:** For a variable with `local_name = tgrs` declared in `GFS_statein_type`, +the generator looks up `'GFS_statein_type'` in the map, finds `'physics%Statein'`, and +constructs the target as `physics%Statein%tgrs`. For the thread-indexed interstitial DDT, +`physics%Interstitial(cdata%thrd_no)%` is produced automatically. + +This dictionary is the **entire** mechanism by which the prebuild bridge converts flat +metadata into correct DDT-member accessor expressions. It is a hand-maintained workaround +that the redesigned generator must **eliminate**: all information needed to derive these +accessor expressions is already present in the CCPP metadata, provided the metadata storage +model is designed correctly to capture the DDT hierarchy and instance/thread indexing. + +--- + +### 9.2 Host model DDT structure + +``` +! Module-level variables accessible globally: +physics (type physics_type, from module scm_type_defs) +cdata (type ccpp_t, from module ccpp_types) +one (integer parameter = 1, from module ccpp_types) + +! physics_type contains: +physics%Model → GFS_control_type (control parameters: integers, logicals, 1D arrays) +physics%Statein → GFS_statein_type (input atmospheric state: 2D/3D real arrays) +physics%Stateout → GFS_stateout_type (output tendencies) +physics%Sfcprop → GFS_sfcprop_type (surface properties: 2D real arrays) +physics%Coupling → GFS_coupling_type (coupling fields) +physics%Grid → GFS_grid_type (grid geometry) +physics%Tbd → GFS_tbd_type (to-be-determined / miscellaneous) +physics%Cldprop → GFS_cldprop_type (cloud microphysics properties) +physics%Radtend → GFS_radtend_type (radiation tendencies) +physics%Diag → GFS_diag_type (diagnostic output arrays) +physics%Interstitial(1:thrd_cnt) → GFS_interstitial_type (per-thread scratch space) +``` + +The interstitial DDT is an array indexed by thread number. Even though the SCM is +single-threaded, all caps use `physics%Interstitial(cdata%thrd_no)` (i.e., index 1). +This is the pattern that enables OpenMP parallelism in the full UFS models. + +--- + +### 9.3 The horizontal dimension in the SCM + +The SCM uses a **chunked** horizontal loop even though `im = 1`. The chunk mechanism is: + +```fortran +chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) +chunk_end = physics%Model%chunk_end(cdata%chunk_no) +``` + +All 2D and 3D array slice expressions in caps use this pattern: +```fortran +physics%Statein%tgrs(chunk_begin:chunk_end, one:levs) +``` + +In the SCM, `chunk_begin = chunk_end = 1` always, but the pattern is general enough for +multi-column models. The `one` lower bound (a named integer constant = 1) is a framework +convention used consistently throughout all caps. + +--- + +### 9.4 Four categories of local variables in group caps + +Every group cap generates four categories of local variable declarations before its scheme +calls: + +**Category 1 — Loop bounds and scalars (always present):** +```fortran +integer :: chunk_begin, chunk_end +integer :: levs +chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) +chunk_end = physics%Model%chunk_end(cdata%chunk_no) +levs = physics%Model%levs +``` + +**Category 2 — Fixed-index extractions (tracer indices, surface-level slices):** + +For a tracer `qgrs(:,:,ntqv)`: +```fortran +! No local variable declared — the expression is used inline at the call site: +call scheme_run(qv = physics%Statein%qgrs(chunk_begin:chunk_end, one:levs, physics%Model%ntqv), ...) +``` + +For a surface-level slice `prsi(:,1)`: +```fortran +call scheme_run(prsi_sfc = physics%Statein%prsi(chunk_begin:chunk_end, 1), ...) +``` + +The fixed index may be a literal integer (`1`) or a runtime scalar variable from a DDT +field (`physics%Model%ntqv`). Both are inlined at the call site. + +**Category 3 — Optional variable pointer arrays:** + +One pointer-array type and one pointer-array variable are declared for each optional +variable. They are dimensioned by thread count: +```fortran +type :: real_kind_phys_rank2_ptr_arr_type + real(kind_phys), dimension(:,:), pointer :: p => null() +end type real_kind_phys_rank2_ptr_arr_type +type(real_kind_phys_rank2_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array +``` + +Before each scheme call that uses the variable, the condition is evaluated and the pointer +either associated or left null: +```fortran +if (physics%Model%lndp_type /= 0) then + sfc_wts_1_ptr_array(cdata%thrd_no)%p => & + physics%Coupling%sfc_wts(chunk_begin:chunk_end, one:physics%Model%n_var_lndp) +end if +``` + +Passed to the scheme as a keyword argument: +```fortran +call gfs_surface_generic_pre_run(..., sfc_wts=sfc_wts_1_ptr_array(cdata%thrd_no)%p, ...) +``` + +After the call, the pointer is nullified: +```fortran +if (physics%Model%lndp_type /= 0) then + nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) +end if +``` + +**Category 4 — Unit conversion local variables:** + +Not present in the SCM (GFS uses consistent SI units throughout). When present in other +models, a local array is declared, populated before the call, and passed as the argument: +```fortran +real(kind_phys) :: converted_var(chunk_begin:chunk_end) +converted_var(:) = physics%Statein%source_field(chunk_begin:chunk_end) * conversion_factor +call scheme_run(..., target_arg=converted_var, ...) +``` + +--- + +### 9.5 Array size checks + +Every array argument — mandatory or optional — has a size check immediately before the +scheme call. The check uses `size()` and computes the expected size from dimension variables: + +```fortran +! Mandatory variable — outer condition is always .true. +if (.true.) then + if (size(physics%Statein%tgrs(chunk_begin:chunk_end, one:levs)) /= & + (chunk_end-chunk_begin+1)*(levs-one+1)) then + write(cdata%errmsg, '(a,i8,a,i8)') & + 'Detected size mismatch for variable tgrs: expected ', expected, ' but got ', actual + ierr = 1 + return + end if +end if + +! Optional variable — outer condition mirrors the active= expression +if (physics%Model%lndp_type /= 0) then + if (associated(sfc_wts_1_ptr_array(cdata%thrd_no)%p)) then + if (size(sfc_wts_1_ptr_array(cdata%thrd_no)%p) /= expected_size) then + ...error... + end if + end if +end if +``` + +--- + +### 9.6 The `initialized(200)` array and instance management + +```fortran +logical, dimension(200), save :: initialized = .false. +``` + +`cdata%ccpp_instance` is a 1-based integer assigned to each independent CCPP state object. +In an ensemble, each ensemble member gets a different instance number (1–200). The `init_cap` +sets `initialized(cdata%ccpp_instance) = .true.` at the end of successful init. The +`run_cap` checks `if (.not. initialized(cdata%ccpp_instance))` and aborts with an error +if init was never called for that instance. The `final_cap` resets the flag to `.false.`. + +The value 200 is hardcoded — it is the maximum supported number of simultaneous model +instances. This could be made configurable. + +--- + +### 9.7 Suite and group cap hierarchy + +Three-level cap hierarchy: + +``` +ccpp_static_api.F90 (module ccpp_static_api) + → dispatches by suite_name + optional group_name + → owns physics, cdata, constants via module use + → calls suite-level caps: + +ccpp_scm_gfs_v16_cap.F90 (module ccpp_scm_gfs_v16_cap) + → aggregates arguments from all groups + → calls group caps in order per phase: + +ccpp_scm_gfs_v16_time_vary_cap.F90 (module ccpp_scm_gfs_v16_time_vary_cap) +ccpp_scm_gfs_v16_radiation_cap.F90 (module ccpp_scm_gfs_v16_radiation_cap) +ccpp_scm_gfs_v16_phys_ps_cap.F90 (module ccpp_scm_gfs_v16_phys_ps_cap) +ccpp_scm_gfs_v16_phys_ts_cap.F90 (module ccpp_scm_gfs_v16_phys_ts_cap) +``` + +Each level is a pure Fortran module. Argument passing is explicit keyword-argument style +at every level; no implicit global data (except in the static API, which uses `use`). + +--- + +### 9.8 Static API: module-level variable ownership + +The static API module uses all host-model modules and accesses their variables at module +scope. It does **not** take host data as subroutine arguments — instead it fills the +group cap arguments from its own module-use-associated variables: + +```fortran +module ccpp_static_api + use scm_type_defs, only: physics + use ccpp_types, only: cdata, one + use scm_physical_constants, only: con_g, con_pi, con_t0c, ... + use gfs_typedefs, only: ltp + use ccpp_scm_gfs_v16_cap, only: scm_gfs_v16_run_cap, ... + ... +contains + subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) + ! cdata passed in, others accessed from module scope + select case (to_lower(trim(suite_name))) + case ('scm_gfs_v16') + if (present(group_name)) then + select case (to_lower(trim(group_name))) + case ('phys_ps') + ierr = scm_gfs_v16_phys_ps_run_cap(one=one, physics=physics, cdata=cdata, ...) + ... + end select + else + ierr = scm_gfs_v16_run_cap(one=one, physics=physics, cdata=cdata, ...) + end if + case ('scm_gfs_v17_p8') + ... + end select + end subroutine +end module +``` + +This design means the static API file must be recompiled whenever any host-model module +changes (because it `use`s them), and it must be regenerated whenever suites change. +Its location in the **source tree** (not build tree) is a deliberate SCM design choice: +the file is committed to the repository as a generated artifact. + +--- + +### 9.9 Build system + +Prebuild runs at **cmake configure time** via `execute_process()`, before any compilation +starts. This is unusual but simplifies the cmake dependency graph. + +```cmake +execute_process( + COMMAND ccpp/framework/scripts/ccpp_prebuild.py + --config=ccpp/config/ccpp_prebuild_config.py + --suites=${CCPP_SUITES} + --builddir=${CMAKE_CURRENT_BINARY_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../.. + OUTPUT_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.out + ERROR_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.err +) +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_CAPS.cmake) # → ${CAPS} +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_SCHEMES.cmake) # → ${SCHEMES} +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} +include(scm/src/CCPP_STATIC_API.cmake) # → ${API} +``` + +**Suite selection:** If `CCPP_SUITES` is not set by the user, a helper script +`suite_info.py` selects a compiler-appropriate subset. The full set of 63 suites is +used for production; subsets speed up development builds. + +--- + +### 9.10 Observations relevant to the redesign + +1. **`TYPEDEFS_NEW_METADATA` is a workaround that the redesign must eliminate.** The + DDT accessor information (which type lives at which accessor path) can be fully derived + from the CCPP metadata itself, given a well-designed metadata storage model. The + redesign must derive DDT accessor expressions automatically from the metadata rather + than requiring a separate hand-maintained dictionary. This is one of the primary + motivations for the new metadata storage design. + +2. **Three-level cap hierarchy (group → suite → static API) should be preserved.** + It provides clean separation: group caps are independently testable, suite caps + aggregate phases, the static API is the single host-callable entry point. + +3. **The static API's module-level `use` of host data is model-specific.** In models + where host data is not module-level (e.g., passed as subroutine arguments), the + static API pattern changes. The SCM is the simplest case because `physics` and `cdata` + are global module variables. + +4. **Instance and thread indexing are two orthogonal dimensions of host data access.** + Host model data uses two distinct indexing patterns that must be handled correctly: + + - **Regular state data** (Statein, Stateout, Sfcprop, etc.): dimensioned by instance + number — `physics%Statein(ccpp_instance_number)%array(1:horizontal_dimension, 1:vertical_dimension, ...)`. + In models supporting multiple in-memory model instances (ensemble), the top-level + DDT is an array indexed by `cdata%ccpp_instance`. + + - **Interstitial (per-thread scratch) data**: dimensioned by both instance and thread — + `physics%Interstitial(ccpp_instance_number, ccpp_thread_number)%array(1:horizontal_loop_extent, ...)`. + Critically, the horizontal dimension of interstitial arrays is sized to + `horizontal_loop_extent` (one OpenMP thread's chunk), not `horizontal_dimension` + (the full column count). `max_number_of_threads` instances are allocated per model + instance. Interstitial data can only be used during the **run phase** — this is a + known limitation of ccpp-prebuild that the redesign should address or at minimum + preserve explicitly. + +5. **Optional variable pointer arrays dimensioned by thread count** are the current + solution to thread-safe optional variable handling. This pattern is verbose (one + derived type + one array per optional variable per cap function) but correct. + The redesign could simplify this. + +6. **~550 optional variables in this model.** Optional/conditional variables are not + a corner case — they are a first-class feature. The redesign must handle them + efficiently and correctly. + +7. **Array size checks are debug-only and should not appear in the redesign by default.** + In prebuild they are only generated when the `--debug` flag is passed. The redesigned + generator should not produce them in normal mode — out-of-bounds access is caught at + runtime by compiler flags (e.g., `-fcheck=bounds` with gfortran, `-check bounds` with + ifort). The 12,991-line group cap is partly a consequence of generating these checks + unconditionally in the debug mode artifact examined here. + +8. **No unit conversions appear in GFS/SCM.** Unit conversion infrastructure must be + present in the redesign but the GFS physics package is self-consistent in units. + Unit conversions are more relevant for other host models. + +9. **The `one` constant** (integer parameter = 1) is passed as an explicit argument + everywhere and used as the lower bound in all array slices. This is a framework + convention. The redesign should decide whether this convention is preserved or + whether array lower bounds are handled differently. + +10. **Subcycles produce actual Fortran `do` loops inside the generated group cap.** + The loop from `1` to `cdata%loop_max` is generated directly in the cap function, + not left to the host model: + ```fortran + cdata%loop_max = 2 + do cdata%loop_cnt = 1, cdata%loop_max + call scheme_A_run(...) + if (ierr /= 0) return + call scheme_B_run(...) + if (ierr /= 0) return + end do + ``` + `cdata%loop_max` is set at the start of the subcycle block (from the `loop=` attribute + in the SDF XML) and `cdata%loop_cnt` is the current iteration counter, both visible + to schemes via the `ccpp_t` DDT. + +--- + +## 10. Real-world example: CAM-SIMA (capgen) + +*Source:* `EXT/cam-sima/` — uses `ccpp-capgen`. + +CAM-SIMA is the only model currently using capgen. It is still primarily a research model. +Unlike the SCM it uses a full 3D grid with OpenMP parallelism, but exposes host model +data as flat module variables rather than DDTs in the metadata layer. This example reveals +both what capgen can do and where it fundamentally fails. + +**Scale:** 1 suite (`cam7`), 2 run groups (`physics_before_coupler`, `physics_after_coupler`), +~75 scheme calls, 18 host `.meta` files, 893-line host cap, 2865-line suite cap. + +--- + +### 10.1 Suite structure + +`suite_cam7.xml` has two groups and no subcycles: + +| Group | Schemes (approx.) | Purpose | +|---|---|---| +| `physics_before_coupler` | 52 scheme calls | Cloud fraction, energy checks, dry adiabatic adjustment, Zhang-McFarlane deep convection full cycle, constituent tendency application | +| `physics_after_coupler` | ~20 scheme calls | Tropopause diagnostics, gravity wave drag (7 parameterizations + diagnostics), tendency application, energy consistency | + +CCPP phases in use: register, initialize, timestep_initial, run (per group), timestep_final, finalize. + +--- + +### 10.2 Host model variable structure + +**18 host `.meta` files, all of type `module`.** There are no `host` or `ddt` table types +anywhere. All host variables are flat scalars or arrays in Fortran modules. + +CAM-SIMA does **not** expose its physics DDTs (`phys_state`, `phys_tend`, `cam_in`, etc.) +through metadata. These types exist in `physics_types.F90` but have no `.meta` file. +The generated host cap accesses them directly via `use physics_types, only: phys_state, ...` +and passes individual DDT members as flat keyword arguments: +```fortran +! In cam_ccpp_cap.F90 — direct access to non-metadataized DDT members: +call cam7_physics_before_coupler(..., pint=phys_state%pint, t=phys_state%t, & + dtdt_total=phys_tend%dtdt_total, landfrac=cam_in%landfrac, ...) +``` + +This means capgen has no knowledge of how host data is structured. The host cap is +partly machine-generated and partly depends on manually wiring non-metadataized sources. +**This is a fundamental architectural gap** — changes to `physics_types` are invisible +to the framework. + +**Key host variables by module:** + +| Module | Key variables | +|---|---| +| `physics_grid` | `columns_on_task` (horizontal_dimension), `col_start`, `col_end`, lat, lon, area | +| `vert_coord` | `pver` (vertical_layer_dimension), `pverp` (vertical_interface_dimension) | +| `physconst` | ~35 physical constants, all `protected = True` | +| `cam_constituents` | `num_advected` (count of advected tracers) | +| `spmd_utils` | `mpicom`, `masterproc`, `npes`, `iam` | + +No instance indexing (`physics(1)`) and no thread-indexed DDTs appear — CAM-SIMA uses +a fundamentally different data model from the GFS/SCM stack. + +--- + +### 10.3 The two-cap architecture + +Capgen generates two distinct Fortran files: + +**`cam_ccpp_cap.F90` — the host cap (893 lines)** +- Module `cam_ccpp_cap` +- Imports non-metadataized host variables directly via `use physics_types`, `use physconst`, etc. +- Manages the constituent object (`ccpp_model_constituents_t`) — registration, initialization, gather/scatter, index lookup +- Public subroutines: `cam_ccpp_physics_run`, `cam_ccpp_physics_initialize`, etc. — the entry points the host model calls +- Dispatches to the suite cap, passing ~61–76 flat keyword arguments + +**`ccpp_cam7_cap.F90` — the suite cap (2865 lines)** +- Module `ccpp_cam7_cap` +- No host-specific imports — knows nothing about `physics_types`, `phys_state`, etc. +- All arguments are flat scalars and arrays, fully matched to metadata standard_names +- Contains all scheme calls, suite-level persistent variables, local temporaries, state machine +- The suite cap could in principle be used with any host model that provides the same standard names + +This two-cap split is **architecturally correct**: it separates host-specific binding +from physics-neutral dispatch. The redesign should preserve this separation. + +--- + +### 10.4 The flat-field argument problem — concrete evidence + +The run-phase subroutines expose the core problem with capgen's approach directly: + +```fortran +subroutine cam7_physics_before_coupler(errflg, errmsg, col_start, col_end, pver, dtime, & + gravit, pint, te_ini_dyn, teout, amiroot, iulog, ptend_s, temp, dtdt_total, cpair, & + lagrang, layer_surf, layer_toa, interface_surf, interface_toa, ncnst, piln, pmid, pdel, & + rpdel, qv, carr, cprops, rair, zvir, zi, zm, cp_or_cv_dycore, u, v, pintdry, phis, & + te_cur_phys, te_cur_dyn, tw_cur, latice, latvap, energy_formula_physics, & + energy_formula_dycore, cappa, q_tend, const_tend, qmin, pverp, cpwv, cpliq, rh2o, lat, & + long, pblh, mcon, tpert, dlf, rprd, ql, rliq, landfrac, cpair3, ttend_dp, tmelt, & + top_lev, ke, ke_lnd, cldfrc, domomtran, momcu, momcd, il1g, nstep, & + dudt_total, dvdt_total, fracis, dpdry, ps) +``` + +**61 dummy arguments for one group cap.** `physics_after_coupler` has 76. These are +individual flat arrays and scalars — no DDT in sight. This is exactly the problem that +three developers failed to fix: in the GFS/UFS context, this would be 1,200+ arguments. +The GFS physics stack simply cannot be connected to capgen in its current form. + +In contrast, the prebuild equivalent for the same data would pass `physics` (one DDT argument) +and `cdata` — two arguments covering hundreds of variables. + +--- + +### 10.5 Suite-level persistent variables — the framework-owned data pattern + +The suite cap allocates and owns arrays that persist across group calls within a timestep. +These are allocated in `cam7_initialize` and deallocated in `cam7_finalize`: + +```fortran +! Suite-level persistent (allocated in initialize, freed in finalize): +real(kind_phys), allocatable :: windu_tend(:,:) ! GW drag u-tendency accumulator +real(kind_phys), allocatable :: windv_tend(:,:) ! GW drag v-tendency accumulator +real(kind_phys), allocatable :: scaling_dycore(:,:) ! energy scaling factor +real(kind_phys), allocatable :: tend_te_tnd(:) ! energy tendency accumulator +real(kind_phys), allocatable :: tend_tw_tnd(:) ! water tendency accumulator +real(kind_phys), allocatable :: temp_ini(:,:) ! temperature saved at timestep start +real(kind_phys), allocatable :: z_ini(:,:) ! height saved at timestep start +real(kind_phys), allocatable :: flx_vap(:), flx_cnd(:), flx_ice(:), flx_sen(:) +logical, allocatable :: doconvtran(:) ! per-constituent convection flag +type(coords1d) :: p ! pressure coordinate DDT for GW drag +``` + +These are physics-internal variables — the host model does not know about them, does not +own them, and does not need to. This is the capgen "data ownership" model: the suite cap +is the data owner for variables that only matter within the physics. + +**This pattern is correct and desirable.** The complexity in capgen comes not from the +concept but from how these variables are discovered during analysis (scope-chain promotion) +and passed around (via VarDictionary). The redesign needs a simpler mechanism to achieve +the same result: statically enumerate physics-internal variables during analysis and have +the suite cap own them as named allocatables. + +During the run phase, suite-level persistent arrays are subsetted when passed to schemes: +```fortran +call gw_common_run(..., windu_tend=windu_tend(col_start:col_end, 1:pver), ...) +``` + +Run-phase local temporaries (e.g., `cape`, `cme`, `mu`, `md`) are allocated at function +entry and deallocated at exit: +```fortran +allocate(cape(col_start:col_end)) +... +call zm_convr_run(..., cape=cape, ...) +... +deallocate(cape) +``` + +These temporaries use `col_start` as the lower bound so that assumed-shape dummy arguments +in schemes see a 1-based array — a subtle but important detail. + +--- + +### 10.6 Horizontal chunking model + +CAM-SIMA uses `col_start`/`col_end` (passed as arguments to every run subroutine) to +define the current horizontal chunk: + +```fortran +ncol = col_end - col_start + 1 +``` + +Schemes declare `horizontal_loop_extent` and receive `ncol`. The horizontal dimension +in the host (storage dimension) is `columns_on_task`. The subsetting from storage to +loop extent happens at the boundary between host cap and suite cap — the host cap +passes the right subsections: + +```fortran +! In cam_ccpp_cap.F90: +call cam7_physics_before_coupler(..., col_start=col_start, col_end=col_end, & + pmid=phys_state%pmid, ...) ! full arrays passed; suite cap subsets internally +``` + +Inside the suite cap, persistent arrays are subsetted explicitly when passed to schemes: +```fortran +windu_tend(col_start:col_end, 1:pver) +``` +Local temporaries allocated as `allocate(cape(col_start:col_end))` are already +correctly sized and passed as assumed-shape `(:)`. + +--- + +### 10.7 State machine + +The suite cap has a character module variable tracking lifecycle state: + +```fortran +character(len=16) :: ccpp_suite_state = 'uninitialized' +``` + +Transitions: `uninitialized` → register → `uninitialized` → initialize → `initialized` +→ timestep_initial → `in_time_step` → (run, no state change) → timestep_final → +`initialized` → finalize → `uninitialized`. + +Each phase entry point checks the expected prior state: +```fortran +if (trim(ccpp_suite_state) /= 'in_time_step') then + errflg = 1 + write(errmsg, '(3a)') "Invalid initial CCPP state, '", trim(ccpp_suite_state), & + "' in cam7_physics_before_coupler" + return +end if +``` + +Non-run phases also include an OpenMP thread guard: +```fortran +#ifdef _OPENMP + if (omp_get_thread_num() > 1) then + errflg = 1 + errmsg = "Cannot call initialize routine from a threaded region" + return + end if +#endif +``` + +The state machine is simple, complete, and useful. The redesign should preserve it. + +--- + +### 10.8 Constituent variable handling + +CAM-SIMA demonstrates the full constituent lifecycle: + +```fortran +! In cam_ccpp_cap.F90: +type(ccpp_model_constituents_t), target :: cam_constituents_obj + +! Registration (scheme-declared constituents): +call suite_cam7_constituents_num_consts(num_consts) +call suite_cam7_constituents_const_name(iconst, const_name) +call cam_constituents_obj%new_field(const_name, ...) + +! Initialization (host-declared constituents like water vapor): +cam_model_const_stdnames(1) = "water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water" +call cam_constituents_obj%new_field(cam_model_const_stdnames(1), ...) + +! Per-timestep gather from host: +call cam_ccpp_gather_constituents(phys_state%q, ...) + +! Passing to suite cap: +call cam7_physics_before_coupler(..., + qv = cam_constituents_obj%vars_layer(:, :, cam_model_const_indices(1)), + carr = cam_constituents_obj%vars_layer, + cprops = cam_constituents_obj%const_metadata, ...) + +! Per-timestep scatter back to host: +call cam_ccpp_update_constituents(phys_state%q, ...) +``` + +The suite cap sees constituents as: +- `carr(:,:,:)` — the full rank-3 constituent array (ncol, nlev, ncnst) +- `qv(:,:)` — water vapor slice extracted in the host cap: `cam_constituents_obj%vars_layer(:,:,cam_model_const_indices(1))` +- `cprops(:)` — array of `ccpp_constituent_prop_ptr_t` metadata objects +- `doconvtran(1:ncnst)` — suite-level logical array set by scheme init indicating which constituents are convected + +This constituent API is sophisticated and worth preserving or improving in the redesign. + +--- + +### 10.9 Known defects in the capgen output + +**Repeated scheme init/final calls.** Capgen generates one init call per occurrence of +a scheme name in the XML, without deduplication: +- `qneg_init` called 5 times (once per `qneg` entry in the suite XML) +- `qneg_timestep_final` called 5 times +- `check_energy_chng_init` called twice +- `save_ttend_from_convect_deep_timestep_init` called 3 times + +If these routines have internal state, allocations, or side effects, this is a correctness +defect. The redesign must deduplicate init/final calls by unique scheme name. + +**Unit conversion embedded silently in the cap.** Before `zm_conv_convtran_run`: +```fortran +dpdry_local(:,1:pver) = 1.0E-2_kind_phys * dpdry(:,1:pver) ! Pa → hPa +``` +This is generated from the metadata units mismatch but appears as an opaque transform +in the cap. The redesign should make this visible (e.g., a comment naming the standard +name, the source units, and the target units). + +--- + +### 10.10 Build system — capgen invocation + +Capgen is invoked from Python (`cam_autogen.py`), not from cmake: + +```python +from ccpp_capgen import capgen +capgen_db = capgen(run_env, return_db=True) +``` + +This is a programmatic API call, not a subprocess. The `CCPPDatabaseObj` returned +(`capgen_db`) is then used directly in Python to query scheme lists, constituent names, +and file paths — avoiding the datatable XML query step that cmake-based invocations need. + +Output files consumed by the build: +- `cam_ccpp_cap.F90` — compiled into the atmosphere component +- `ccpp_cam7_cap.F90` — compiled into the atmosphere component +- `ccpp_kinds.F90` — compiled into the atmosphere component +- Utility files from `ccpp_framework/src/` (copied to build dir) +- `ccpp_datatable.xml` — queried by the build system for file lists + +--- + +### 10.11 Observations relevant to the redesign + +1. **The two-cap split (host cap + suite cap) is the right architecture.** It cleanly + separates host-specific binding from physics-neutral dispatch. The redesign must + preserve this. + +2. **Flat-field arguments in the suite cap are the critical failure.** 61–76 dummy + arguments per run subroutine is already large for a research model; for UFS/GFS + with 1,200+ variables it is completely infeasible. The redesign must pass DDTs. + +3. **The CAM-SIMA host does not use DDTs in metadata.** All host variables are flat + module variables. This is a fundamentally different host model architecture from + GFS/SCM. The redesign must support both styles: flat-module hosts (CAM-SIMA) and + deep-DDT hosts (GFS/SCM). + +4. **Non-metadataized variables hardwired into the host cap is a serious gap.** + `phys_state`, `phys_tend`, `cam_in` from `physics_types` have no `.meta` files. + The host cap accesses them directly. This means the framework cannot verify or + track these variables. The redesign should either require full metadata coverage + or have an explicit mechanism for declaring non-metadataized pass-through variables. + +5. **Suite-level persistent variables (framework-owned data) work well in practice.** + `windu_tend`, `scaling_dycore`, `temp_ini`, etc. are owned by the suite cap, invisible + to the host, and persist across group calls. This is the right pattern for + physics-internal state. The redesign needs this but with a simpler discovery mechanism + than capgen's scope-chain promotion. + +6. **Deduplicate init/final calls.** The redesign must deduplicate `_init`, `_finalize`, + `_timestep_init`, and `_timestep_final` calls by unique scheme name (not by occurrence + in the XML). + +7. **The constituent API in the host cap is comprehensive.** The `ccpp_model_constituents_t` + object with its register/init/gather/scatter/index API is sophisticated and should be + preserved or improved. + +8. **The suite-variables introspection subroutine** (`ccpp_physics_suite_variables`, + enumerating 83 standard names as inputs/outputs) is a useful capability for build + system integration and should be in the redesign. + +9. **The programmatic Python API** (`capgen(run_env, return_db=True)`) is valuable + for hosts like CAM-SIMA that invoke the generator from Python. The redesign should + support both CLI and programmatic invocation. + +10. **Unit conversions must be annotated in the generated cap**, not silently embedded + as magic-number multiplications. A comment with source units, target units, and the + standard name involved is the minimum. + +11. **The horizontal chunking model** (`col_start`/`col_end` as explicit arguments, + `ncol = col_end - col_start + 1` computed at entry) works and is clean. Suite-level + persistent arrays are allocated full-size and subsetted at call sites. + +12. **No optional variables in this model.** CAM-SIMA does not exercise optional/active + variable handling. This feature must be in the redesign but is not demonstrated here. + +--- + +## 11. Real-world example: UFS Weather Model (prebuild) + +The UFS Weather Model is the most complex and production-critical of the three examples. +It is a fully-coupled, 3-D operational NWP model. The CCPP physics is used in the +atmospheric component (`UFSATM`). Unlike the SCM (column model, process-split only) and +CAM-SIMA (capgen, flat-field arguments), UFS uses prebuild in a 3-D blocked/threaded +configuration that is architecturally distinct from both prior examples. + +The two suites analyzed here are: +- `FV3_GFS_v17_coupled_p8` — the primary operational GFS suite +- `FV3_GFS_v17_coupled_p8_ugwpv1` — a variant replacing `unified_ugwp` with `ugwpv1` + +The ugwpv1 suite is structurally identical to the base suite except for the `phys_ps` +group (4 extra scheme calls), so all observations below apply to both. + +--- + +### 11.1 Suite structure + +The primary suite has 5 groups: + +| Group | Subcycles | Scheme calls | Phase called | +|-------|-----------|-------------|--------------| +| `time_vary` | 1 | 4 | timestep_init (domain-level, no blocking) | +| `radiation` | 1 | 8 | run (block/thread loop) | +| `phys_ps` | 3 (loop=1, loop=2, loop=1) | 21 | run (block/thread loop) | +| `phys_ts` | 3 (loop=1, loop=1, loop=1) | 12 | run (block/thread loop) | +| `stochastics` | 1 | 2 | run (block/thread loop) | + +The `time_vary` group is the only one called at timestep_init/finalize. All other groups +are called from the run phase via the OpenMP blocked loop. This is a fundamentally +different usage pattern from SCM (which runs everything sequentially) and CAM-SIMA +(which has no run phase at all for the groups analyzed). + +The `phys_ps` group has a surface iteration subcycle with `loop="2"`, which generates an +actual Fortran `do` loop in the cap body: +```fortran +! Start of next subcycle +cdata%loop_max = 2 +do cdata%loop_cnt = 1, cdata%loop_max + ! ... sfc_diff, sfc_nst, noahmpdrv, sfc_land, sfc_cice, sfc_sice ... +end do +``` + +--- + +### 11.2 Cap hierarchy and scale + +The three-level hierarchy is preserved from prebuild: + +``` +ccpp_static_api.F90 (627 lines) ← suite+group name dispatch + ↓ +ccpp_fv3_gfs_v17_coupled_p8_cap.F90 (363 lines) ← calls all group caps in order + ↓ +ccpp_fv3_gfs_v17_coupled_p8_time_vary_cap.F90 (1404 lines) +ccpp_fv3_gfs_v17_coupled_p8_radiation_cap.F90 (967 lines) +ccpp_fv3_gfs_v17_coupled_p8_phys_ps_cap.F90 (4226 lines) ← 200 optional ptr arrays +ccpp_fv3_gfs_v17_coupled_p8_phys_ts_cap.F90 (1953 lines) +ccpp_fv3_gfs_v17_coupled_p8_stochastics_cap.F90 (443 lines) +``` + +The ugwpv1 variant generates another 10,220 lines of largely redundant code (identical +caps with one suite-name prefix change and minor scheme-list differences). Total for both +suites: 18,333 lines of generated Fortran. + +This redundancy is a key motivation for the redesign: suite variants that share groups +should not regenerate identical cap code. The redesign should support group-level cap +sharing across suite variants. + +--- + +### 11.3 Host model DDT structure + +All host data lives in `CCPP_data.F90` as module-level `save, target` variables: + +```fortran +type(GFS_control_type) :: GFS_control ! config/control +type(GFS_statein_type) :: GFS_statein ! atmospheric state in +type(GFS_stateout_type) :: GFS_stateout ! atmospheric state out +type(GFS_grid_type) :: GFS_grid ! grid geometry +type(GFS_tbd_type) :: GFS_tbd ! temporal interp data +type(GFS_cldprop_type) :: GFS_cldprop ! cloud properties +type(GFS_sfcprop_type) :: GFS_sfcprop ! surface properties +type(GFS_radtend_type) :: GFS_radtend ! radiation tendencies +type(GFS_coupling_type) :: GFS_coupling ! coupling fields +type(GFS_diag_type) :: GFS_intdiag ! diagnostics +type(GFS_interstitial_type), allocatable (:) :: GFS_interstitial ! scratch, per thread +``` + +Plus three `ccpp_t` instances for different levels of parallelism (see §11.5). + +This is structurally similar to the SCM's `physics` DDT hierarchy, but with one key +difference: all DDTs are at the same flat level rather than nested (no `physics%Statein`, +only `GFS_statein`). Each DDT maps to a distinct functional role. + +The `GFS_typedefs.F90` file (not auto-generated) defines all DDT types along with ~30 +physical constants (`con_pi`, `con_g`, `con_rd`, etc.) that also appear in the metadata. + +--- + +### 11.4 DDT arguments in the cap chain + +The static API imports all DDTs and physical constants from `CCPP_data` and `GFS_typedefs` +via `use` statements, then passes them as named arguments to group cap functions. This is +the full DDT-argument pattern that prebuild implements: + +```fortran +! In ccpp_static_api.F90: +use ccpp_data, only: gfs_control, gfs_statein, gfs_sfcprop, ... +use gfs_typedefs, only: con_pi, con_g, con_rd, ... + +ierr = fv3_gfs_v17_coupled_p8_phys_ps_run_cap( & + one=one, gfs_control=gfs_control, cdata=cdata, & + gfs_statein=gfs_statein, gfs_sfcprop=gfs_sfcprop, & + con_g=con_g, con_pi=con_pi, ... & + gfs_interstitial=gfs_interstitial) +``` + +The group cap receives these as typed `intent(*), target` dummy arguments and uses them +directly to construct call-site subsections. This means **the group cap is fully portable +— it does not use any host module directly**, only what it receives as arguments. + +The `target` attribute is required because the cap creates pointer sections of these DDTs +(array subsections via pointer assignment) when handling optional variables. + +--- + +### 11.5 The dual cdata architecture + +UFS uses two distinct sets of `ccpp_t` handles with different scopes: + +**Domain-level (`cdata_domain`)**: Used for non-run phases (init, finalize, time_vary +timestep_init/finalize). Called once per step, no blocking: +```fortran +cdata_domain%blk_no = 1; cdata_domain%chunk_no = 1 +cdata_domain%thrd_no = 1; cdata_domain%thrd_cnt = 1 +``` + +**Block/thread-level (`cdata_block(nb, nt)`)**: Used for run phase (radiation, phys_ps, +phys_ts, stochastics). Allocated as a 2-D array `(1:nblks, 1:nthrdsX)` where `nthrdsX` +accounts for non-uniform last-block sizing: +```fortran +cdata_block(nb,nt)%blk_no = nb +cdata_block(nb,nt)%chunk_no = nb ! block number = chunk number +cdata_block(nb,nt)%thrd_no = nt +cdata_block(nb,nt)%thrd_cnt = nthrdsX +``` + +The redesign must support this dual cdata usage: a single `cdata` handle for domain-level +phases and a 2-D array of handles for blocked run phases. + +--- + +### 11.6 OpenMP threading model + +Non-run phases allow internal threading in physics schemes: +```fortran +GFS_control%nthreads = nthrds ! all N threads available to physics +call ccpp_physics_timestep_init(cdata_domain, ...) +``` + +Run phase uses all threads for blocking, so physics must not spawn additional threads: +```fortran +GFS_control%nthreads = 1 ! no internal threading allowed +!$OMP parallel num_threads(nthrds) ... +!$OMP do schedule(dynamic,1) +do nb = 1, nblks + call GFS_Interstitial(nt)%create(ixs=chunk_begin(nb), ixe=chunk_end(nb), model=GFS_control) + call ccpp_physics_run(cdata_block(nb,nt), group_name="phys_ps", ...) + call GFS_Interstitial(nt)%destroy(GFS_control) +end do +!$OMP end do +!$OMP end parallel +``` + +The `nt = omp_get_thread_num()+1` pattern (1-based thread index) is used throughout. +Each thread owns one `GFS_Interstitial(nt)` and one `cdata_block(nb,nt)` per block +iteration. The dynamic schedule means different threads process different blocks at +different times, which is why the interstitial must be created/destroyed per-iteration +rather than pre-allocated per-thread. + +--- + +### 11.7 Horizontal dimension: the chunk_begin/chunk_end pattern + +For non-run phases, the full horizontal dimension is used at every call site: +```fortran +tgrs(one:gfs_control%ncols, one:gfs_control%levs) +``` + +For run phases, the chunk range is looked up from the control DDT using the block number: +```fortran +tgrs(gfs_control%chunk_begin(cdata%chunk_no) : gfs_control%chunk_end(cdata%chunk_no), & + one:gfs_control%levs) +``` + +The chunk size (horizontal extent `im`) is retrieved as: +```fortran +im = gfs_control%blksz(cdata%blk_no) +``` + +`blksz(nb)` handles **non-uniform block sizes**: the last block may be smaller than the +others if the domain size is not divisible by the number of blocks. The `chunk_begin`/ +`chunk_end` arrays (indexed by chunk number = block number) give the global offset range. + +This is a cleaner pattern than SCM's `chunk_begin`/`chunk_end` as explicit dummy +arguments, because UFS looks them up from the already-passed `gfs_control` DDT. + +**Critical implication for the redesign**: The subsetting pattern `(chunk_begin:chunk_end)` +appears at every single array call site in the run phase — literally hundreds of times in +the phys_ps cap alone. This boilerplate is generated by prebuild from the metadata. In +the redesign, this subsetting must remain at the call site (not higher up) to allow each +thread to process its own chunk independently. + +--- + +### 11.8 The GFS_interstitial — pointer-based scratch DDT + +`GFS_interstitial_type` (defined in `CCPP_typedefs.F90`) is a DDT where **every field is +a pointer**, initialized to null: +```fortran +type GFS_interstitial_type + real(kind_phys), pointer :: adjsfculw_land(:) => null() + real(kind_phys), pointer :: del(:,:) => null() + ! ... ~200+ pointer fields +end type +``` + +This is dramatically different from the SCM's interstitial (which is a regular allocatable +DDT allocated once per thread at startup). The UFS interstitial is: +1. **Created** (`GFS_Interstitial(nt)%create(ixs, ixe, model)`) before each block — this + allocates all required fields to the chunk size `ixe-ixs+1` +2. **Reset** (`GFS_Interstitial(nt)%reset(model)`) to zero before radiation and phys_ps +3. **Destroyed** (`GFS_Interstitial(nt)%destroy(model)`) after each block — deallocates + +This design exists because different blocks (especially the last block) can have different +sizes. Pre-allocating to the maximum size wastes memory at scale; per-block allocation +ensures exact sizing. The pointer-based design also allows the `create()` method to +selectively allocate only the fields needed for the current physics configuration. + +In the caps, the interstitial is accessed as: +```fortran +gfs_interstitial(cdata%thrd_no)%del(chunk_begin:chunk_end, one:levs) +``` + +The interstitial array is 1-D (indexed by thread, not by `(instance, thread)` as in SCM). +This works because UFS has only one model instance at runtime — no ensemble-in-memory. + +--- + +### 11.9 Optional variables — the pointer array pattern at scale + +The phys_ps run cap has **200 optional pointer arrays** in its local variable section. +Each looks like: +```fortran +type :: real_kind_phys_rank1_ptr_arr_type + real(kind_phys), dimension(:), pointer :: p => null() +end type real_kind_phys_rank1_ptr_arr_type +type(real_kind_phys_rank1_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array +``` + +Usage pattern (consistent with SCM but with threading dimension): +```fortran +if (gfs_control%lndp_type /= 0) then + sfc_wts_1_ptr_array(cdata%thrd_no)%p => & + gfs_coupling%sfc_wts(chunk_begin:chunk_end, one:gfs_control%n_var_lndp) +end if +! ... scheme call ... +if (gfs_control%lndp_type /= 0) then + nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) +end if +``` + +The array is dimensioned by `cdata%thrd_cnt` (total thread count) and indexed by +`cdata%thrd_no` (current thread number). This handles the threaded run phase where +multiple threads are simultaneously executing the same run cap function with different +chunk ranges. Each thread independently associates and nullifies its own pointer slot. + +200 optional variables in `phys_ps` alone. This is the regime for which the SCM had ~550 +total optional vars — confirming that operational 3-D GFS physics is heavily optional-var +driven. The design is sound but generates enormous boilerplate. + +A key observation: the type definition for each pointer wrapper (`integer_..._ptr_arr_type`, +`real_kind_phys_rank1_ptr_arr_type`, etc.) is **re-declared inside every single function +that needs it**. This results in duplicate type definitions across all group caps. The +redesign should define these wrapper types once in a shared module. + +--- + +### 11.10 Physical constants as metadata variables + +The UFS static API has an extensive USE list of physical constants from `gfs_typedefs`: +``` +con_pi, con_g, con_t0c, con_hfus, con_solr_2008, con_solr_2002, con_c, con_plnk, +con_boltz, con_rd, ltp, con_zero, con_rerth, con_p0, con_rv, con_cp, con_rgas, +con_amd, con_amw, con_avgd, con_hvap, con_eps, con_omega, con_fvirt, con_ttp, +con_thgni, con_epsm1, con_rog, con_rocp, con_tice, con_sbc, con_jcal, con_rhw0, +rlapse, rhowater, karman, con_1ovg, con_cliq, con_cvap, rainmin, con_epsm1 (30+ total) +``` + +These travel through the full chain: static API USE → suite cap argument → group cap +argument → scheme call argument. Each constant is declared as a separate scalar dummy +argument (`real(kind_phys), intent(in), target :: con_pi`) in every group cap that needs +it. + +This is correct but verbose. The redesign should consider whether constants should be +gathered into a dedicated DDT (e.g., `gfs_constants_type`) so the cap chain carries one +argument instead of 30. This would also eliminate the need to explicitly enumerate which +constants each group needs — they could all come along in the constants DDT. + +--- + +### 11.11 The `one` lower-bound anchor + +The integer constant `one = 1` (from `ccpp_types`) is passed as an explicit argument +throughout the UFS cap chain for the same reason as in SCM: it anchors lower array bounds +without triggering association-status issues: +```fortran +type(gfs_interstitial_type), intent(inout), target :: gfs_interstitial(one:) +tgrs(one:gfs_control%ncols, one:gfs_control%levs) +``` + +This pattern is ubiquitous and is a known prebuild idiom. + +--- + +### 11.12 No framework-owned persistent variables + +Unlike CAM-SIMA (which allocates scheme-persistent variables in the suite cap), the UFS +has no framework-owned persistent state in any cap. All persistent state lives in the host +DDTs (`GFS_tbd`, `GFS_sfcprop`, etc.). The interstitial DDT (`GFS_interstitial`) is purely +transient — created and destroyed each block. + +This is consistent with UFS's prebuild-based architecture. Whether framework-owned +persistent variables would be beneficial for UFS is an open question for the redesign. + +--- + +### 11.13 Build system and driver + +Prebuild is invoked from CMake (not programmatically) and generates: +- Group cap files (one per group × suites) +- Suite cap files (one per suite) +- `ccpp_static_api.F90` +- `CCPP_CAPS.cmake`, `CCPP_SCHEMES.cmake`, `CCPP_TYPEDEFS.cmake` — consumed by CMake to + enumerate files to compile + +The host driver (`CCPP_driver.F90`) is **hand-written**, not auto-generated. It owns the +OpenMP loop, the cdata allocation/setup, the interstitial create/destroy, and the +diagnostic bucket zeroing. This is a significant difference from CAM-SIMA where the +equivalent driver code is partially generated. In the redesign, this host driver code +should remain hand-written — it encodes model-specific threading and blocking decisions +that cannot be derived from metadata alone. + +--- + +### 11.14 Observations relevant to the redesign + +1. **The DDT-argument cap chain is fully validated at UFS scale.** Passing 10+ DDTs plus + 30+ scalar constants as named arguments through three cap levels works correctly in + production. The redesign must replicate this exactly. + +2. **The chunk_begin/chunk_end subsetting at call sites is non-negotiable.** Hundreds of + array sections per group cap. The generator must produce this from the metadata + `horizontal_dimension` standard name and the `active` flag for optional variables. + This is prebuild's core value at 3-D scale. + + *Design direction*: Rather than carrying `chunk_no` in cdata and having the cap look + up `gfs_control%chunk_begin(chunk_no)`, the redesign should pass + `horizontal_loop_begin` and `horizontal_loop_end` as explicit arguments directly to + `ccpp_physics_run()` (and analogous calls). This decouples the cap from knowing about + the host's internal chunk-lookup arrays. The host driver sets these for each block + iteration and passes them in; the cap uses them directly. + +3. **The domain-vs-block execution contexts must be supported, but the cdata object is + not necessarily the right mechanism.** The key information is: instance number, thread + number, horizontal_loop_begin, horizontal_loop_end, error flag/message. If all of + these are explicit named arguments to `ccpp_physics_*`, the cdata object becomes + redundant scaffolding. This is an open design question to be discussed separately, but + the UFS analysis shows that cdata carries exactly these values — the object is a + transport container, not a framework abstraction. + +4. **The `blksz` non-uniform block size is a first-class concern.** The generator must + produce `im = gfs_control%blksz(cdata%blk_no)` (or an equivalent `horizontal_loop_extent` + computed from the explicit begin/end) for the horizontal extent argument in run phases. + +5. **GFS_interstitial as a pointer-DDT is the correct design for 3-D models.** Creating + and destroying per block avoids memory waste from over-allocation to the maximum chunk + size. The pointer-based field design enables selective allocation. The redesign should + document this pattern and support it. (Whether the generator should emit the + `type(X_interstitial_type)` DDT definition itself or only the caps is TBD.) + +6. **200 optional pointer arrays in one group cap is manageable but the wrapper type + proliferation is not.** The 4 wrapper types (`integer_r1_ptr_arr_type`, + `real_r1_ptr_arr_type`, `real_r2_ptr_arr_type`, `character_len3_r1_ptr_arr_type`) + should be defined once in a shared module (e.g., `ccpp_types.F90`) and reused across + all caps, eliminating thousands of duplicate lines. + +7. **Physical constants as metadata variables must be gathered into a constants DDT.** + The redesign will collect all physics constants into a single `constants_type` DDT + (or equivalent), reducing 30+ individual scalar arguments in the cap chain to one + argument. This requires a metadata declaration mechanism for compound read-only + objects (i.e., constants do not need intent tracking the way state variables do). + +8. **No framework-owned persistent variables in UFS** confirms that this feature is + optional and model-specific. The redesign needs to support it (for CAM-SIMA-like + models) but should not force it on models that do not need it. + +9. **The host driver is correctly hand-written.** The OpenMP blocking, interstitial + lifecycle, diagnostic bucket management — these are model-specific decisions that + belong in the host driver, not in generated code. The redesign should not try to + generate the driver. + +10. **Suite variant cap redundancy is not a concern.** For research/development, multiple + suites are active simultaneously and generated code size doesn't matter. For + production, only one suite is compiled and used at a time. The redesign need not + prioritize eliminating redundant group cap code across suite variants. + +--- + +## 12. Real-world example: Navy NEPTUNE (prebuild, restricted) + +The NEPTUNE source code cannot be shared. The following is based on architectural +description provided by the lead developer. + +NEPTUNE uses `ccpp-prebuild` with the same GFS physics as UFS and nearly identical suites. +Its unique distinguishing feature is **multiple coexisting CCPP physics instances** — it +is the only model among the four examples that exercises this capability at runtime. + +--- + +### 12.1 Multiple instances — the N-dimensioned DDT array mechanism + +In NEPTUNE, the host model allocates N copies of all GFS DDTs as 1-D arrays indexed by +instance number: + +```fortran +type(GFS_sfcprop_type), allocatable :: gfs_sfcprop(1:N) +type(GFS_statein_type), allocatable :: gfs_statein(1:N) +type(GFS_stateout_type), allocatable :: gfs_stateout(1:N) +! ... all GFS DDTs dimensioned 1:N +type(GFS_control_type), allocatable :: gfs_control(1:N) +``` + +The static API imports these module-level arrays via `use` statements (same as UFS). +The instance selection happens at the call site inside the group cap, using +`cdata%ccpp_instance` as the array index: + +```fortran +call foo_run( & + tair = gfs_statein(cdata%ccpp_instance)%tair( & + gfs_control(cdata%ccpp_instance)%chunk_begin(cdata%chunk_no) : & + gfs_control(cdata%ccpp_instance)%chunk_end(cdata%chunk_no), & + 1:nvertical), & + ...) +``` + +Three things are happening simultaneously at each call-site array section: +1. **Instance selection**: `gfs_statein(cdata%ccpp_instance)` picks the correct DDT from + the N-element array +2. **Chunk subsetting**: `chunk_begin(chunk_no):chunk_end(chunk_no)` applies the run-phase + horizontal slice +3. **Vertical bound**: explicit `1:nvertical` + +This is the same pattern as UFS except the DDTs are 1-D arrays rather than scalars. +The generator must produce this instance-indexed subsetting when the host declares its +DDTs as arrays. + +--- + +### 12.2 What NEPTUNE tells us about `cdata%ccpp_instance` + +The `initialized(200)` array in every group cap (confirmed in both SCM and UFS caps) now +has its full motivation: it handles up to 200 simultaneous instances without requiring +per-instance cap code. The `cdata%ccpp_instance` value (1-based) is the runtime selector +into both the host DDT arrays and the `initialized` guard array. + +NEPTUNE is the reason `200` is not `1`. In single-instance models (UFS, SCM, CAM-SIMA) +`cdata%ccpp_instance` is always 1 and the N-dimensioned DDT arrays have `N=1`. + +--- + +### 12.3 Observations relevant to the redesign + +1. **Multiple instances require only one change at the call site**: inserting the instance + index at the correct dimension position. Everything else (chunking, optional variables, + threading) composes with this unchanged. + +2. **The instance dimension can appear anywhere in any host variable — not just as an + index into an array of DDTs.** A flat array `flat_field(1:ninstance, 1:nhoriz, 1:nvert)` + is equally valid; its call site becomes: + ```fortran + flat_field(instance_number, horiz_begin:horiz_end, 1:nvert) + ``` + The generator handles this by classifying each dimension by its declared standard name. + `instance_dimension` is a registered standard name (like `horizontal_dimension` and + `vertical_dimension`) — the generator knows its semantics regardless of where it + appears in the dimension list or whether the variable is a DDT array element or a + plain array. See §13.4 for the full dimension classification model. + +3. **No new cap-level mechanism is needed for multi-instance.** The instance number + (from the control layer, see §13) is sufficient. The cap code shape is the same; + only the call-site indexing expression differs based on the declared dimension roles. + +--- + +## 13. Cross-cutting design decision: how host data enters the cap chain + +Across all four models, two mechanisms are used for getting host model data into the +generated caps: + +| Mechanism | Models using it | Description | +|-----------|----------------|-------------| +| **Module USE** | UFS, SCM, CAM-SIMA, NEPTUNE | Static API has `use ccpp_data, only: gfs_statein, ...`. Data module name is known at generation time. | +| **Command-line arguments** | capgen (optional) | Generator accepts host variable access paths as CLI flags; generated caps receive data as explicit dummy arguments. | + +### 13.1 The capgen dual-mechanism problem + +Capgen supports both mechanisms, and this is a direct source of its complexity. The +variable-matching logic, VarDictionary scope chains, and `CCPPDatabaseObj` all exist +partly to handle the routing of variables that may arrive via either path. Maintaining +two entry points to the data layer doubles the surface area that must be tested and +reasoned about. + +### 13.2 The proposed single-mechanism approach + +The redesign will use **module USE exclusively** for all host data. The reasoning: + +- All four production models already use module USE, including CAM-SIMA (the capgen + model), which does not use capgen's CLI-argument path in practice. +- Module names are stable, known at generation time, and make the generated code + self-documenting (`use ccpp_data, only: gfs_statein` is unambiguous). +- Eliminating the CLI-argument entry path eliminates an entire class of generator + complexity. + +### 13.3 Runtime control variables — the thin explicit layer + +While all *data* enters via module USE, a set of *control* variables must be passed at +runtime because they change from call to call. These are not physics data; they tell the +cap *how* to index into the data it already has access to: + +| Variable | Purpose | When it matters | +|----------|---------|----------------| +| `ccpp_instance` | Select the instance dimension in host variables | NEPTUNE (N>1); others use 1 | +| `ccpp_thread_no` | Index optional pointer arrays per thread | Run phase with OpenMP | +| `horizontal_loop_begin` | Start of horizontal chunk to process | Run phase | +| `horizontal_loop_end` | End of horizontal chunk to process | Run phase | +| `ccpp_nthreads` | Max threads available for internal physics use | Non-run phases (currently `gfs_control%nthreads`) | +| `errmsg` / `errflg` | Error reporting return path | All phases | + +These are exactly the values that `cdata` carries in the current implementation. +Whether they are packaged as a `ccpp_t` struct or passed as individual named arguments to +`ccpp_physics_*` is an open design question for implementation. Either way, the generator +only needs to know about these variables and their standard names — it does not need to +accept host data paths on the command line. + +### 13.4 The dimension classification model + +A host variable's metadata declares the **standard name of each of its dimensions** in +order. The generator classifies every dimension into one of three categories and +constructs the call-site expression accordingly. + +**Category 1 — Registered dimensions.** The generator knows the semantics of these +standard names and generates special call-site expressions for them: + +| Standard name | Call-site expression | Notes | +|--------------|---------------------|-------| +| `instance_dimension` | `instance_number` (scalar index) | Omitted if variable has no instance dimension | +| `horizontal_dimension` | `1:horizontal_dimension` (non-run) or `horiz_begin:horiz_end` (run) | Phase-dependent | +| `vertical_dimension` | `1:vertical_dimension` | Fixed range | + +`instance_dimension` has the same registered status as `horizontal_dimension` and +`vertical_dimension`. Single-instance models simply do not declare any variables with +an `instance_dimension`, and the generator omits that index entirely. + +**Category 2 — Arbitrary host-declared dimensions.** Any dimension whose standard name +is not in the registered set. These are declared in host metadata pointing to a Fortran +expression accessible via module USE — either a flat module variable or a DDT member +(e.g. `gfs_control%ntrac`, `gfs_control%kice`). The generator emits `1:expression` +at the call site, resolved at generation time from the metadata. Fixed-index extractions +(e.g. `gfs_statein%qgrs(..., gfs_control%ntqv)`) are a special case: the dimension +value is a scalar index rather than a range upper bound, and the metadata must declare +which case applies. + +**Category 3 — Optional selector.** Not a dimension per se, but a boolean `active` +condition declared in variable metadata. Generates a pointer-association guard around +the call site (the pattern described in §9 and §11). + +This three-category model works uniformly regardless of host layout: +- `gfs_statein(instance)%tair(horiz, vert)` — registered instance + registered horizontal + registered vertical +- `flat_field(instance, horiz, vert, ntrac)` — registered + registered + registered + arbitrary +- `flat_field(horiz, vert)` — no instance dimension, single-instance model + +No special-casing per host model is needed in the generator. + +### 13.5 `type = control` — metadata declaration for runtime control variables + +The registered dimensions (§13.4 Category 1) are *dimension names* that appear in a +variable's `dimensions = (...)` list. Their actual *runtime values* are supplied by a +separate set of variables declared with `type = control` in host metadata. + +| `type = control` standard name | Fills in registered dimension / purpose | +|-------------------------------|----------------------------------------| +| `ccpp_instance` | `instance_dimension` — scalar index selecting the active instance | +| `ccpp_thread_no` | Not a dimension; indexes optional pointer arrays per thread | +| `horizontal_loop_begin` | Lower bound of `horizontal_dimension` in run phase | +| `horizontal_loop_end` | Upper bound of `horizontal_dimension` in run phase | +| `ccpp_nthreads` | Not a dimension; max threads available for internal physics use | +| `errmsg` / `errflg` | Error reporting return path | + +Variables declared `type = control` are: +- **Passed explicitly as runtime arguments** to `ccpp_physics_*` by the host driver + (not accessed via module USE, because their values change per call) +- **Used by the generator** to construct call-site indexing expressions for registered + dimensions, and to generate the `ccpp_nthreads` assignment before non-run scheme calls +- **Available to physics schemes** by standard name like any other variable — if a scheme + declares a variable with a matching standard name (e.g. `ccpp_nthreads`, + `horizontal_loop_begin`), the framework passes it as a scheme argument in the normal way + +This is similar in concept to capgen's `type = host` annotation but with a narrower, +well-defined scope. The name `control` is intentional: these variables *control* how +the cap indexes into the data, not what the data is. + +The set of recognized standard names for `type = control` variables is fixed and small. +Declaring them explicitly in metadata — rather than having the generator recognize magic +names — keeps the mechanism open and self-documenting. + +### 13.6 Consequences for the generator + +1. The generator reads host metadata to learn: + - Module names for all host data variables (emitted as `use` statements in the static API) + - The dimension standard names of each variable (for call-site expression construction) + - Which variables are `type = control` (for the runtime argument layer) +2. At cap generation time, the static API's `use` statements are emitted from the module + names — no runtime flexibility, no CLI data routing. +3. Call-site subsetting for every variable is constructed purely from its declared + dimension standard names: registered dimensions use the Category 1 rules; arbitrary + dimensions are resolved to Fortran expressions via the host metadata. +4. The only runtime inputs to the cap are the `type = control` variables. Their values + are supplied by the host driver for each `ccpp_physics_*` call. diff --git a/doc/redesign_analysis.md - updated 20260513T0733.md b/doc/redesign_analysis.md - updated 20260513T0733.md new file mode 100644 index 00000000..67252a5f --- /dev/null +++ b/doc/redesign_analysis.md - updated 20260513T0733.md @@ -0,0 +1,2639 @@ +# CCPP Framework Code Generator — Technical Analysis for Redesign + +*Analysis date: 2026-05-04. Clarifications added: 2026-05-05.* + +This document is a deep-dive technical analysis of the two existing CCPP Framework code generators — +`ccpp-prebuild` and `ccpp-capgen` — produced as input to a planned complete redesign. +It covers execution flow, data structures, feature sets, build system integration, and +key architectural differences. + +--- + +## Table of Contents + +1. [Background and motivation](#1-background-and-motivation) +2. [ccpp-prebuild — detailed analysis](#2-ccpp-prebuild--detailed-analysis) +3. [ccpp-capgen — detailed analysis](#3-ccpp-capgen--detailed-analysis) +4. [Shared infrastructure](#4-shared-infrastructure) +5. [Feature comparison](#5-feature-comparison) +6. [Build system integration](#6-build-system-integration) +7. [Key architectural differences](#7-key-architectural-differences) +8. [Design considerations for the redesign](#8-design-considerations-for-the-redesign) +9. [Real-world example: CCPP Single Column Model (SCM)](#9-real-world-example-ccpp-single-column-model-scm) +10. [Real-world example: CAM-SIMA (capgen)](#10-real-world-example-cam-sima-capgen) +11. [Real-world example: UFS Weather Model (prebuild)](#11-real-world-example-ufs-weather-model-prebuild) +12. [Real-world example: Navy NEPTUNE (prebuild, restricted)](#12-real-world-example-navy-neptune-prebuild-restricted) +13. [Cross-cutting design decision: how host data enters the cap chain](#13-cross-cutting-design-decision-how-host-data-enters-the-cap-chain) + +--- + +## 1. Background and motivation + +The CCPP Framework is a code generator that analyzes metadata describing variables required +by physical parameterizations in numerical weather prediction (NWP) models, compares them +against metadata provided by a host model, and generates Fortran interface ("cap") code that +connects the two. + +There are two generations of the generator: + +**`ccpp-prebuild`** (`scripts/ccpp_prebuild.py`): +- Simple, mostly procedural Python +- Used in: NOAA UFS Weather Model, Navy NEPTUNE, CCPP-SCM +- Extremely reliable in research, development, and operations +- Fewer capabilities; simpler design + +**`ccpp-capgen`** (`scripts/ccpp_capgen.py`): +- Highly complex, object-oriented Python taken to the extreme +- Used in: NCAR CAM-SIMA (still mostly a research/development model) +- Many advanced features designed but never implemented (funding/priority gaps) +- Notoriously difficult to develop; no remaining team member fully understands it + +**The original plan** was to update `ccpp-capgen` with missing features from `ccpp-prebuild` +and transition all models to it. **This plan has been abandoned** in favor of a complete +redesign that draws the best lessons from both generations. + +The immediate trigger for abandoning capgen was the failure — after considerable effort by +three developers — to make capgen pass DDT arguments to group caps the way prebuild does. +This is the root cause of capgen's severe performance problem (seconds for prebuild, +10+ minutes for capgen on the same suite set) and of its broken handling of optional +variables under Fortran compiler debugging flags. + +--- + +## 2. ccpp-prebuild — detailed analysis + +### 2.1 Command-line arguments and configuration + +Entry point: `scripts/ccpp_prebuild.py`, `main()`. + +Arguments parsed by `argparse`: + +| Argument | Required | Purpose | +|---|---|---| +| `--config` | yes | Path to host-model Python config module | +| `--suites` | no | Comma-separated suite names (without `.xml`) | +| `--builddir` | no | Override build directory from config | +| `--namespace` | no | Appended to static API module name | +| `--debug` | no | Insert Fortran array-size checks in generated caps | +| `--clean` | no | Remove generated files and exit | +| `--verbose` | no | Set logging to DEBUG | + +The `--config` file is a plain Python module imported dynamically via `importlib`. +Key variables it must define: + +| Config variable | Purpose | +|---|---| +| `VARIABLE_DEFINITION_FILES` | List of host-model Fortran sources with metadata hooks | +| `SCHEME_FILES` | List of physics scheme Fortran sources | +| `CAPS_DIR` | Output directory for generated cap `.F90` files | +| `SUITES_DIR` | Directory containing suite definition XML files | +| `STATIC_API_DIR` | Output directory for `ccpp_static_api.F90` | +| `TYPEDEFS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for typedef build snippets | +| `SCHEMES_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for scheme build snippets | +| `CAPS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for cap build snippets | +| `HTML_VARTABLE_FILE`, `LATEX_VARTABLE_FILE` | Documentation output paths | +| `TYPEDEFS_NEW_METADATA` | Optional: dict enabling DDT member name translation bridge | + +The config file can contain arbitrary Python expressions — computed file lists, +conditional logic, environment-variable lookups — making it very flexible. + +### 2.2 Step-by-step execution pipeline + +``` +1. Import config module dynamically via importlib + +2. gather_variable_definitions() + for each file in VARIABLE_DEFINITION_FILES: + parse_variable_tables(file) [metadata_parser.py] + → metadata_define: OrderedDict[standard_name → [mkcap.Var]] + +3. collect_physics_subroutines() + for each file in SCHEME_FILES: + parse_scheme_tables(file) [metadata_parser.py] + → metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] + → arguments_request: OrderedDict[scheme → OrderedDict[subroutine → [std_names]]] + → dependencies_request: OrderedDict[scheme → [abs_paths]] + → schemes_in_files: OrderedDict[scheme → abs_path] + +4. compare_metadata() [batch matching] + for each std_name in metadata_request: + check exists in metadata_define + check type/kind/rank compatibility + register unit conversions in var.actions + copy local_name as var.target + → metadata: OrderedDict[std_name → [Var]] (targets and actions set) + +5. check_optional_arguments() [warnings only] + +6. For each requested suite XML: + Suite.parse(xml) [mkstatic.py] → Suite + Group objects + Group.write() → ccpp___cap.F90 + Suite.write() → ccpp__cap.F90 + +7. API.write() [mkstatic.py] + → ccpp_static_api[_].F90 + +8. Write build-system snippets [mkcap.py writers] + → CCPP_CAPS.cmake/mk/sh + → CCPP_SCHEMES.cmake/mk/sh + → CCPP_TYPEDEFS.cmake/mk/sh + → CCPP_API.cmake/sh + +9. mkdoc.metadata_to_html() → HTML variable table + mkdoc.metadata_to_latex() → LaTeX variable table +``` + +### 2.3 Data structures — the "flat dict" model + +Everything in prebuild lives in flat Python `OrderedDict` structures. There is no object +hierarchy; variables are simple Python objects with plain attributes. + +```python +# Top-level data containers +metadata_define: OrderedDict[standard_name → [mkcap.Var]] # 1 Var per std_name +metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] # N Vars (one per scheme×subroutine) +arguments_request: OrderedDict[scheme_name → OrderedDict[subroutine_name → [std_names]]] +dependencies_request: OrderedDict[scheme_name → [abs_paths]] +schemes_in_files: OrderedDict[scheme_name → abs_path] +``` + +`mkcap.Var` attributes: + +| Attribute | Type | Description | +|---|---|---| +| `standard_name` | str | CF-convention unique identifier | +| `long_name` | str | Human-readable description | +| `units` | str | Physical units | +| `local_name` | str | Fortran local name (may be DDT member reference) | +| `type` | str | Fortran type (real, integer, logical, or DDT name) | +| `kind` | str | Fortran kind parameter | +| `dimensions` | list[str] | Dimension standard names | +| `intent` | str | in / out / inout | +| `active` | str | `'T'`, `'F'`, or expression string | +| `optional` | str | `'T'` or `'F'` | +| `pointer` | bool | Whether Fortran POINTER attribute needed | +| `target` | str | Set during matching: the host model local_name | +| `actions` | dict | `{'in': fn, 'out': fn}` for unit conversions | +| `container` | str | Encoded provenance: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | + +**Performance note on `container` and `target`**: these two attributes act as a lookup +cache computed once during the `compare_metadata()` batch step. The `container` string +encodes where each variable lives in the host model (module and, if applicable, the +DDT member chain). The `target` records the resolved Fortran local name. Both are +computed once and then used directly during Fortran cap generation — no further dictionary +lookups are needed. This is a major contributor to prebuild's speed advantage. + +### 2.4 Metadata parsing and the bridge to capgen + +`metadata_parser.py` is a shared module that acts as a bridge. It detects whether a +metadata section in a Fortran source file uses the old pipe-delimited format (deprecated, +warning emitted) or the new `.meta` format (triggered by `!! \htmlinclude .html` +in the Fortran source comment hook). + +For `.meta` files, `read_new_metadata()` in `metadata_parser.py`: +1. Calls capgen's `metadata_table.parse_metadata_file()` → `[MetadataTable]` +2. Converts each `metavar.Var` to a `mkcap.Var` +3. Normalizes `active` to `'T'`/`'F'`/expression, `optional` to `'T'`/`'F'` + +The `TYPEDEFS_NEW_METADATA` config variable (when provided) triggers an additional +pass via `convert_local_name_from_new_metadata()` which translates flat +standard-name-style local names into DDT member references such as +`Atm(blk_no)%q(:,:,:,graupel_index)`. This is the bridge that makes the newer +`.meta` format work with the older DDT-heavy host model code. + +### 2.5 Variable matching — `compare_metadata()` + +A single batch function processes all matching. For each standard name in `metadata_request`: + +1. Check it exists in `metadata_define` — error if missing +2. Check there is exactly one definition — error if ambiguous +3. Call `var.compatible(other_var)` — checks equality of `standard_name`, `type`, `kind`, and rank +4. Register unit conversions: if units differ, `var.convert_from()` / `var.convert_to()` + stores a conversion function in `var.actions` +5. Check `active` attribute: if host variable is conditionally allocated and scheme variable + is not `optional`, issue a warning (not an error) +6. Copy `local_name` from the define side as `var.target` +7. Build module use list from container strings + +Result: `metadata` dict where each `Var` has `.target` set to the host model local name +and `.actions` populated with any needed unit conversion functions. + +### 2.6 Generated Fortran files + +#### Group cap: `ccpp___cap.F90` + +One module per group. For each CCPP stage (tsinit, init, run, tsfinal, finalize), a subroutine: + +```fortran +module ccpp_suite_A_physics_cap + use scheme_module, only: scheme_run + use host_module_A, only: ddt_A ! DDT, not flat fields + use host_module_B, only: ddt_B + implicit none + contains + + subroutine suite_A_physics_run_cap(ddt_A, ddt_B, im, iaend, ierr, ...) + type(ddt_A_type), intent(inout), target :: ddt_A ! entire DDT passed + type(ddt_B_type), intent(inout), target :: ddt_B + integer, intent(in) :: im, iaend ! loop bounds + integer, intent(out) :: ierr + logical, save :: initialized(200) = .false. + ! optional variable: local pointer, conditionally associated + real(kind_phys), pointer :: opt_var(:) => null() + if (ddt_A%active_flag) then + opt_var => ddt_A%opt_field + end if + ! unit conversion: local variable + real(kind_phys) :: converted_var(im) + converted_var(:) = ddt_B%field(:im) * conversion_factor + ! fixed-index extraction: local pointer for a specific tracer + real(kind_phys), pointer :: q_water_vapor(:,:) => null() + q_water_vapor => ddt_A%q(:,:,ntqv) ! ntqv = water vapor index in tracer array + ! call scheme with loop-bound application and extracted variables at the call site + call scheme_run( & + arg1 = ddt_A%field1(1:im), & ! horizontal loop-bound applied here + arg2 = ddt_A%field2(1:im,:), & ! loop-bound + all levels + qv = q_water_vapor(1:im,:), & ! specific tracer, loop-bound applied + arg3 = converted_var, & ! unit-converted local var + opt_arg = opt_var, & ! optional pointer + ...) + if (ierr /= 0) return + end subroutine +end module +``` + +Key points: +- **DDTs are passed as arguments, not flat fields.** Hundreds of variables arrive as + one or a small number of DDT arguments. This is the fundamental architectural choice + that makes prebuild fast and safe with compiler debugging flags. +- **Two distinct "subsetting" operations happen at the scheme call site:** + 1. *Loop-bound application*: horizontal range `1:im` (or `im` for scalar extents) + applied in the scheme call argument expressions. + 2. *Fixed-index extraction*: a specific element along one dimension is selected, + e.g. `q_water_vapor => ddt%q(:,:,ntqv)` extracts the water vapor tracer from the + full tracer array. A local pointer (or local variable for unit conversions) is + declared just before the scheme call and passed as the scheme argument. The group + cap always receives the full data; these extractions are local to the group cap. +- **Optional variables** are handled by declaring a local `pointer` variable and + conditionally associating it with the DDT field based on the `active` expression. + An unassociated pointer is passed to the scheme if the variable is inactive. This + avoids compiler exceptions when mandatory debugging flags are enabled, because the + unallocated field is never directly referenced — only the already-null pointer is. +- `logical :: initialized(200), save` — per-instance initialization tracking. The + 200 is the maximum number of complete model instances that can coexist in memory + simultaneously (used in ensemble approaches where multiple copies of the full model + state live in memory at once). Each instance has its own initialization flag. +- For the `run` phase, `im` and `iaend` (or similar) carry `horizontal_loop_begin` + and `horizontal_loop_end`, enabling OpenMP thread-level parallelism where each + thread processes a horizontal slice. +- Explicit keyword argument passing in scheme calls. +- Unit conversion: a local variable is declared and populated before the call; the + local variable is then passed to the scheme. +- Error check after each scheme call; returns immediately on error. +- `--debug` flag inserts Fortran array-size assertions. + +#### Suite cap: `ccpp__cap.F90` + +Imports all group cap functions and exposes one function per stage that chains group calls. + +#### Static API: `ccpp_static_api[_].F90` + +A single Fortran module `ccpp_static_api` with one subroutine per stage: + +```fortran +subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) + character(len=*), intent(in) :: suite_name, group_name + select case(trim(suite_name)) + case('suite_A') + select case(trim(group_name)) + case('physics') + call suite_A_physics_run_cap(cdata, ierr) + ... + end select + ... + end select +end subroutine +``` + +This is the **single entry point** the host model calls. The host model passes `suite_name` +and `group_name` at runtime; the static API dispatches to the appropriate cap function. + +### 2.7 Build system snippet files generated + +Six output files (Makefile, CMakefile, shell source) for three variable sets: + +| File | Content | +|---|---| +| `CCPP_CAPS.cmake` | `set(CAPS /abs/path/cap1.F90 /abs/path/cap2.F90 ...)` | +| `CCPP_SCHEMES.cmake` | `set(SCHEMES /abs/path/scheme1.F90 ...)` | +| `CCPP_TYPEDEFS.cmake` | `set(TYPEDEFS module1 module2 ...)` (module names, not paths) | +| `CCPP_API.cmake` | `set(API /abs/path/ccpp_static_api.F90)` | + +All files are written as `.tmp` first and compared against the existing version; they are +replaced only if the content changed, which avoids unnecessary recompilation of downstream +Fortran targets. + +### 2.8 What `mkcap.py`, `mkstatic.py`, and `mkdoc.py` each do + +**`mkcap.py`**: +- Defines the `mkcap.Var` class (prebuild's variable data class) +- Defines six file-writer classes: `CapsMakefile`, `CapsCMakefile`, `CapsSourcefile`, + `SchemesMakefile`, `SchemesCMakefile`, `SchemesSourcefile`, `TypedefsMakefile`, + `TypedefsCMakefile`, `TypedefsSourcefile` +- Each writer has a `write(file_list)` method that produces a formatted include file +- Does NOT generate any Fortran + +**`mkstatic.py`**: +- Defines `Suite`, `Group`, `Subcycle` classes that parse suite definition XML and + generate Fortran caps +- `Suite.parse()`: reads SDF XML via `xml.etree.ElementTree`, builds `Group` and + `Subcycle` objects +- `Suite.write()`: drives cap generation for all groups and the suite-level cap +- `Group.write()`: generates the group cap Fortran — argument list construction, + module `use` statements, unit conversion code, scheme calls, error handling +- Defines `API` class: generates the static API Fortran module (suite_name/group_name + dispatch switch) +- `CCPP_SUITE_VARIABLES` dict: mandatory variables always included (error message, + error code, loop counter, loop extent) +- Helper functions `extract_parents_and_indices_from_local_name()` and + `extract_dimensions_from_local_name()` handle complex DDT member access like + `Atm(blk_no)%q(:,:,:,graupel_index)` — these are critical for DDT-heavy host models + +**`mkdoc.py`**: +- `metadata_to_html()`: produces an HTML table of all host-model provided variables + (standard_name, long_name, units, rank, type, kind, source, local_name) +- `metadata_to_latex()`: produces a LaTeX table combining host-defined and scheme-requested + variables, annotating which schemes use each variable and whether unit conversion is needed +- Informational outputs only; do not affect the build + +--- + +## 3. ccpp-capgen — detailed analysis + +### 3.1 Command-line arguments + +Entry point: `scripts/ccpp_capgen.py`, `_main_func()`. +Arguments parsed via `framework_env.parse_command_line()` into a `CCPPFrameworkEnv` object: + +| Argument | Required | Purpose | +|---|---|---| +| `--host-files` | yes | `.meta` files or `.txt` indirect file lists | +| `--scheme-files` | yes | Same format | +| `--suites` | yes | `.xml` SDF files or `.txt` lists | +| `--output-root` | no | Directory for generated files | +| `--host-name` | no | If given, generates a host cap | +| `--ccpp-datafile` | no | Path for datatable XML (default: `datatable.xml`) | +| `--kind-type` | no (repeatable) | Fortran kind mappings, syntax `=[:]`. Module defaults to `iso_fortran_env` for ISO_FORTRAN_ENV specs. Examples: `kind_phys=REAL64`, `kind_phys=my_host_kinds:kind_r8`. If omitted, `kind_phys=iso_fortran_env:REAL64` is injected. | +| `--preproc-directives` | no | Fortran preprocessor macros | +| `--use-error-obj` | no | Use error object instead of scalar error variables | +| `--force-overwrite` | no | Always regenerate output | +| `--clean` | no | Remove files listed in datatable and exit | +| `--verbose` | no (repeatable) | Increase log verbosity | + +`CCPPFrameworkEnv` (defined in `framework_env.py`) consolidates all settings into typed +properties and stores a `kind_dict` mapping CCPP kind names to `[kind_spec, module]` pairs. + +### 3.2 Step-by-step execution pipeline + +``` +1. create_file_list() + expand .txt indirect file lists, validate .meta extensions + +2. register_ddts(scheme_files) + pre-scan all scheme .meta files + register DDT type names via register_fortran_ddt_name() + (so the host parser can recognize them as non-intrinsic types) + +3. parse_host_model_files() + for each host .meta file: + metadata_table.parse_metadata_file() → [MetadataTable] + find_associated_fortran_file() → matching .F90 path + parse_fortran_file() → Fortran declarations (via fortran_tools) + check_fortran_against_metadata() → cross-validation (type, kind, rank, intent) + accumulate MetadataSection headers: DDT, module, host types + +4. HostModel(table_dict, host_name, run_env) + process DDT headers: → DDTLibrary + ddt_dict (VarDictionary) + process module/host headers: → main VarDictionary + __var_locations + add ConstituentVarDict synthetically for ccpp_model_constituents_t + +5. API(sdfs, host_model, scheme_headers, run_env) + for each SDF XML: + Suite construction: + auto-create 5 phase groups: register, initialize, timestep_initial, + timestep_final, finalize + parse elements → Group objects (RUN_PHASE_NAME) + parse / tags → Scheme objects in full-phase groups + Suite.analyze(host_model, scheme_library, ddt_library, run_env): + Group.analyze() → Scheme.analyze(): + for each scheme argument: + VarDictionary.find_variable() [scope chain search] + Var.compatible() [→ VarCompatObj with transformations] + loop dim substitution for _run phase + register constituent if constituent=True + variable promotion: group outputs → suite level if needed by later group + +6. ccpp_api.write(outdir, run_env) + suite cap .F90 per suite + group caps (embedded or separate) + host cap .F90 (if --host-name given) + ccpp_kinds.F90 + +7. generate_ccpp_datatable() → datatable.xml +``` + +### 3.3 Object hierarchy + +``` +API (ccpp_suite.py) + └── Suite (extends VarDictionary) [one per SDF XML] + parent → ConstituentVarDict (extends VarDictionary) + parent → API + ├── Group (suite_objects.py, extends VarDictionary) [one per ] + │ call_list: CallList (extends VarDictionary) + │ ├── Subcycle (suite_objects.py) + │ │ └── Scheme (suite_objects.py, extends SuiteObject) + │ └── Scheme (for full-phase groups: init, register, etc.) + └── (auto groups: register, initialize, timestep_initial, + timestep_final, finalize) + +HostModel (host_model.py, extends VarDictionary) + ├── ddt_lib: DDTLibrary + │ └── {ddt_name → MetadataSection} + ├── ddt_dict: VarDictionary (all DDT field variables, expanded) + └── loop_vars: VarDictionary (run-time dimension variables) + +VarDictionary (metavar.py) + ├── {standard_name → Var} + └── parent_dict → VarDictionary ← scope chain for find_variable() + +Var (metavar.py) + └── __prop_dict: {property_name → validated_value} + +VarDDT (ddt_library.py, extends Var) + └── __field: Var | VarDDT ← recursive DDT traversal chain +``` + +### 3.4 Variable matching — scope-chain and VarCompatObj + +Unlike prebuild's single batch `compare_metadata()`, capgen performs incremental, +scope-aware matching during the suite analysis phase. + +For each scheme argument in `Scheme.analyze()`: +1. `VarDictionary.find_variable(standard_name)` — searches scope chain: + local group dict → suite dict → ConstituentVarDict → host model dict +2. `Var.compatible(other, run_env)` returns a `VarCompatObj` — not a bool. + `VarCompatObj` carries: + - Whether the variables are equivalent (no transformation needed) + - Whether they are compatible with transformations (unit conversion, dimension + substitution, `top_at_one` flip) + - The reason for any incompatibility (for error messages) +3. For `_run` phase: `horizontal_dimension` is automatically substituted with + `horizontal_loop_begin:horizontal_loop_end` +4. For `constituent = True` variables: auto-registered in `ConstituentVarDict`; + allocation/management code is generated +5. Variable promotion: if a Group produces a variable needed by a later Group, it is + promoted to Suite-level scope + +`VarCompatObj` compatibility considers: +- Type equality +- Kind equality (with ISO kind aliases) +- Units compatibility (triggers unit conversion if compatible) +- Dimension substitutability (horizontal loop vs. full dimension, vertical extent) +- `top_at_one` orientation (triggers flip if needed) +- `protected` status (cannot be an output if protected) +- `CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` + from `var_props.py` + +### 3.5 `metavar.Var` properties + +`metavar.Var` stores all properties in a validated `__prop_dict`. Properties: + +**Specification properties** (all metadata contexts): + +| Property | Type | Notes | +|---|---|---| +| `local_name` | str | Valid Fortran identifier | +| `standard_name` | str | CF-convention, lowercase+underscores | +| `long_name` | str | Human-readable description | +| `units` | str | Physical units string | +| `dimensions` | list | Dimension standard names or `()` | +| `type` | str | Intrinsic or registered DDT name | +| `kind` | str | Fortran kind parameter | +| `active` | str | Conditional allocation expression | +| `optional` | bool | Whether scheme can handle missing var | +| `protected` | bool | Cannot be written by schemes | +| `allocatable` | bool | Has ALLOCATABLE attribute | +| `state_variable` | bool | Persists across timesteps | +| `persistence` | str | `timestep` or `run` | +| `default_value` | str | Fortran expression | +| `diagnostic_name` | str | Diagnostic output name | +| `target` | bool | Has TARGET attribute | +| `polymorphic` | bool | CLASS(*) type | +| `top_at_one` | bool | Vertical ordering: top at index 1 | + +**Scheme-only properties**: + +| Property | Type | Notes | +|---|---|---| +| `intent` | str | in / out / inout | + +**Constituent properties**: + +| Property | Type | Notes | +|---|---|---| +| `constituent` | bool | Is a CCPP-managed constituent (tracer) | +| `advected` | bool | Is advected by the dynamical core | +| `molar_mass` | float | Molecular weight (positive) | + +### 3.6 Capgen-only features + +**Fortran cross-validation** (`check_fortran_against_metadata()`): +- Parses the actual `.F90` file alongside the `.meta` file +- Checks that every metadata entry matches the real Fortran declaration: + variable count, local_name, type, kind, intent (for schemes), dimension rank/names +- Catches bugs where metadata was updated but the Fortran source was not (or vice versa) + +**State machine** (`ccpp_state_machine.py`, `state_machine.py`): +- `CCPP_STATE_MACH`: a `StateMachine` instance with 6 transitions +- Valid state sequence: `register → uninitialized → initialized → in_time_step` +- Suite caps include a `character(len=16) :: ccpp_suite_state` variable +- State-checking code at the start of each phase function enforces correct call ordering +- `CCPP_STATE_MACH.function_match()` uses compiled regex to identify which CCPP phase + a subroutine name belongs to + +**Constituent variable support** (`constituents.py`): +- `ConstituentVarDict` (extends `VarDictionary`) manages traceable species (tracers) +- When a scheme declares `constituent = True`, `find_variable()` auto-creates the variable +- Allocation code for the constituent array is auto-generated +- Constants: `CONST_DDT_NAME = "ccpp_model_constituents_t"`, + `CONST_PROP_TYPE = "ccpp_constituent_properties_t"` + +**DDT library** (`ddt_library.py`): +- `VarDDT(Var)`: represents a DDT field variable at any nesting level +- Traversal chain: `VarDDT → VarDDT → ... → Var` (innermost is the actual leaf field) +- `DDTLibrary`: dictionary of DDT `MetadataSection` objects +- `collect_ddt_fields()` expands DDT variables into component fields in `ddt_dict` + +**Host cap generation** (`host_cap.py`): +- Generated only when `--host-name` is given +- Produces `_ccpp_cap.F90` +- Subroutines: `_ccpp_physics_(api_vars)` + that call into suite cap functions + +**`ccpp_kinds.F90`**: +- Simple Fortran module `ccpp_kinds`. **Always generated**, even when no `--kind-type` + is supplied (in that case `kind_phys=iso_fortran_env:REAL64` is injected + automatically and an INFO log line is emitted). +- One `use , only: ` line per module (modules sorted; specs deduped per + module). Each kind is then re-exported as + `integer, parameter, public :: = `. +- Supports host-supplied kind modules: `--kind-type kind_phys=my_host_kinds:kind_r8` + emits `use my_host_kinds, only: kind_r8` and + `integer, parameter, public :: kind_phys = kind_r8`. +- Listed in `` of `datatable.xml` (matches original capgen) so + the build system picks it up via `ccpp_datafile.py --ccpp-files`. +- USEd by all generated Fortran files that declare kind-typed variables — the group + cap, the suite types module, and the suite data module. The static API and suite + cap have no kind references and do not USE it. + +**Datatable XML** (`ccpp_datafile.py`): +- Produced after generation; lists all generated files, scheme entries, variable properties, + suite configurations +- Queryable by the build system via `ccpp_datafile.py --suite-files` etc. +- Supports `--clean` workflow: reads the file list, removes all generated files, deletes itself +- `DatatableReport` class provides a programmatic query API + +**In-memory database** (`ccpp_database_obj.py`): +- `CCPPDatabaseObj`: wraps `HostModel` and `API` for programmatic access to capgen results +- Returned when `capgen()` is called with `return_db=True` +- Provides `host_model_dict()`, `suite_list()`, `constituent_dictionary(suite)` + +**Variable tracking tool** (`ccpp_track_variables.py`): +- Standalone diagnostic: traces a specific variable through a suite, showing which schemes + use it and with what intent +- Uses prebuild's `import_config` and capgen's `Suite`/`parse_metadata_file` together + +**Fortran-to-metadata tool** (`ccpp_fortran_to_metadata.py`): +- Standalone utility: parses annotated Fortran source files and generates skeleton `.meta` + files — used to bootstrap new scheme metadata + +--- + +## 4. Shared infrastructure + +### 4.1 Module sharing map + +| Module | Used by prebuild | Used by capgen | Notes | +|---|---|---|---| +| `metadata_parser.py` | yes | partial | **Bridge module**: calls capgen's parser, returns mkcap.Var | +| `metadata_table.py` | via bridge | yes (primary) | Native `.meta` format parser | +| `metavar.py` | no | yes | Primary `Var` class, `VarDictionary` | +| `var_props.py` | no | yes | `VariableProperty`, `VarCompatObj`, dimension constants | +| `mkcap.py` | yes | no | `mkcap.Var` class + build-snippet writers | +| `mkstatic.py` | yes | no | Suite/Group/API Fortran generators | +| `mkdoc.py` | yes | no | HTML/LaTeX documentation generators | +| `common.py` | yes | partial | `CCPP_STAGES`, container encoding | +| `framework_env.py` | dummy instance | yes (primary) | `CCPPFrameworkEnv` | +| `file_utils.py` | no | yes | `create_file_list`, `move_modified_files` | +| `code_block.py` | no | yes | Structured Fortran output | +| `ddt_library.py` | no | yes | `DDTLibrary`, `VarDDT` | +| `host_model.py` | no | yes | `HostModel` class | +| `host_cap.py` | no | yes | Host cap generation | +| `ccpp_suite.py` | no | yes | `Suite`, `API` classes | +| `suite_objects.py` | no | yes | `Scheme`, `Group`, `Subcycle`, `CallList` | +| `constituents.py` | no | yes | `ConstituentVarDict` | +| `ccpp_datafile.py` | no | yes | Datatable XML | +| `ccpp_database_obj.py` | no | yes | `CCPPDatabaseObj` | +| `ccpp_state_machine.py` | no | yes | `CCPP_STATE_MACH` | +| `state_machine.py` | no | yes | `StateMachine` base class | +| `ccpp_fortran_to_metadata.py` | no | yes | Fortran→metadata bootstrap tool | +| `ccpp_track_variables.py` | partial | partial | Uses both worlds | + +**The key architectural debt**: `metadata_parser.py` is a prebuild module that internally +calls capgen's `metadata_table.parse_metadata_file()` and converts results to `mkcap.Var` +objects. This creates a one-way dependency (prebuild → capgen's parser infrastructure) +while presenting a prebuild-style API to `ccpp_prebuild.py`. It exists only because +prebuild predates the `.meta` format. + +### 4.2 The `.meta` file format + +The `.meta` format is the native format for capgen and the expected format for all new +scheme development. The Fortran source file contains a comment hook pointing to the `.meta` +file: + +```fortran +!! \section arg_table_scheme_name_run Argument Table +!! \htmlinclude scheme_name_run.html +``` + +The `.meta` file itself uses an INI-style format: + +```ini +[ccpp-table-properties] + name = scheme_name + type = scheme + source_path = ../src + dependencies_path = ../some/path + dependencies = utility_module.F90, another.F90 + +[ccpp-arg-table] + name = scheme_name_run + type = scheme +[ im ] + standard_name = horizontal_loop_extent + long_name = horizontal loop extent + units = count + type = integer + dimensions = () + intent = in +[ dz ] + standard_name = layer_thickness + long_name = thickness of each model layer + units = m + type = real + kind = kind_phys + dimensions = (horizontal_loop_extent, vertical_layer_dimension) + intent = in +``` + +Multiple `[ccpp-arg-table]` sections are allowed in a scheme file (one per phase: +`_init`, `_run`, `_finalize`, `_timestep_init`, `_timestep_finalize`). +Singleton tables (DDT, module, host) allow only one section. + +The three table-level properties in `[ccpp-table-properties]` that carry path information: + +| Property | Purpose | Resolution | +|---|---|---| +| `source_path` | Relative path from the `.meta` file's directory to the directory containing the corresponding `.F90` Fortran source file | `os.path.normpath(os.path.join(meta_dir, source_path))`. Defaults to `meta_dir` when absent. | +| `dependencies_path` | Optional subdirectory relative to `meta_dir`; used as the base directory for resolving entries in `dependencies` | `os.path.normpath(os.path.join(meta_dir, dependencies_path))`. Defaults to `meta_dir` when absent. | +| `dependencies` | Comma-separated list of dependency file names or relative paths | Each entry resolved via `os.path.normpath(os.path.join(dep_base, entry))` where `dep_base` is the resolved `dependencies_path`. The value `none` is ignored. | + +**Implementation note — `flush_table_props` pattern:** The INI parser processes the +`[ccpp-table-properties]` and `[ccpp-arg-table]` headers in one streaming pass. Extra +table-level keys (`source_path`, `dependencies_path`, `dependencies`) are collected in a +`pending_props` dict alongside `name` and `type`. The parser must apply these properties +to the `MetadataTable` object — via a `flush_table_props()` call — at every +state-transition point (first `[ccpp-arg-table]` header, next `[ccpp-table-properties]` +header, and end-of-file) **before** resetting `pending_props`. Without this, the extra +properties are silently discarded. + +### 4.3 Variable property validation (`var_props.py`) + +`VariableProperty` encapsulates a single metadata property with its name, Python type, +optionality, default, valid-value constraints, and a check function. Check functions used: + +| Checker | What it validates | +|---|---| +| `check_local_name` | Valid Fortran identifier | +| `check_cf_standard_name` | Lowercase, underscores, alphanumeric only | +| `check_fortran_type` | Intrinsic type or registered DDT name | +| `check_units` | Valid unit string (normalizes `+` in exponents) | +| `check_dimensions` | Valid dimension specification | +| `check_default_value` | Valid Fortran expression | +| `check_molar_mass` | Positive float (for constituents) | + +`CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` +in `var_props.py` define the recognized dimension forms and the run-time substitution +map (e.g., `horizontal_dimension → horizontal_loop_begin:horizontal_loop_end`). + +--- + +## 5. Feature comparison + +| Feature | prebuild | capgen | Notes | +|---|---|---|---| +| **Input formats** | | | | +| Native `.meta` format | via bridge | yes | | +| Old pipe-delimited format | deprecated warn | not supported | | +| **Parsing and validation** | | | | +| Fortran source cross-validation | no | yes | capgen parses actual .F90 to cross-check | +| Preprocessor directive support | no | yes | `--preproc-directives` | +| **Variable handling** | | | | +| Variable data class | `mkcap.Var` (flat attrs) | `metavar.Var` (validated prop dict) | | +| Scope-chain variable search | no | yes | group→suite→constituent→host | +| Variable promotion group→suite | no | yes | | +| Unit conversion | yes | yes | | +| Optional/active variables | yes (fully) | yes | both: local pointer + conditional ASSOCIATE | +| DDT library (first-class) | no | yes | `VarDDT` recursive chain | +| **Suite and cap generation** | | | | +| Suite definition (SDF XML) | yes | yes | Same XML format | +| Subcycle loops | yes | yes | | +| State machine in generated caps | no | yes | Runtime state enforcement | +| Static API module (dispatch switch) | yes | no | `ccpp_static_api.F90` | +| Host cap generation | no | yes | `_ccpp_cap.F90` | +| `ccpp_kinds.F90` | no | yes | | +| **Constituent/tracer support** | | | | +| Constituent variable management | no | yes | Auto-allocation, `ConstituentVarDict` | +| **Build system output** | | | | +| CMake/Makefile file-list snippets | yes | no | Six snippet files | +| Datatable XML (queryable) | no | yes | `ccpp_datafile.py` | +| Clean via datatable | no | yes | | +| **Documentation** | | | | +| HTML variable table | yes | no (stub, raises error) | `mkdoc.metadata_to_html` | +| LaTeX variable table | yes | no | `mkdoc.metadata_to_latex` | +| **Developer tools** | | | | +| Variable tracking diagnostic | yes | no | `ccpp_track_variables.py` | +| Fortran-to-metadata bootstrap | no | yes | `ccpp_fortran_to_metadata.py` | +| **Runtime API** | | | | +| In-memory database object | no | yes | `CCPPDatabaseObj` | +| **Debug / developer aids** | | | | +| Debug array-size checks in caps | yes (`--debug`) | no | | +| Namespace suffix for API name | yes (`--namespace`) | no | | +| **Configuration** | | | | +| Config mechanism | Python module (flexible) | CLI args only | | + +**Known gaps and corrections:** + +- Capgen's `--generate-docfiles` is declared in the CLI but raises + `CCPPError("not yet supported")` — documentation generation is unimplemented. +- Prebuild handles `TYPEDEFS_NEW_METADATA` for mixed old/new metadata deployments; + capgen has no equivalent because it only accepts the new format. +- Capgen validates Fortran source against metadata; prebuild trusts metadata and never + reads Fortran code. +- Capgen has no `--namespace` equivalent for the generated API module name. +- Capgen's `CCPPDatabaseObj` and datatable XML allow programmatic querying; prebuild + has no equivalent. +- Prebuild's static API pattern (single Fortran module with runtime dispatch) is absent + from capgen, which uses a different host-cap integration model. +- **Capgen cannot pass DDTs to group caps** — it passes everything as flat fields. + Despite considerable effort by multiple developers, this has not been fixed. This is + the primary reason capgen is being abandoned. +- **Capgen does not support multiple model instances in memory** (ensemble approach). + Prebuild's `initialized(200)` array handles this correctly. +- **Capgen does not own or allocate any data.** Wait — this is a prebuild characteristic. + Capgen *does* allocate data for physics-internal variables (variables used only within + the physics, not provided by the host model) at the suite level. Prebuild requires the + host model to provide and own all data, including any physics-internal scratch space. + +--- + +## 6. Build system integration + +### 6.1 How a host model invokes ccpp-prebuild + +Direct call (as in the test suite): +```bash +python ../../scripts/ccpp_prebuild.py \ + --config=ccpp_prebuild_config.py \ + --builddir=build \ + --suites=suite_A,suite_B \ + [--debug] [--namespace mymodel] +``` + +Typical CMake integration: +```cmake +# Run prebuild at configure time +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_prebuild.py + --config=${HOST_CCPP_PREBUILD_CONFIG} + --builddir=${CMAKE_CURRENT_BINARY_DIR} + --suites=${CCPP_SUITES} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + RESULT_VARIABLE PREBUILD_RESULT +) +if(NOT PREBUILD_RESULT EQUAL 0) + message(FATAL_ERROR "ccpp_prebuild.py failed") +endif() + +# Consume the generated snippet files +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) # → ${CAPS} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) # → ${SCHEMES} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} +include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) # → ${API} + +add_library(ccpp_physics OBJECT ${CAPS} ${SCHEMES} ${API}) +``` + +### 6.2 How a host model invokes ccpp-capgen + +Direct call: +```bash +python scripts/ccpp_capgen.py \ + --host-files host_data.meta,host_model.meta \ + --scheme-files scheme1.meta,scheme2.meta \ + --suites suite_A.xml,suite_B.xml \ + --output-root ${BUILD_DIR}/ccpp \ + --host-name my_host \ + --kind-type kind_phys=REAL64 \ + --ccpp-datafile ${BUILD_DIR}/ccpp/datatable.xml +``` + +Typical CMake integration: +```cmake +# Run capgen at configure time +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_capgen.py + --host-files ${HOST_META_FILES} + --scheme-files ${SCHEME_META_FILES} + --suites ${SUITE_SDFS} + --output-root ${CMAKE_CURRENT_BINARY_DIR}/ccpp + --host-name ${HOST_MODEL_NAME} + --ccpp-datafile ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + RESULT_VARIABLE CAPGEN_RESULT +) + +# Query the datatable for generated file lists +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py + ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + --suite-files + OUTPUT_VARIABLE SUITE_CAPS OUTPUT_STRIP_TRAILING_WHITESPACE +) +execute_process( + COMMAND ${Python3_EXECUTABLE} + ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py + ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml + --host-files + OUTPUT_VARIABLE HOST_CAP OUTPUT_STRIP_TRAILING_WHITESPACE +) + +add_library(ccpp_physics OBJECT ${SUITE_CAPS} ${HOST_CAP}) +``` + +### 6.3 Available datatable query flags + +``` +--host-files → generated host cap .F90 files +--suite-files → generated suite cap .F90 files +--utility-files → generated utility .F90 files (e.g. ccpp_kinds.F90) +--ccpp-files → all generated .F90 files +--process-list → physics process types in the suite +--module-list → Fortran module names needed +--dependencies → scheme dependency files +--suite-list → configured suite names +--required-variables → variables required by all suites +--input-variables → input-only variables for a suite +--output-variables → output variables for a suite +--host-variables → variables provided by the host model +``` + +--- + +## 7. Key architectural differences + +### 7.1 Data model + +| Dimension | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| Variable representation | `mkcap.Var` with plain Python attributes | `metavar.Var` with validated `__prop_dict` | +| Variable storage | Two flat `OrderedDict`s | Scope-chain `VarDictionary` tree | +| Container encoding | Encoded string: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | Explicit class hierarchy | +| DDT handling | Encoded as string in `local_name`; helper regexes to extract | First-class `VarDDT` recursive chain | +| Variable matching | One batch `compare_metadata()` call | Incremental during suite analysis | +| Matching result | `bool` + side effects on `.target` / `.actions` | Rich `VarCompatObj` with transformation info | +| **Cap argument style** | **DDTs passed to group caps** | **Flat fields passed to group caps** | +| Subsetting location | At the scheme call site inside the group cap | Done at a higher level, before group cap | +| Data ownership | Host model owns all data including physics-internal | Capgen allocates physics-internal suite-level data | +| Multiple model instances | Yes — `initialized(200)` array, one flag per instance | No | +| Optional variable handling | Local pointer, conditionally associated | Same mechanism, but blocked by flat-field issue | + +### 7.2 Error handling + +| Aspect | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| Style | `(success, result)` tuples + `logging.error()` | `CCPPError` / `ParseInternalError` exceptions | +| Collection | Errors accumulate via `logging`; `main()` checks success | Raised immediately at point of detection | +| Location info | Filename from context; line numbers sometimes | `ParseContext` objects with file + line number | +| User errors vs bugs | Not distinguished | `CCPPError` (user) vs `ParseInternalError` (programmer) | + +### 7.3 Extensibility + +| Aspect | ccpp-prebuild | ccpp-capgen | +|---|---|---| +| New metadata property | Add to `VALID_ITEMS` dict + `mkcap.Var` attribute | Add one `VariableProperty` entry + checker fn | +| New CCPP phase | Update `CCPP_STAGES` + regenerate static API template | Add one transition tuple to `CCPP_STATE_MACH` | +| New compatibility rule | Modify `var.compatible()` in `mkcap.py` | Extend `VarCompatObj` in `var_props.py` | +| New host model | Write a new Python config file | New `.meta` files + CLI invocation | + +### 7.4 Performance + +Prebuild generates caps for multiple suites in seconds. Capgen, on the same suite set +with the same physics, takes more than 10 minutes. Two independent causes: + +**Cause 1 — Repeated scope-chain traversal.** Every variable lookup in capgen traverses +a five-level `VarDictionary` parent chain (group → suite → constituent dict → host model +→ DDT dict) for every scheme argument in every group in every suite. Prebuild's +`compare_metadata()` does one flat dict lookup per standard name, once, and caches the +result in `var.container` and `var.target`. All subsequent use during Fortran generation +reads these cached attributes directly. + +**Cause 2 — Flat-field cap arguments.** This is likely the dominant cost. Capgen resolves +every scheme argument down to its individual flat field, generates a `use` statement and +an explicit argument for each one, and emits them in the generated Fortran. A DDT with +200 fields becomes 200 individual argument declarations, 200 `use` statements, and 200 +argument positions in the scheme call. Prebuild passes the DDT itself — one argument, +one `use` statement — and then subsets at the call site. + +**Consequence for correctness.** Passing flat fields in capgen also breaks optional +variable handling under Fortran compiler debugging flags. When a field inside a DDT is +conditionally allocated (optional), passing it as a flat field requires dereferencing the +DDT to extract the field — which the compiler will flag as an error if debugging is on +and the field happens to be unallocated. Prebuild avoids this entirely by passing the +DDT and using a local pointer at the scheme call site. + +### 7.5 Team comprehension and maintainability + +This is the critical real-world difference. `ccpp-prebuild` is understood by the whole +team because it is procedural Python: you can read `ccpp_prebuild.py` top-to-bottom and +follow what happens. The data structures are flat dicts; the control flow is linear. + +`ccpp-capgen` has a five-level class hierarchy, scope-chain dictionary lookups, +`VarCompatObj` carrying transformation state, `ConstituentVarDict` as a pluggable +scope-chain node, and a `StateMachine` with regex-based dispatch. No remaining team +member fully understands all of it. Development is extremely slow and risky. + +The failed effort to make capgen pass DDTs instead of flat fields is the concrete proof +point: three developers spent considerable time and could not fix it without fully +understanding the interplay between `VarDDT`, `DDTLibrary`, `VarDictionary` scope chains, +and the Fortran writer. This is the proximate reason for the redesign. + +--- + +## 8. Design considerations for the redesign + +The following observations from this analysis should inform the redesign: + +### 8.1 What to keep from prebuild +- Procedural, top-down control flow — easy to read and debug +- Config file as a Python module — extremely flexible without adding CLI arguments +- The static API pattern (`ccpp_static_api.F90` with runtime suite/group dispatch) — + proven, simple integration for the host model +- **DDT arguments in group caps** — pass DDTs, not flat fields; this is the core correctness + and performance requirement +- **Subsetting at the scheme call site** — group caps always receive full data; loop-bound + application and fixed-index extraction happen in the individual scheme call expressions + or via a local variable/pointer declared just before the call +- **Optional variable pattern** — local pointer declared in the group cap, conditionally + associated based on the `active` expression, then passed to the scheme; this is safe + under all compiler debugging modes +- The `initialized(N)` per-instance tracking — handles multiple simultaneous model + instances in memory (ensemble approach); `N` is the max number of instances +- **Framework-owned data needs a simpler design** — capgen's variable promotion and + `ConstituentVarDict` scope-chain approach is too complex; a cleaner mechanism for + framework-allocated physics-internal data is needed (to be designed) +- HTML and LaTeX documentation generation +- The six CMake/Makefile/shell snippet output files — simple and direct (can be revisited) + +### 8.2 What to keep from capgen +- Native `.meta` file parsing (eliminate the `metadata_parser.py` bridge entirely) +- Fortran source cross-validation (`check_fortran_against_metadata()`) — catches real bugs +- Rich compatibility reporting (`VarCompatObj`-style) — better error messages +- `ccpp_kinds.F90` generation — important for portability +- Datatable XML as output accounting (strictly better than six include files) +- `--preproc-directives` support +- Constituent variable support (needed for CAM-SIMA) +- State machine enforcement (optional feature, but architecturally clean) + +### 8.3 What to eliminate +- The `mkcap.Var` / `metavar.Var` duality — one variable class, natively reading `.meta` +- The `metadata_parser.py` bridge module — it exists only because of the old format +- The scope-chain `VarDictionary` hierarchy — replace with flat, explicit lookup: + one host dict, one scheme dict; no parent-chain traversal +- The five-level class inheritance (Suite → VarDictionary → ParseSource → ...) +- `ConstituentVarDict` as a scope-chain node — a simple explicit constituent registry suffices +- Capgen's variable promotion (group → suite level) — this complexity exists only because + capgen allocates physics-internal data; if the host always owns all data, promotion + is unnecessary +- Capgen's flat-field cap generation — DDT arguments must be the foundation + +### 8.4 Framework-owned data — open design question + +Capgen's variable promotion mechanism (promoting a variable from group scope to suite scope +when a later group needs it) and the `ConstituentVarDict` complexity exist because capgen +allocates and manages physics-internal data — variables used only within the physics, +not visible to the host model. This capability is **wanted** in the redesign: the host +model should not have to declare and own scratch variables that are purely internal to the +physics. + +The problem is not the concept but the implementation. Capgen's approach — weaving +framework-allocated variables into the `VarDictionary` scope chain and promoting them +upward — produces the complexity that made capgen unmaintainable. + +**Open question for the redesign:** What is a simpler mechanism for the framework to +allocate, own, and pass physics-internal variables? Candidate approaches (to be evaluated +with real-world examples): + +- A completely separate, flat "framework data" dictionary, distinct from the host variable + lookup, populated during analysis and passed explicitly to the caps as a dedicated + argument (e.g., a framework-managed DDT or allocatable array container). +- A simplified promotion concept: variables are statically promoted to the widest scope + that needs them during the analysis phase, but stored in a simple flat dict rather than + via a scope-chain lookup. +- Constituent variables (tracers) as a special sub-case with their own well-defined + allocation interface, separate from generic physics-internal data. + +This question will be revisited once real-world examples clarify how many and what kind of +physics-internal variables actually need to be managed. + +### 8.5 Critical design decisions for the redesign prompt + +1. **DDT cap arguments are non-negotiable.** Group caps must receive DDTs. The entire + subsetting, optional-variable, and performance story depends on this. + +2. **Data ownership**: host-owns-all (prebuild model) vs. generator-allocates-internals + (capgen model). This single decision determines whether variable promotion and + suite-level allocation are needed. + +3. **Integration pattern**: static API (prebuild style, `suite_name` + `group_name` dispatch) + vs. host cap (capgen style, separate host-side Fortran glue). Models currently using + each pattern depend on it. + +4. **Config mechanism**: Python module (prebuild style, flexible) vs. pure CLI + file lists + (capgen style, scriptable). The Python module config is very powerful for complex models. + +5. **DDT member access parsing**: `extract_parents_and_indices_from_local_name()` and + `extract_dimensions_from_local_name()` in `mkstatic.py` handle expressions like + `Atm(blk_no)%q(:,:,:,graupel_index)`. The redesign needs a clean, explicit design for + parsing and emitting these — not an afterthought regex patch. + +6. **Output accounting**: datatable XML (capgen) is the right answer. The six CMake snippet + files (prebuild) are redundant and harder to extend. + +7. **Multiple model instances**: the redesign must preserve the `initialized(N)` pattern + or an equivalent. The value of `N` may need to be configurable. + +8. **Backward compatibility of generated Fortran interfaces**: real-world model examples + will define exactly which naming conventions, argument orders, and module structures the + host models depend on. + +### 8.6 Implementation decisions made during redesign + +The following decisions were made during implementation of `capgen-ng` and are recorded +here as amendments to the analysis above. + +**State machine parameters are local to each generated group cap module.** +The original redesign prompt described the integer state constants as coming from a +shared framework library module. In practice they are generated as `private` named +parameters directly inside each group cap module: + +```fortran +integer, parameter, private :: CCPP_GROUP_UNINITIALIZED = 0 +integer, parameter, private :: CCPP_GROUP_INITIALIZED = 1 +integer, parameter, private :: CCPP_GROUP_IN_TIMESTEP = 2 +``` + +This keeps generated files self-contained — no implicit dependency on a framework +runtime library at the caps level. The values are replicated across all generated group +cap files, but the names are the contract. + +**`source_path` is used by the validator, not the generator.** +The generator trusts metadata and never opens Fortran source files. `source_path` is +meaningful only to the standalone validator tool, which uses it to auto-discover the +`.F90` file paired with each `.meta` file (same base name, different directory). + +**`dependencies` paths are written to `datatable.xml`.** +The resolved absolute paths from each scheme's `dependencies` table-level property are +collected and written to the `` section of `datatable.xml`, sorted and +deduplicated. The CMake build system reads these via `ccpp_datafile.py` to add external +dependency files to the build graph. + +**Optional variable (pointer wrapper) implementation decisions.** +Optional arguments (Case 2 and Case 4) use per-suite Fortran derived types for pointer +wrappers. All unique `(type, kind, rank)` combinations needed by any optional arg across +all groups in a suite are collected and written to `ccpp__types.F90`. Each type +name is generated as `{type}_{kind}_rank{N}_ptr_type` (e.g. `real_kind_phys_rank1_ptr_type`). +Group cap modules `USE` this file. The types file is omitted entirely when no optional +args exist in the suite. The active condition for a pointer assignment is inherited from +the **host variable's** `active` attribute when the scheme itself specifies no `active`. + +**Character length (`len=N` / `len=*`) rules.** +Character kind declarations follow specific compatibility rules enforced by the resolver: + +- `len=*` in a **scheme** is always compatible with any host `len=` — assumed-length + dummy arguments accept any host-declared length. No transform is generated. +- Matching specific `len=N` in both host and scheme requires no transform (naturally equal). +- Mismatched specific lengths (`len=512` host vs `len=128` scheme) are a **metadata error**; + the scheme must declare `len=*` or match the defining metadata exactly. +- `len=*` in the **host** with a specific `len=N` in the scheme is also an error. + +The resolver raises `CCPPError` for the illegal cases. No kind transform is ever generated +for character variables — lengths are a Fortran compatibility constraint, not a unit conversion. + +**`source_path` is used by the validator, not the generator.** +The group cap's `state_alloc` subroutine always takes `number_of_instances` as an +explicit `intent(in)` integer argument — it never USEs any host module to obtain it. +The call chain is: `ccpp_init` → `_init` → each group's `state_alloc`. At each +level the argument is conditional: + +- **Multi-instance host** (`number_of_instances` declared in host metadata with local + name e.g. `ninstances`): + - `ccpp_init(suite_name, ninstances, errmsg, errflg)` — static API receives it + - `_init(ninstances, errmsg, errflg)` — suite cap threads it through + - `state_alloc(ninstances, errmsg, errflg)` — group cap allocates array of that size +- **Single-instance host** (no `number_of_instances` in host metadata): + - All three signatures omit the argument + - `state_alloc(1, errmsg, errflg)` — the literal `1` is passed + +State array **indexing** uses `instance_number`'s local name (e.g. `inst_num`) from +the control metadata. For single-instance hosts the literal `1` is used. `instance_number` +is injected into the group cap's `_init` and `_final` subroutine signatures even when no +scheme in those phases uses it — the state guard and state transition require it: + +```fortran +subroutine ccpp___init(inst_num, ...) + if (ccpp_group_state(inst_num) >= CCPP_GROUP_INITIALIZED) return + ! ... scheme _init calls ... + ccpp_group_state(inst_num) = CCPP_GROUP_INITIALIZED +``` + +This injection does **not** happen for `_run`, `_timestep_init`, or `_timestep_final` +unless a scheme in those phases explicitly requests `instance_number`. The suite cap's +`_physics_init` and `_physics_final` dispatch subroutines similarly pass +`instance_number` to the group cap calls when the host provides it. + +**Control variable validation — flat unconditional required set.** +All required control variables (`suite_name`, `horizontal_loop_begin`, `horizontal_loop_end`, +`thread_number`, `number_of_threads`, `number_of_physics_threads`, `ccpp_error_code`, +`ccpp_error_message`, `instance_number`) are unconditional — every host must declare all +of them. Single-threaded or single-instance models pass `1` or `''` for any they don't +actively use. The generator validates the complete set after parsing host metadata, +collects all missing-variable errors together, and halts before emitting any code. +`instance_number` in particular is NOT conditional on `instance_dimension` usage — +it is always required. + +**`group_name` is conditionally included, not in the required set.** +`group_name` is included in the static API signature only if the host declares it in +their `type=control` table. When absent, the cap calls all groups unconditionally. The +generator warns (not errors) if `group_name` is absent and any suite has multiple groups. +When present: a required (non-optional) character argument; `''` or `'all'` calls all +groups in order; any other value dispatches to the named group only. + +**`horizontal_loop_extent` eliminated; schemes always use `horizontal_dimension`.** +Scheme metadata always declares `horizontal_dimension` as the horizontal extent +dimension, regardless of phase. There is no `horizontal_loop_extent` standard name in +the new design. The distinction between run-phase chunked processing and full-domain +init/final processing is handled entirely at the host level — the host passes actual +chunk bounds to `ccpp_physics_run` and `1`/`ncols` to all other phases. The cap always +generates `(horizontal_loop_begin:horizontal_loop_end)` for scheme call-site array +slices. For suite-owned array allocation sizing, `horizontal_dimension` from the host +`type=host` table (module USE) is used directly. This separation means allocation +correctness does not depend on the host passing any particular control variable values. + +**Uniform signature across all `ccpp_physics_*` entry points.** +All five physics entry points (`ccpp_physics_init`, `ccpp_physics_timestep_init`, +`ccpp_physics_run`, `ccpp_physics_timestep_final`, `ccpp_physics_final`) share the +same control argument set. No per-phase signature variations. `horizontal_loop_begin` +and `horizontal_loop_end` are in scope for all phases — a `_init` scheme that declares +`horizontal_dimension` correctly receives `(lb:ub)` slicing just as a `_run` scheme +would, with the host responsible for passing the right values. + +--- + +## 9. Real-world example: CCPP Single Column Model (SCM) + +*Source:* `EXT/ccpp-scm/` — uses `ccpp-prebuild`. + +The SCM is a horizontally degenerate model (always `im = 1`, no OpenMP threading) but +it compiles the largest set of suites in the CCPP ecosystem, making it the most complete +real-world picture of what prebuild must handle. + +**Scale:** 63 suites, 257 scheme files (137 scheme entries in config, many containing +multiple modules), 300 generated cap files, ~1,200+ host model variables, ~550 optional +(conditionally active) variables. + +--- + +### 9.1 The `TYPEDEFS_NEW_METADATA` bridge — the DDT accessor map + +This is the most important SCM-specific configuration. It maps each DDT type name to the +Fortran expression used to access an instance of that type from the host model's top-level +scope. It is what allows the code generator to convert a `local_name` like `tgrs` (declared +inside `GFS_statein_type`) into the cap argument expression +`physics%Statein%tgrs(...)`. + +```python +TYPEDEFS_NEW_METADATA = { + 'GFS_typedefs': { + 'GFS_diag_type' : 'physics%Diag', + 'GFS_control_type' : 'physics%Model', + 'GFS_cldprop_type' : 'physics%Cldprop', + 'GFS_tbd_type' : 'physics%Tbd', + 'GFS_sfcprop_type' : 'physics%Sfcprop', + 'GFS_coupling_type': 'physics%Coupling', + 'GFS_statein_type' : 'physics%Statein', + 'GFS_radtend_type' : 'physics%Radtend', + 'GFS_grid_type' : 'physics%Grid', + 'GFS_stateout_type': 'physics%Stateout', + 'GFS_typedefs' : '', + }, + 'CCPP_typedefs': { + 'GFS_interstitial_type': 'physics%Interstitial(cdata%thrd_no)', + 'CCPP_typedefs' : '', + }, + 'scm_type_defs': { + 'physics_type': 'physics', + 'scm_type_defs': '', + }, + 'ccpp_types': { + 'ccpp_t' : 'cdata', + 'ccpp_types': '', + 'MPI_Comm': '', + }, + # ... plus 8 more entries for physics-side modules (machine, radsw_param, etc.) +} +``` + +**How it works:** For a variable with `local_name = tgrs` declared in `GFS_statein_type`, +the generator looks up `'GFS_statein_type'` in the map, finds `'physics%Statein'`, and +constructs the target as `physics%Statein%tgrs`. For the thread-indexed interstitial DDT, +`physics%Interstitial(cdata%thrd_no)%` is produced automatically. + +This dictionary is the **entire** mechanism by which the prebuild bridge converts flat +metadata into correct DDT-member accessor expressions. It is a hand-maintained workaround +that the redesigned generator must **eliminate**: all information needed to derive these +accessor expressions is already present in the CCPP metadata, provided the metadata storage +model is designed correctly to capture the DDT hierarchy and instance/thread indexing. + +--- + +### 9.2 Host model DDT structure + +``` +! Module-level variables accessible globally: +physics (type physics_type, from module scm_type_defs) +cdata (type ccpp_t, from module ccpp_types) +one (integer parameter = 1, from module ccpp_types) + +! physics_type contains: +physics%Model → GFS_control_type (control parameters: integers, logicals, 1D arrays) +physics%Statein → GFS_statein_type (input atmospheric state: 2D/3D real arrays) +physics%Stateout → GFS_stateout_type (output tendencies) +physics%Sfcprop → GFS_sfcprop_type (surface properties: 2D real arrays) +physics%Coupling → GFS_coupling_type (coupling fields) +physics%Grid → GFS_grid_type (grid geometry) +physics%Tbd → GFS_tbd_type (to-be-determined / miscellaneous) +physics%Cldprop → GFS_cldprop_type (cloud microphysics properties) +physics%Radtend → GFS_radtend_type (radiation tendencies) +physics%Diag → GFS_diag_type (diagnostic output arrays) +physics%Interstitial(1:thrd_cnt) → GFS_interstitial_type (per-thread scratch space) +``` + +The interstitial DDT is an array indexed by thread number. Even though the SCM is +single-threaded, all caps use `physics%Interstitial(cdata%thrd_no)` (i.e., index 1). +This is the pattern that enables OpenMP parallelism in the full UFS models. + +--- + +### 9.3 The horizontal dimension in the SCM + +The SCM uses a **chunked** horizontal loop even though `im = 1`. The chunk mechanism is: + +```fortran +chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) +chunk_end = physics%Model%chunk_end(cdata%chunk_no) +``` + +All 2D and 3D array slice expressions in caps use this pattern: +```fortran +physics%Statein%tgrs(chunk_begin:chunk_end, one:levs) +``` + +In the SCM, `chunk_begin = chunk_end = 1` always, but the pattern is general enough for +multi-column models. The `one` lower bound (a named integer constant = 1) is a framework +convention used consistently throughout all caps. + +--- + +### 9.4 Four categories of local variables in group caps + +Every group cap generates four categories of local variable declarations before its scheme +calls: + +**Category 1 — Loop bounds and scalars (always present):** +```fortran +integer :: chunk_begin, chunk_end +integer :: levs +chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) +chunk_end = physics%Model%chunk_end(cdata%chunk_no) +levs = physics%Model%levs +``` + +**Category 2 — Fixed-index extractions (tracer indices, surface-level slices):** + +For a tracer `qgrs(:,:,ntqv)`: +```fortran +! No local variable declared — the expression is used inline at the call site: +call scheme_run(qv = physics%Statein%qgrs(chunk_begin:chunk_end, one:levs, physics%Model%ntqv), ...) +``` + +For a surface-level slice `prsi(:,1)`: +```fortran +call scheme_run(prsi_sfc = physics%Statein%prsi(chunk_begin:chunk_end, 1), ...) +``` + +The fixed index may be a literal integer (`1`) or a runtime scalar variable from a DDT +field (`physics%Model%ntqv`). Both are inlined at the call site. + +**Category 3 — Optional variable pointer arrays:** + +One pointer-array type and one pointer-array variable are declared for each optional +variable. They are dimensioned by thread count: +```fortran +type :: real_kind_phys_rank2_ptr_arr_type + real(kind_phys), dimension(:,:), pointer :: p => null() +end type real_kind_phys_rank2_ptr_arr_type +type(real_kind_phys_rank2_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array +``` + +Before each scheme call that uses the variable, the condition is evaluated and the pointer +either associated or left null: +```fortran +if (physics%Model%lndp_type /= 0) then + sfc_wts_1_ptr_array(cdata%thrd_no)%p => & + physics%Coupling%sfc_wts(chunk_begin:chunk_end, one:physics%Model%n_var_lndp) +end if +``` + +Passed to the scheme as a keyword argument: +```fortran +call gfs_surface_generic_pre_run(..., sfc_wts=sfc_wts_1_ptr_array(cdata%thrd_no)%p, ...) +``` + +After the call, the pointer is nullified: +```fortran +if (physics%Model%lndp_type /= 0) then + nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) +end if +``` + +**Category 4 — Unit conversion local variables:** + +Not present in the SCM (GFS uses consistent SI units throughout). When present in other +models, a local array is declared, populated before the call, and passed as the argument: +```fortran +real(kind_phys) :: converted_var(chunk_begin:chunk_end) +converted_var(:) = physics%Statein%source_field(chunk_begin:chunk_end) * conversion_factor +call scheme_run(..., target_arg=converted_var, ...) +``` + +--- + +### 9.5 Array size checks + +Every array argument — mandatory or optional — has a size check immediately before the +scheme call. The check uses `size()` and computes the expected size from dimension variables: + +```fortran +! Mandatory variable — outer condition is always .true. +if (.true.) then + if (size(physics%Statein%tgrs(chunk_begin:chunk_end, one:levs)) /= & + (chunk_end-chunk_begin+1)*(levs-one+1)) then + write(cdata%errmsg, '(a,i8,a,i8)') & + 'Detected size mismatch for variable tgrs: expected ', expected, ' but got ', actual + ierr = 1 + return + end if +end if + +! Optional variable — outer condition mirrors the active= expression +if (physics%Model%lndp_type /= 0) then + if (associated(sfc_wts_1_ptr_array(cdata%thrd_no)%p)) then + if (size(sfc_wts_1_ptr_array(cdata%thrd_no)%p) /= expected_size) then + ...error... + end if + end if +end if +``` + +--- + +### 9.6 The `initialized(200)` array and instance management + +```fortran +logical, dimension(200), save :: initialized = .false. +``` + +`cdata%ccpp_instance` is a 1-based integer assigned to each independent CCPP state object. +In an ensemble, each ensemble member gets a different instance number (1–200). The `init_cap` +sets `initialized(cdata%ccpp_instance) = .true.` at the end of successful init. The +`run_cap` checks `if (.not. initialized(cdata%ccpp_instance))` and aborts with an error +if init was never called for that instance. The `final_cap` resets the flag to `.false.`. + +The value 200 is hardcoded — it is the maximum supported number of simultaneous model +instances. This could be made configurable. + +--- + +### 9.7 Suite and group cap hierarchy + +Three-level cap hierarchy: + +``` +ccpp_static_api.F90 (module ccpp_static_api) + → dispatches by suite_name + optional group_name + → owns physics, cdata, constants via module use + → calls suite-level caps: + +ccpp_scm_gfs_v16_cap.F90 (module ccpp_scm_gfs_v16_cap) + → aggregates arguments from all groups + → calls group caps in order per phase: + +ccpp_scm_gfs_v16_time_vary_cap.F90 (module ccpp_scm_gfs_v16_time_vary_cap) +ccpp_scm_gfs_v16_radiation_cap.F90 (module ccpp_scm_gfs_v16_radiation_cap) +ccpp_scm_gfs_v16_phys_ps_cap.F90 (module ccpp_scm_gfs_v16_phys_ps_cap) +ccpp_scm_gfs_v16_phys_ts_cap.F90 (module ccpp_scm_gfs_v16_phys_ts_cap) +``` + +Each level is a pure Fortran module. Argument passing is explicit keyword-argument style +at every level; no implicit global data (except in the static API, which uses `use`). + +--- + +### 9.8 Static API: module-level variable ownership + +The static API module uses all host-model modules and accesses their variables at module +scope. It does **not** take host data as subroutine arguments — instead it fills the +group cap arguments from its own module-use-associated variables: + +```fortran +module ccpp_static_api + use scm_type_defs, only: physics + use ccpp_types, only: cdata, one + use scm_physical_constants, only: con_g, con_pi, con_t0c, ... + use gfs_typedefs, only: ltp + use ccpp_scm_gfs_v16_cap, only: scm_gfs_v16_run_cap, ... + ... +contains + subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) + ! cdata passed in, others accessed from module scope + select case (to_lower(trim(suite_name))) + case ('scm_gfs_v16') + if (present(group_name)) then + select case (to_lower(trim(group_name))) + case ('phys_ps') + ierr = scm_gfs_v16_phys_ps_run_cap(one=one, physics=physics, cdata=cdata, ...) + ... + end select + else + ierr = scm_gfs_v16_run_cap(one=one, physics=physics, cdata=cdata, ...) + end if + case ('scm_gfs_v17_p8') + ... + end select + end subroutine +end module +``` + +This design means the static API file must be recompiled whenever any host-model module +changes (because it `use`s them), and it must be regenerated whenever suites change. +Its location in the **source tree** (not build tree) is a deliberate SCM design choice: +the file is committed to the repository as a generated artifact. + +--- + +### 9.9 Build system + +Prebuild runs at **cmake configure time** via `execute_process()`, before any compilation +starts. This is unusual but simplifies the cmake dependency graph. + +```cmake +execute_process( + COMMAND ccpp/framework/scripts/ccpp_prebuild.py + --config=ccpp/config/ccpp_prebuild_config.py + --suites=${CCPP_SUITES} + --builddir=${CMAKE_CURRENT_BINARY_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../.. + OUTPUT_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.out + ERROR_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.err +) +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_CAPS.cmake) # → ${CAPS} +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_SCHEMES.cmake) # → ${SCHEMES} +include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} +include(scm/src/CCPP_STATIC_API.cmake) # → ${API} +``` + +**Suite selection:** If `CCPP_SUITES` is not set by the user, a helper script +`suite_info.py` selects a compiler-appropriate subset. The full set of 63 suites is +used for production; subsets speed up development builds. + +--- + +### 9.10 Observations relevant to the redesign + +1. **`TYPEDEFS_NEW_METADATA` is a workaround that the redesign must eliminate.** The + DDT accessor information (which type lives at which accessor path) can be fully derived + from the CCPP metadata itself, given a well-designed metadata storage model. The + redesign must derive DDT accessor expressions automatically from the metadata rather + than requiring a separate hand-maintained dictionary. This is one of the primary + motivations for the new metadata storage design. + +2. **Three-level cap hierarchy (group → suite → static API) should be preserved.** + It provides clean separation: group caps are independently testable, suite caps + aggregate phases, the static API is the single host-callable entry point. + +3. **The static API's module-level `use` of host data is model-specific.** In models + where host data is not module-level (e.g., passed as subroutine arguments), the + static API pattern changes. The SCM is the simplest case because `physics` and `cdata` + are global module variables. + +4. **Instance and thread indexing are two orthogonal dimensions of host data access.** + Host model data uses two distinct indexing patterns that must be handled correctly: + + - **Regular state data** (Statein, Stateout, Sfcprop, etc.): dimensioned by instance + number — `physics%Statein(ccpp_instance_number)%array(1:horizontal_dimension, 1:vertical_dimension, ...)`. + In models supporting multiple in-memory model instances (ensemble), the top-level + DDT is an array indexed by `cdata%ccpp_instance`. + + - **Interstitial (per-thread scratch) data**: dimensioned by both instance and thread — + `physics%Interstitial(ccpp_instance_number, ccpp_thread_number)%array(1:horizontal_loop_extent, ...)`. + Critically, the horizontal dimension of interstitial arrays is sized to + `horizontal_loop_extent` (one OpenMP thread's chunk), not `horizontal_dimension` + (the full column count). `max_number_of_threads` instances are allocated per model + instance. Interstitial data can only be used during the **run phase** — this is a + known limitation of ccpp-prebuild that the redesign should address or at minimum + preserve explicitly. + +5. **Optional variable pointer arrays dimensioned by thread count** are the current + solution to thread-safe optional variable handling. This pattern is verbose (one + derived type + one array per optional variable per cap function) but correct. + The redesign could simplify this. + +6. **~550 optional variables in this model.** Optional/conditional variables are not + a corner case — they are a first-class feature. The redesign must handle them + efficiently and correctly. + +7. **Array size checks are debug-only and should not appear in the redesign by default.** + In prebuild they are only generated when the `--debug` flag is passed. The redesigned + generator should not produce them in normal mode — out-of-bounds access is caught at + runtime by compiler flags (e.g., `-fcheck=bounds` with gfortran, `-check bounds` with + ifort). The 12,991-line group cap is partly a consequence of generating these checks + unconditionally in the debug mode artifact examined here. + +8. **No unit conversions appear in GFS/SCM.** Unit conversion infrastructure must be + present in the redesign but the GFS physics package is self-consistent in units. + Unit conversions are more relevant for other host models. + +9. **The `one` constant** (integer parameter = 1) is passed as an explicit argument + everywhere and used as the lower bound in all array slices. This is a framework + convention. The redesign should decide whether this convention is preserved or + whether array lower bounds are handled differently. + +10. **Subcycles produce actual Fortran `do` loops inside the generated group cap.** + The loop from `1` to `cdata%loop_max` is generated directly in the cap function, + not left to the host model: + ```fortran + cdata%loop_max = 2 + do cdata%loop_cnt = 1, cdata%loop_max + call scheme_A_run(...) + if (ierr /= 0) return + call scheme_B_run(...) + if (ierr /= 0) return + end do + ``` + `cdata%loop_max` is set at the start of the subcycle block (from the `loop=` attribute + in the SDF XML) and `cdata%loop_cnt` is the current iteration counter, both visible + to schemes via the `ccpp_t` DDT. + +--- + +## 10. Real-world example: CAM-SIMA (capgen) + +*Source:* `EXT/cam-sima/` — uses `ccpp-capgen`. + +CAM-SIMA is the only model currently using capgen. It is still primarily a research model. +Unlike the SCM it uses a full 3D grid with OpenMP parallelism, but exposes host model +data as flat module variables rather than DDTs in the metadata layer. This example reveals +both what capgen can do and where it fundamentally fails. + +**Scale:** 1 suite (`cam7`), 2 run groups (`physics_before_coupler`, `physics_after_coupler`), +~75 scheme calls, 18 host `.meta` files, 893-line host cap, 2865-line suite cap. + +--- + +### 10.1 Suite structure + +`suite_cam7.xml` has two groups and no subcycles: + +| Group | Schemes (approx.) | Purpose | +|---|---|---| +| `physics_before_coupler` | 52 scheme calls | Cloud fraction, energy checks, dry adiabatic adjustment, Zhang-McFarlane deep convection full cycle, constituent tendency application | +| `physics_after_coupler` | ~20 scheme calls | Tropopause diagnostics, gravity wave drag (7 parameterizations + diagnostics), tendency application, energy consistency | + +CCPP phases in use: register, initialize, timestep_initial, run (per group), timestep_final, finalize. + +--- + +### 10.2 Host model variable structure + +**18 host `.meta` files, all of type `module`.** There are no `host` or `ddt` table types +anywhere. All host variables are flat scalars or arrays in Fortran modules. + +CAM-SIMA does **not** expose its physics DDTs (`phys_state`, `phys_tend`, `cam_in`, etc.) +through metadata. These types exist in `physics_types.F90` but have no `.meta` file. +The generated host cap accesses them directly via `use physics_types, only: phys_state, ...` +and passes individual DDT members as flat keyword arguments: +```fortran +! In cam_ccpp_cap.F90 — direct access to non-metadataized DDT members: +call cam7_physics_before_coupler(..., pint=phys_state%pint, t=phys_state%t, & + dtdt_total=phys_tend%dtdt_total, landfrac=cam_in%landfrac, ...) +``` + +This means capgen has no knowledge of how host data is structured. The host cap is +partly machine-generated and partly depends on manually wiring non-metadataized sources. +**This is a fundamental architectural gap** — changes to `physics_types` are invisible +to the framework. + +**Key host variables by module:** + +| Module | Key variables | +|---|---| +| `physics_grid` | `columns_on_task` (horizontal_dimension), `col_start`, `col_end`, lat, lon, area | +| `vert_coord` | `pver` (vertical_layer_dimension), `pverp` (vertical_interface_dimension) | +| `physconst` | ~35 physical constants, all `protected = True` | +| `cam_constituents` | `num_advected` (count of advected tracers) | +| `spmd_utils` | `mpicom`, `masterproc`, `npes`, `iam` | + +No instance indexing (`physics(1)`) and no thread-indexed DDTs appear — CAM-SIMA uses +a fundamentally different data model from the GFS/SCM stack. + +--- + +### 10.3 The two-cap architecture + +Capgen generates two distinct Fortran files: + +**`cam_ccpp_cap.F90` — the host cap (893 lines)** +- Module `cam_ccpp_cap` +- Imports non-metadataized host variables directly via `use physics_types`, `use physconst`, etc. +- Manages the constituent object (`ccpp_model_constituents_t`) — registration, initialization, gather/scatter, index lookup +- Public subroutines: `cam_ccpp_physics_run`, `cam_ccpp_physics_initialize`, etc. — the entry points the host model calls +- Dispatches to the suite cap, passing ~61–76 flat keyword arguments + +**`ccpp_cam7_cap.F90` — the suite cap (2865 lines)** +- Module `ccpp_cam7_cap` +- No host-specific imports — knows nothing about `physics_types`, `phys_state`, etc. +- All arguments are flat scalars and arrays, fully matched to metadata standard_names +- Contains all scheme calls, suite-level persistent variables, local temporaries, state machine +- The suite cap could in principle be used with any host model that provides the same standard names + +This two-cap split is **architecturally correct**: it separates host-specific binding +from physics-neutral dispatch. The redesign should preserve this separation. + +--- + +### 10.4 The flat-field argument problem — concrete evidence + +The run-phase subroutines expose the core problem with capgen's approach directly: + +```fortran +subroutine cam7_physics_before_coupler(errflg, errmsg, col_start, col_end, pver, dtime, & + gravit, pint, te_ini_dyn, teout, amiroot, iulog, ptend_s, temp, dtdt_total, cpair, & + lagrang, layer_surf, layer_toa, interface_surf, interface_toa, ncnst, piln, pmid, pdel, & + rpdel, qv, carr, cprops, rair, zvir, zi, zm, cp_or_cv_dycore, u, v, pintdry, phis, & + te_cur_phys, te_cur_dyn, tw_cur, latice, latvap, energy_formula_physics, & + energy_formula_dycore, cappa, q_tend, const_tend, qmin, pverp, cpwv, cpliq, rh2o, lat, & + long, pblh, mcon, tpert, dlf, rprd, ql, rliq, landfrac, cpair3, ttend_dp, tmelt, & + top_lev, ke, ke_lnd, cldfrc, domomtran, momcu, momcd, il1g, nstep, & + dudt_total, dvdt_total, fracis, dpdry, ps) +``` + +**61 dummy arguments for one group cap.** `physics_after_coupler` has 76. These are +individual flat arrays and scalars — no DDT in sight. This is exactly the problem that +three developers failed to fix: in the GFS/UFS context, this would be 1,200+ arguments. +The GFS physics stack simply cannot be connected to capgen in its current form. + +In contrast, the prebuild equivalent for the same data would pass `physics` (one DDT argument) +and `cdata` — two arguments covering hundreds of variables. + +--- + +### 10.5 Suite-level persistent variables — the framework-owned data pattern + +The suite cap allocates and owns arrays that persist across group calls within a timestep. +These are allocated in `cam7_initialize` and deallocated in `cam7_finalize`: + +```fortran +! Suite-level persistent (allocated in initialize, freed in finalize): +real(kind_phys), allocatable :: windu_tend(:,:) ! GW drag u-tendency accumulator +real(kind_phys), allocatable :: windv_tend(:,:) ! GW drag v-tendency accumulator +real(kind_phys), allocatable :: scaling_dycore(:,:) ! energy scaling factor +real(kind_phys), allocatable :: tend_te_tnd(:) ! energy tendency accumulator +real(kind_phys), allocatable :: tend_tw_tnd(:) ! water tendency accumulator +real(kind_phys), allocatable :: temp_ini(:,:) ! temperature saved at timestep start +real(kind_phys), allocatable :: z_ini(:,:) ! height saved at timestep start +real(kind_phys), allocatable :: flx_vap(:), flx_cnd(:), flx_ice(:), flx_sen(:) +logical, allocatable :: doconvtran(:) ! per-constituent convection flag +type(coords1d) :: p ! pressure coordinate DDT for GW drag +``` + +These are physics-internal variables — the host model does not know about them, does not +own them, and does not need to. This is the capgen "data ownership" model: the suite cap +is the data owner for variables that only matter within the physics. + +**This pattern is correct and desirable.** The complexity in capgen comes not from the +concept but from how these variables are discovered during analysis (scope-chain promotion) +and passed around (via VarDictionary). The redesign needs a simpler mechanism to achieve +the same result: statically enumerate physics-internal variables during analysis and have +the suite cap own them as named allocatables. + +During the run phase, suite-level persistent arrays are subsetted when passed to schemes: +```fortran +call gw_common_run(..., windu_tend=windu_tend(col_start:col_end, 1:pver), ...) +``` + +Run-phase local temporaries (e.g., `cape`, `cme`, `mu`, `md`) are allocated at function +entry and deallocated at exit: +```fortran +allocate(cape(col_start:col_end)) +... +call zm_convr_run(..., cape=cape, ...) +... +deallocate(cape) +``` + +These temporaries use `col_start` as the lower bound so that assumed-shape dummy arguments +in schemes see a 1-based array — a subtle but important detail. + +--- + +### 10.6 Horizontal chunking model + +CAM-SIMA uses `col_start`/`col_end` (passed as arguments to every run subroutine) to +define the current horizontal chunk: + +```fortran +ncol = col_end - col_start + 1 +``` + +Schemes declare `horizontal_loop_extent` and receive `ncol`. The horizontal dimension +in the host (storage dimension) is `columns_on_task`. The subsetting from storage to +loop extent happens at the boundary between host cap and suite cap — the host cap +passes the right subsections: + +```fortran +! In cam_ccpp_cap.F90: +call cam7_physics_before_coupler(..., col_start=col_start, col_end=col_end, & + pmid=phys_state%pmid, ...) ! full arrays passed; suite cap subsets internally +``` + +Inside the suite cap, persistent arrays are subsetted explicitly when passed to schemes: +```fortran +windu_tend(col_start:col_end, 1:pver) +``` +Local temporaries allocated as `allocate(cape(col_start:col_end))` are already +correctly sized and passed as assumed-shape `(:)`. + +--- + +### 10.7 State machine + +The suite cap has a character module variable tracking lifecycle state: + +```fortran +character(len=16) :: ccpp_suite_state = 'uninitialized' +``` + +Transitions: `uninitialized` → register → `uninitialized` → initialize → `initialized` +→ timestep_initial → `in_time_step` → (run, no state change) → timestep_final → +`initialized` → finalize → `uninitialized`. + +Each phase entry point checks the expected prior state: +```fortran +if (trim(ccpp_suite_state) /= 'in_time_step') then + errflg = 1 + write(errmsg, '(3a)') "Invalid initial CCPP state, '", trim(ccpp_suite_state), & + "' in cam7_physics_before_coupler" + return +end if +``` + +Non-run phases also include an OpenMP thread guard: +```fortran +#ifdef _OPENMP + if (omp_get_thread_num() > 1) then + errflg = 1 + errmsg = "Cannot call initialize routine from a threaded region" + return + end if +#endif +``` + +The state machine is simple, complete, and useful. The redesign should preserve it. + +--- + +### 10.8 Constituent variable handling + +CAM-SIMA demonstrates the full constituent lifecycle: + +```fortran +! In cam_ccpp_cap.F90: +type(ccpp_model_constituents_t), target :: cam_constituents_obj + +! Registration (scheme-declared constituents): +call suite_cam7_constituents_num_consts(num_consts) +call suite_cam7_constituents_const_name(iconst, const_name) +call cam_constituents_obj%new_field(const_name, ...) + +! Initialization (host-declared constituents like water vapor): +cam_model_const_stdnames(1) = "water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water" +call cam_constituents_obj%new_field(cam_model_const_stdnames(1), ...) + +! Per-timestep gather from host: +call cam_ccpp_gather_constituents(phys_state%q, ...) + +! Passing to suite cap: +call cam7_physics_before_coupler(..., + qv = cam_constituents_obj%vars_layer(:, :, cam_model_const_indices(1)), + carr = cam_constituents_obj%vars_layer, + cprops = cam_constituents_obj%const_metadata, ...) + +! Per-timestep scatter back to host: +call cam_ccpp_update_constituents(phys_state%q, ...) +``` + +The suite cap sees constituents as: +- `carr(:,:,:)` — the full rank-3 constituent array (ncol, nlev, ncnst) +- `qv(:,:)` — water vapor slice extracted in the host cap: `cam_constituents_obj%vars_layer(:,:,cam_model_const_indices(1))` +- `cprops(:)` — array of `ccpp_constituent_prop_ptr_t` metadata objects +- `doconvtran(1:ncnst)` — suite-level logical array set by scheme init indicating which constituents are convected + +This constituent API is sophisticated and worth preserving or improving in the redesign. + +--- + +### 10.9 Known defects in the capgen output + +**Repeated scheme init/final calls.** Capgen generates one init call per occurrence of +a scheme name in the XML, without deduplication: +- `qneg_init` called 5 times (once per `qneg` entry in the suite XML) +- `qneg_timestep_final` called 5 times +- `check_energy_chng_init` called twice +- `save_ttend_from_convect_deep_timestep_init` called 3 times + +If these routines have internal state, allocations, or side effects, this is a correctness +defect. The redesign must deduplicate init/final calls by unique scheme name. + +**Unit conversion embedded silently in the cap.** Before `zm_conv_convtran_run`: +```fortran +dpdry_local(:,1:pver) = 1.0E-2_kind_phys * dpdry(:,1:pver) ! Pa → hPa +``` +This is generated from the metadata units mismatch but appears as an opaque transform +in the cap. The redesign should make this visible (e.g., a comment naming the standard +name, the source units, and the target units). + +--- + +### 10.10 Build system — capgen invocation + +Capgen is invoked from Python (`cam_autogen.py`), not from cmake: + +```python +from ccpp_capgen import capgen +capgen_db = capgen(run_env, return_db=True) +``` + +This is a programmatic API call, not a subprocess. The `CCPPDatabaseObj` returned +(`capgen_db`) is then used directly in Python to query scheme lists, constituent names, +and file paths — avoiding the datatable XML query step that cmake-based invocations need. + +Output files consumed by the build: +- `cam_ccpp_cap.F90` — compiled into the atmosphere component +- `ccpp_cam7_cap.F90` — compiled into the atmosphere component +- `ccpp_kinds.F90` — compiled into the atmosphere component +- Utility files from `ccpp_framework/src/` (copied to build dir) +- `ccpp_datatable.xml` — queried by the build system for file lists + +--- + +### 10.11 Observations relevant to the redesign + +1. **The two-cap split (host cap + suite cap) is the right architecture.** It cleanly + separates host-specific binding from physics-neutral dispatch. The redesign must + preserve this. + +2. **Flat-field arguments in the suite cap are the critical failure.** 61–76 dummy + arguments per run subroutine is already large for a research model; for UFS/GFS + with 1,200+ variables it is completely infeasible. The redesign must pass DDTs. + +3. **The CAM-SIMA host does not use DDTs in metadata.** All host variables are flat + module variables. This is a fundamentally different host model architecture from + GFS/SCM. The redesign must support both styles: flat-module hosts (CAM-SIMA) and + deep-DDT hosts (GFS/SCM). + +4. **Non-metadataized variables hardwired into the host cap is a serious gap.** + `phys_state`, `phys_tend`, `cam_in` from `physics_types` have no `.meta` files. + The host cap accesses them directly. This means the framework cannot verify or + track these variables. The redesign should either require full metadata coverage + or have an explicit mechanism for declaring non-metadataized pass-through variables. + +5. **Suite-level persistent variables (framework-owned data) work well in practice.** + `windu_tend`, `scaling_dycore`, `temp_ini`, etc. are owned by the suite cap, invisible + to the host, and persist across group calls. This is the right pattern for + physics-internal state. The redesign needs this but with a simpler discovery mechanism + than capgen's scope-chain promotion. + +6. **Deduplicate init/final calls.** The redesign must deduplicate `_init`, `_finalize`, + `_timestep_init`, and `_timestep_final` calls by unique scheme name (not by occurrence + in the XML). + +7. **The constituent API in the host cap is comprehensive.** The `ccpp_model_constituents_t` + object with its register/init/gather/scatter/index API is sophisticated and should be + preserved or improved. + +8. **The suite-variables introspection subroutine** (`ccpp_physics_suite_variables`, + enumerating 83 standard names as inputs/outputs) is a useful capability for build + system integration and should be in the redesign. + +9. **The programmatic Python API** (`capgen(run_env, return_db=True)`) is valuable + for hosts like CAM-SIMA that invoke the generator from Python. The redesign should + support both CLI and programmatic invocation. + +10. **Unit conversions must be annotated in the generated cap**, not silently embedded + as magic-number multiplications. A comment with source units, target units, and the + standard name involved is the minimum. + +11. **The horizontal chunking model** (`col_start`/`col_end` as explicit arguments, + `ncol = col_end - col_start + 1` computed at entry) works and is clean. Suite-level + persistent arrays are allocated full-size and subsetted at call sites. + +12. **No optional variables in this model.** CAM-SIMA does not exercise optional/active + variable handling. This feature must be in the redesign but is not demonstrated here. + +--- + +## 11. Real-world example: UFS Weather Model (prebuild) + +The UFS Weather Model is the most complex and production-critical of the three examples. +It is a fully-coupled, 3-D operational NWP model. The CCPP physics is used in the +atmospheric component (`UFSATM`). Unlike the SCM (column model, process-split only) and +CAM-SIMA (capgen, flat-field arguments), UFS uses prebuild in a 3-D blocked/threaded +configuration that is architecturally distinct from both prior examples. + +The two suites analyzed here are: +- `FV3_GFS_v17_coupled_p8` — the primary operational GFS suite +- `FV3_GFS_v17_coupled_p8_ugwpv1` — a variant replacing `unified_ugwp` with `ugwpv1` + +The ugwpv1 suite is structurally identical to the base suite except for the `phys_ps` +group (4 extra scheme calls), so all observations below apply to both. + +--- + +### 11.1 Suite structure + +The primary suite has 5 groups: + +| Group | Subcycles | Scheme calls | Phase called | +|-------|-----------|-------------|--------------| +| `time_vary` | 1 | 4 | timestep_init (domain-level, no blocking) | +| `radiation` | 1 | 8 | run (block/thread loop) | +| `phys_ps` | 3 (loop=1, loop=2, loop=1) | 21 | run (block/thread loop) | +| `phys_ts` | 3 (loop=1, loop=1, loop=1) | 12 | run (block/thread loop) | +| `stochastics` | 1 | 2 | run (block/thread loop) | + +The `time_vary` group is the only one called at timestep_init/finalize. All other groups +are called from the run phase via the OpenMP blocked loop. This is a fundamentally +different usage pattern from SCM (which runs everything sequentially) and CAM-SIMA +(which has no run phase at all for the groups analyzed). + +The `phys_ps` group has a surface iteration subcycle with `loop="2"`, which generates an +actual Fortran `do` loop in the cap body: +```fortran +! Start of next subcycle +cdata%loop_max = 2 +do cdata%loop_cnt = 1, cdata%loop_max + ! ... sfc_diff, sfc_nst, noahmpdrv, sfc_land, sfc_cice, sfc_sice ... +end do +``` + +--- + +### 11.2 Cap hierarchy and scale + +The three-level hierarchy is preserved from prebuild: + +``` +ccpp_static_api.F90 (627 lines) ← suite+group name dispatch + ↓ +ccpp_fv3_gfs_v17_coupled_p8_cap.F90 (363 lines) ← calls all group caps in order + ↓ +ccpp_fv3_gfs_v17_coupled_p8_time_vary_cap.F90 (1404 lines) +ccpp_fv3_gfs_v17_coupled_p8_radiation_cap.F90 (967 lines) +ccpp_fv3_gfs_v17_coupled_p8_phys_ps_cap.F90 (4226 lines) ← 200 optional ptr arrays +ccpp_fv3_gfs_v17_coupled_p8_phys_ts_cap.F90 (1953 lines) +ccpp_fv3_gfs_v17_coupled_p8_stochastics_cap.F90 (443 lines) +``` + +The ugwpv1 variant generates another 10,220 lines of largely redundant code (identical +caps with one suite-name prefix change and minor scheme-list differences). Total for both +suites: 18,333 lines of generated Fortran. + +This redundancy is a key motivation for the redesign: suite variants that share groups +should not regenerate identical cap code. The redesign should support group-level cap +sharing across suite variants. + +--- + +### 11.3 Host model DDT structure + +All host data lives in `CCPP_data.F90` as module-level `save, target` variables: + +```fortran +type(GFS_control_type) :: GFS_control ! config/control +type(GFS_statein_type) :: GFS_statein ! atmospheric state in +type(GFS_stateout_type) :: GFS_stateout ! atmospheric state out +type(GFS_grid_type) :: GFS_grid ! grid geometry +type(GFS_tbd_type) :: GFS_tbd ! temporal interp data +type(GFS_cldprop_type) :: GFS_cldprop ! cloud properties +type(GFS_sfcprop_type) :: GFS_sfcprop ! surface properties +type(GFS_radtend_type) :: GFS_radtend ! radiation tendencies +type(GFS_coupling_type) :: GFS_coupling ! coupling fields +type(GFS_diag_type) :: GFS_intdiag ! diagnostics +type(GFS_interstitial_type), allocatable (:) :: GFS_interstitial ! scratch, per thread +``` + +Plus three `ccpp_t` instances for different levels of parallelism (see §11.5). + +This is structurally similar to the SCM's `physics` DDT hierarchy, but with one key +difference: all DDTs are at the same flat level rather than nested (no `physics%Statein`, +only `GFS_statein`). Each DDT maps to a distinct functional role. + +The `GFS_typedefs.F90` file (not auto-generated) defines all DDT types along with ~30 +physical constants (`con_pi`, `con_g`, `con_rd`, etc.) that also appear in the metadata. + +--- + +### 11.4 DDT arguments in the cap chain + +The static API imports all DDTs and physical constants from `CCPP_data` and `GFS_typedefs` +via `use` statements, then passes them as named arguments to group cap functions. This is +the full DDT-argument pattern that prebuild implements: + +```fortran +! In ccpp_static_api.F90: +use ccpp_data, only: gfs_control, gfs_statein, gfs_sfcprop, ... +use gfs_typedefs, only: con_pi, con_g, con_rd, ... + +ierr = fv3_gfs_v17_coupled_p8_phys_ps_run_cap( & + one=one, gfs_control=gfs_control, cdata=cdata, & + gfs_statein=gfs_statein, gfs_sfcprop=gfs_sfcprop, & + con_g=con_g, con_pi=con_pi, ... & + gfs_interstitial=gfs_interstitial) +``` + +The group cap receives these as typed `intent(*), target` dummy arguments and uses them +directly to construct call-site subsections. This means **the group cap is fully portable +— it does not use any host module directly**, only what it receives as arguments. + +The `target` attribute is required because the cap creates pointer sections of these DDTs +(array subsections via pointer assignment) when handling optional variables. + +--- + +### 11.5 The dual cdata architecture + +UFS uses two distinct sets of `ccpp_t` handles with different scopes: + +**Domain-level (`cdata_domain`)**: Used for non-run phases (init, finalize, time_vary +timestep_init/finalize). Called once per step, no blocking: +```fortran +cdata_domain%blk_no = 1; cdata_domain%chunk_no = 1 +cdata_domain%thrd_no = 1; cdata_domain%thrd_cnt = 1 +``` + +**Block/thread-level (`cdata_block(nb, nt)`)**: Used for run phase (radiation, phys_ps, +phys_ts, stochastics). Allocated as a 2-D array `(1:nblks, 1:nthrdsX)` where `nthrdsX` +accounts for non-uniform last-block sizing: +```fortran +cdata_block(nb,nt)%blk_no = nb +cdata_block(nb,nt)%chunk_no = nb ! block number = chunk number +cdata_block(nb,nt)%thrd_no = nt +cdata_block(nb,nt)%thrd_cnt = nthrdsX +``` + +The redesign must support this dual cdata usage: a single `cdata` handle for domain-level +phases and a 2-D array of handles for blocked run phases. + +--- + +### 11.6 OpenMP threading model + +Non-run phases allow internal threading in physics schemes: +```fortran +GFS_control%nthreads = nthrds ! all N threads available to physics +call ccpp_physics_timestep_init(cdata_domain, ...) +``` + +Run phase uses all threads for blocking, so physics must not spawn additional threads: +```fortran +GFS_control%nthreads = 1 ! no internal threading allowed +!$OMP parallel num_threads(nthrds) ... +!$OMP do schedule(dynamic,1) +do nb = 1, nblks + call GFS_Interstitial(nt)%create(ixs=chunk_begin(nb), ixe=chunk_end(nb), model=GFS_control) + call ccpp_physics_run(cdata_block(nb,nt), group_name="phys_ps", ...) + call GFS_Interstitial(nt)%destroy(GFS_control) +end do +!$OMP end do +!$OMP end parallel +``` + +The `nt = omp_get_thread_num()+1` pattern (1-based thread index) is used throughout. +Each thread owns one `GFS_Interstitial(nt)` and one `cdata_block(nb,nt)` per block +iteration. The dynamic schedule means different threads process different blocks at +different times, which is why the interstitial must be created/destroyed per-iteration +rather than pre-allocated per-thread. + +--- + +### 11.7 Horizontal dimension: the chunk_begin/chunk_end pattern + +For non-run phases, the full horizontal dimension is used at every call site: +```fortran +tgrs(one:gfs_control%ncols, one:gfs_control%levs) +``` + +For run phases, the chunk range is looked up from the control DDT using the block number: +```fortran +tgrs(gfs_control%chunk_begin(cdata%chunk_no) : gfs_control%chunk_end(cdata%chunk_no), & + one:gfs_control%levs) +``` + +The chunk size (horizontal extent `im`) is retrieved as: +```fortran +im = gfs_control%blksz(cdata%blk_no) +``` + +`blksz(nb)` handles **non-uniform block sizes**: the last block may be smaller than the +others if the domain size is not divisible by the number of blocks. The `chunk_begin`/ +`chunk_end` arrays (indexed by chunk number = block number) give the global offset range. + +This is a cleaner pattern than SCM's `chunk_begin`/`chunk_end` as explicit dummy +arguments, because UFS looks them up from the already-passed `gfs_control` DDT. + +**Critical implication for the redesign**: The subsetting pattern `(chunk_begin:chunk_end)` +appears at every single array call site in the run phase — literally hundreds of times in +the phys_ps cap alone. This boilerplate is generated by prebuild from the metadata. In +the redesign, this subsetting must remain at the call site (not higher up) to allow each +thread to process its own chunk independently. + +--- + +### 11.8 The GFS_interstitial — pointer-based scratch DDT + +`GFS_interstitial_type` (defined in `CCPP_typedefs.F90`) is a DDT where **every field is +a pointer**, initialized to null: +```fortran +type GFS_interstitial_type + real(kind_phys), pointer :: adjsfculw_land(:) => null() + real(kind_phys), pointer :: del(:,:) => null() + ! ... ~200+ pointer fields +end type +``` + +This is dramatically different from the SCM's interstitial (which is a regular allocatable +DDT allocated once per thread at startup). The UFS interstitial is: +1. **Created** (`GFS_Interstitial(nt)%create(ixs, ixe, model)`) before each block — this + allocates all required fields to the chunk size `ixe-ixs+1` +2. **Reset** (`GFS_Interstitial(nt)%reset(model)`) to zero before radiation and phys_ps +3. **Destroyed** (`GFS_Interstitial(nt)%destroy(model)`) after each block — deallocates + +This design exists because different blocks (especially the last block) can have different +sizes. Pre-allocating to the maximum size wastes memory at scale; per-block allocation +ensures exact sizing. The pointer-based design also allows the `create()` method to +selectively allocate only the fields needed for the current physics configuration. + +In the caps, the interstitial is accessed as: +```fortran +gfs_interstitial(cdata%thrd_no)%del(chunk_begin:chunk_end, one:levs) +``` + +The interstitial array is 1-D (indexed by thread, not by `(instance, thread)` as in SCM). +This works because UFS has only one model instance at runtime — no ensemble-in-memory. + +--- + +### 11.9 Optional variables — the pointer array pattern at scale + +The phys_ps run cap has **200 optional pointer arrays** in its local variable section. +Each looks like: +```fortran +type :: real_kind_phys_rank1_ptr_arr_type + real(kind_phys), dimension(:), pointer :: p => null() +end type real_kind_phys_rank1_ptr_arr_type +type(real_kind_phys_rank1_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array +``` + +Usage pattern (consistent with SCM but with threading dimension): +```fortran +if (gfs_control%lndp_type /= 0) then + sfc_wts_1_ptr_array(cdata%thrd_no)%p => & + gfs_coupling%sfc_wts(chunk_begin:chunk_end, one:gfs_control%n_var_lndp) +end if +! ... scheme call ... +if (gfs_control%lndp_type /= 0) then + nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) +end if +``` + +The array is dimensioned by `cdata%thrd_cnt` (total thread count) and indexed by +`cdata%thrd_no` (current thread number). This handles the threaded run phase where +multiple threads are simultaneously executing the same run cap function with different +chunk ranges. Each thread independently associates and nullifies its own pointer slot. + +200 optional variables in `phys_ps` alone. This is the regime for which the SCM had ~550 +total optional vars — confirming that operational 3-D GFS physics is heavily optional-var +driven. The design is sound but generates enormous boilerplate. + +A key observation: the type definition for each pointer wrapper (`integer_..._ptr_arr_type`, +`real_kind_phys_rank1_ptr_arr_type`, etc.) is **re-declared inside every single function +that needs it**. This results in duplicate type definitions across all group caps. The +redesign should define these wrapper types once in a shared module. + +--- + +### 11.10 Physical constants as metadata variables + +The UFS static API has an extensive USE list of physical constants from `gfs_typedefs`: +``` +con_pi, con_g, con_t0c, con_hfus, con_solr_2008, con_solr_2002, con_c, con_plnk, +con_boltz, con_rd, ltp, con_zero, con_rerth, con_p0, con_rv, con_cp, con_rgas, +con_amd, con_amw, con_avgd, con_hvap, con_eps, con_omega, con_fvirt, con_ttp, +con_thgni, con_epsm1, con_rog, con_rocp, con_tice, con_sbc, con_jcal, con_rhw0, +rlapse, rhowater, karman, con_1ovg, con_cliq, con_cvap, rainmin, con_epsm1 (30+ total) +``` + +These travel through the full chain: static API USE → suite cap argument → group cap +argument → scheme call argument. Each constant is declared as a separate scalar dummy +argument (`real(kind_phys), intent(in), target :: con_pi`) in every group cap that needs +it. + +This is correct but verbose. The redesign should consider whether constants should be +gathered into a dedicated DDT (e.g., `gfs_constants_type`) so the cap chain carries one +argument instead of 30. This would also eliminate the need to explicitly enumerate which +constants each group needs — they could all come along in the constants DDT. + +--- + +### 11.11 The `one` lower-bound anchor + +The integer constant `one = 1` (from `ccpp_types`) is passed as an explicit argument +throughout the UFS cap chain for the same reason as in SCM: it anchors lower array bounds +without triggering association-status issues: +```fortran +type(gfs_interstitial_type), intent(inout), target :: gfs_interstitial(one:) +tgrs(one:gfs_control%ncols, one:gfs_control%levs) +``` + +This pattern is ubiquitous and is a known prebuild idiom. + +--- + +### 11.12 No framework-owned persistent variables + +Unlike CAM-SIMA (which allocates scheme-persistent variables in the suite cap), the UFS +has no framework-owned persistent state in any cap. All persistent state lives in the host +DDTs (`GFS_tbd`, `GFS_sfcprop`, etc.). The interstitial DDT (`GFS_interstitial`) is purely +transient — created and destroyed each block. + +This is consistent with UFS's prebuild-based architecture. Whether framework-owned +persistent variables would be beneficial for UFS is an open question for the redesign. + +--- + +### 11.13 Build system and driver + +Prebuild is invoked from CMake (not programmatically) and generates: +- Group cap files (one per group × suites) +- Suite cap files (one per suite) +- `ccpp_static_api.F90` +- `CCPP_CAPS.cmake`, `CCPP_SCHEMES.cmake`, `CCPP_TYPEDEFS.cmake` — consumed by CMake to + enumerate files to compile + +The host driver (`CCPP_driver.F90`) is **hand-written**, not auto-generated. It owns the +OpenMP loop, the cdata allocation/setup, the interstitial create/destroy, and the +diagnostic bucket zeroing. This is a significant difference from CAM-SIMA where the +equivalent driver code is partially generated. In the redesign, this host driver code +should remain hand-written — it encodes model-specific threading and blocking decisions +that cannot be derived from metadata alone. + +--- + +### 11.14 Observations relevant to the redesign + +1. **The DDT-argument cap chain is fully validated at UFS scale.** Passing 10+ DDTs plus + 30+ scalar constants as named arguments through three cap levels works correctly in + production. The redesign must replicate this exactly. + +2. **The chunk_begin/chunk_end subsetting at call sites is non-negotiable.** Hundreds of + array sections per group cap. The generator must produce this from the metadata + `horizontal_dimension` standard name and the `active` flag for optional variables. + This is prebuild's core value at 3-D scale. + + *Design direction*: Rather than carrying `chunk_no` in cdata and having the cap look + up `gfs_control%chunk_begin(chunk_no)`, the redesign should pass + `horizontal_loop_begin` and `horizontal_loop_end` as explicit arguments directly to + `ccpp_physics_run()` (and analogous calls). This decouples the cap from knowing about + the host's internal chunk-lookup arrays. The host driver sets these for each block + iteration and passes them in; the cap uses them directly. + +3. **The domain-vs-block execution contexts must be supported, but the cdata object is + not necessarily the right mechanism.** The key information is: instance number, thread + number, horizontal_loop_begin, horizontal_loop_end, error flag/message. If all of + these are explicit named arguments to `ccpp_physics_*`, the cdata object becomes + redundant scaffolding. This is an open design question to be discussed separately, but + the UFS analysis shows that cdata carries exactly these values — the object is a + transport container, not a framework abstraction. + +4. **The `blksz` non-uniform block size is a first-class concern.** The generator must + produce `im = gfs_control%blksz(cdata%blk_no)` (or an equivalent `horizontal_loop_extent` + computed from the explicit begin/end) for the horizontal extent argument in run phases. + +5. **GFS_interstitial as a pointer-DDT is the correct design for 3-D models.** Creating + and destroying per block avoids memory waste from over-allocation to the maximum chunk + size. The pointer-based field design enables selective allocation. The redesign should + document this pattern and support it. (Whether the generator should emit the + `type(X_interstitial_type)` DDT definition itself or only the caps is TBD.) + +6. **200 optional pointer arrays in one group cap is manageable but the wrapper type + proliferation is not.** The 4 wrapper types (`integer_r1_ptr_arr_type`, + `real_r1_ptr_arr_type`, `real_r2_ptr_arr_type`, `character_len3_r1_ptr_arr_type`) + should be defined once in a shared module (e.g., `ccpp_types.F90`) and reused across + all caps, eliminating thousands of duplicate lines. + +7. **Physical constants as metadata variables must be gathered into a constants DDT.** + The redesign will collect all physics constants into a single `constants_type` DDT + (or equivalent), reducing 30+ individual scalar arguments in the cap chain to one + argument. This requires a metadata declaration mechanism for compound read-only + objects (i.e., constants do not need intent tracking the way state variables do). + +8. **No framework-owned persistent variables in UFS** confirms that this feature is + optional and model-specific. The redesign needs to support it (for CAM-SIMA-like + models) but should not force it on models that do not need it. + +9. **The host driver is correctly hand-written.** The OpenMP blocking, interstitial + lifecycle, diagnostic bucket management — these are model-specific decisions that + belong in the host driver, not in generated code. The redesign should not try to + generate the driver. + +10. **Suite variant cap redundancy is not a concern.** For research/development, multiple + suites are active simultaneously and generated code size doesn't matter. For + production, only one suite is compiled and used at a time. The redesign need not + prioritize eliminating redundant group cap code across suite variants. + +--- + +## 12. Real-world example: Navy NEPTUNE (prebuild, restricted) + +The NEPTUNE source code cannot be shared. The following is based on architectural +description provided by the lead developer. + +NEPTUNE uses `ccpp-prebuild` with the same GFS physics as UFS and nearly identical suites. +Its unique distinguishing feature is **multiple coexisting CCPP physics instances** — it +is the only model among the four examples that exercises this capability at runtime. + +--- + +### 12.1 Multiple instances — the N-dimensioned DDT array mechanism + +In NEPTUNE, the host model allocates N copies of all GFS DDTs as 1-D arrays indexed by +instance number: + +```fortran +type(GFS_sfcprop_type), allocatable :: gfs_sfcprop(1:N) +type(GFS_statein_type), allocatable :: gfs_statein(1:N) +type(GFS_stateout_type), allocatable :: gfs_stateout(1:N) +! ... all GFS DDTs dimensioned 1:N +type(GFS_control_type), allocatable :: gfs_control(1:N) +``` + +The static API imports these module-level arrays via `use` statements (same as UFS). +The instance selection happens at the call site inside the group cap, using +`cdata%ccpp_instance` as the array index: + +```fortran +call foo_run( & + tair = gfs_statein(cdata%ccpp_instance)%tair( & + gfs_control(cdata%ccpp_instance)%chunk_begin(cdata%chunk_no) : & + gfs_control(cdata%ccpp_instance)%chunk_end(cdata%chunk_no), & + 1:nvertical), & + ...) +``` + +Three things are happening simultaneously at each call-site array section: +1. **Instance selection**: `gfs_statein(cdata%ccpp_instance)` picks the correct DDT from + the N-element array +2. **Chunk subsetting**: `chunk_begin(chunk_no):chunk_end(chunk_no)` applies the run-phase + horizontal slice +3. **Vertical bound**: explicit `1:nvertical` + +This is the same pattern as UFS except the DDTs are 1-D arrays rather than scalars. +The generator must produce this instance-indexed subsetting when the host declares its +DDTs as arrays. + +--- + +### 12.2 What NEPTUNE tells us about `cdata%ccpp_instance` + +The `initialized(200)` array in every group cap (confirmed in both SCM and UFS caps) now +has its full motivation: it handles up to 200 simultaneous instances without requiring +per-instance cap code. The `cdata%ccpp_instance` value (1-based) is the runtime selector +into both the host DDT arrays and the `initialized` guard array. + +NEPTUNE is the reason `200` is not `1`. In single-instance models (UFS, SCM, CAM-SIMA) +`cdata%ccpp_instance` is always 1 and the N-dimensioned DDT arrays have `N=1`. + +--- + +### 12.3 Observations relevant to the redesign + +1. **Multiple instances require only one change at the call site**: inserting the instance + index at the correct dimension position. Everything else (chunking, optional variables, + threading) composes with this unchanged. + +2. **The instance dimension can appear anywhere in any host variable — not just as an + index into an array of DDTs.** A flat array `flat_field(1:ninstance, 1:nhoriz, 1:nvert)` + is equally valid; its call site becomes: + ```fortran + flat_field(instance_number, horiz_begin:horiz_end, 1:nvert) + ``` + The generator handles this by classifying each dimension by its declared standard name. + `instance_dimension` is a registered standard name (like `horizontal_dimension` and + `vertical_dimension`) — the generator knows its semantics regardless of where it + appears in the dimension list or whether the variable is a DDT array element or a + plain array. See §13.4 for the full dimension classification model. + +3. **No new cap-level mechanism is needed for multi-instance.** The instance number + (from the control layer, see §13) is sufficient. The cap code shape is the same; + only the call-site indexing expression differs based on the declared dimension roles. + +--- + +## 13. Cross-cutting design decision: how host data enters the cap chain + +Across all four models, two mechanisms are used for getting host model data into the +generated caps: + +| Mechanism | Models using it | Description | +|-----------|----------------|-------------| +| **Module USE** | UFS, SCM, CAM-SIMA, NEPTUNE | Static API has `use ccpp_data, only: gfs_statein, ...`. Data module name is known at generation time. | +| **Command-line arguments** | capgen (optional) | Generator accepts host variable access paths as CLI flags; generated caps receive data as explicit dummy arguments. | + +### 13.1 The capgen dual-mechanism problem + +Capgen supports both mechanisms, and this is a direct source of its complexity. The +variable-matching logic, VarDictionary scope chains, and `CCPPDatabaseObj` all exist +partly to handle the routing of variables that may arrive via either path. Maintaining +two entry points to the data layer doubles the surface area that must be tested and +reasoned about. + +### 13.2 The proposed single-mechanism approach + +The redesign will use **module USE exclusively** for all host data. The reasoning: + +- All four production models already use module USE, including CAM-SIMA (the capgen + model), which does not use capgen's CLI-argument path in practice. +- Module names are stable, known at generation time, and make the generated code + self-documenting (`use ccpp_data, only: gfs_statein` is unambiguous). +- Eliminating the CLI-argument entry path eliminates an entire class of generator + complexity. + +### 13.3 Runtime control variables — the thin explicit layer + +While all *data* enters via module USE, a set of *control* variables must be passed at +runtime because they change from call to call. These are not physics data; they tell the +cap *how* to index into the data it already has access to: + +| Variable | Purpose | When it matters | +|----------|---------|----------------| +| `ccpp_instance` | Select the instance dimension in host variables | NEPTUNE (N>1); others use 1 | +| `ccpp_thread_no` | Index optional pointer arrays per thread | Run phase with OpenMP | +| `horizontal_loop_begin` | Start of horizontal chunk to process | Run phase | +| `horizontal_loop_end` | End of horizontal chunk to process | Run phase | +| `ccpp_nthreads` | Max threads available for internal physics use | Non-run phases (currently `gfs_control%nthreads`) | +| `errmsg` / `errflg` | Error reporting return path | All phases | + +These are exactly the values that `cdata` carries in the current implementation. +Whether they are packaged as a `ccpp_t` struct or passed as individual named arguments to +`ccpp_physics_*` is an open design question for implementation. Either way, the generator +only needs to know about these variables and their standard names — it does not need to +accept host data paths on the command line. + +### 13.4 The dimension classification model + +A host variable's metadata declares the **standard name of each of its dimensions** in +order. The generator classifies every dimension into one of three categories and +constructs the call-site expression accordingly. + +**Category 1 — Registered dimensions.** The generator knows the semantics of these +standard names and generates special call-site expressions for them: + +| Standard name | Call-site expression | Notes | +|--------------|---------------------|-------| +| `instance_dimension` | `instance_number` (scalar index) | Omitted if variable has no instance dimension | +| `horizontal_dimension` | `1:horizontal_dimension` (non-run) or `horiz_begin:horiz_end` (run) | Phase-dependent | +| `vertical_dimension` | `1:vertical_dimension` | Fixed range | + +`instance_dimension` has the same registered status as `horizontal_dimension` and +`vertical_dimension`. Single-instance models simply do not declare any variables with +an `instance_dimension`, and the generator omits that index entirely. + +**Category 2 — Arbitrary host-declared dimensions.** Any dimension whose standard name +is not in the registered set. These are declared in host metadata pointing to a Fortran +expression accessible via module USE — either a flat module variable or a DDT member +(e.g. `gfs_control%ntrac`, `gfs_control%kice`). The generator emits `1:expression` +at the call site, resolved at generation time from the metadata. Fixed-index extractions +(e.g. `gfs_statein%qgrs(..., gfs_control%ntqv)`) are a special case: the dimension +value is a scalar index rather than a range upper bound, and the metadata must declare +which case applies. + +**Category 3 — Optional selector.** Not a dimension per se, but a boolean `active` +condition declared in variable metadata. Generates a pointer-association guard around +the call site (the pattern described in §9 and §11). + +This three-category model works uniformly regardless of host layout: +- `gfs_statein(instance)%tair(horiz, vert)` — registered instance + registered horizontal + registered vertical +- `flat_field(instance, horiz, vert, ntrac)` — registered + registered + registered + arbitrary +- `flat_field(horiz, vert)` — no instance dimension, single-instance model + +No special-casing per host model is needed in the generator. + +### 13.5 `type = control` — metadata declaration for runtime control variables + +The registered dimensions (§13.4 Category 1) are *dimension names* that appear in a +variable's `dimensions = (...)` list. Their actual *runtime values* are supplied by a +separate set of variables declared with `type = control` in host metadata. + +| `type = control` standard name | Fills in registered dimension / purpose | +|-------------------------------|----------------------------------------| +| `ccpp_instance` | `instance_dimension` — scalar index selecting the active instance | +| `ccpp_thread_no` | Not a dimension; indexes optional pointer arrays per thread | +| `horizontal_loop_begin` | Lower bound of `horizontal_dimension` in run phase | +| `horizontal_loop_end` | Upper bound of `horizontal_dimension` in run phase | +| `ccpp_nthreads` | Not a dimension; max threads available for internal physics use | +| `errmsg` / `errflg` | Error reporting return path | + +Variables declared `type = control` are: +- **Passed explicitly as runtime arguments** to `ccpp_physics_*` by the host driver + (not accessed via module USE, because their values change per call) +- **Used by the generator** to construct call-site indexing expressions for registered + dimensions, and to generate the `ccpp_nthreads` assignment before non-run scheme calls +- **Available to physics schemes** by standard name like any other variable — if a scheme + declares a variable with a matching standard name (e.g. `ccpp_nthreads`, + `horizontal_loop_begin`), the framework passes it as a scheme argument in the normal way + +This is similar in concept to capgen's `type = host` annotation but with a narrower, +well-defined scope. The name `control` is intentional: these variables *control* how +the cap indexes into the data, not what the data is. + +The set of recognized standard names for `type = control` variables is fixed and small. +Declaring them explicitly in metadata — rather than having the generator recognize magic +names — keeps the mechanism open and self-documenting. + +### 13.6 Consequences for the generator + +1. The generator reads host metadata to learn: + - Module names for all host data variables (emitted as `use` statements in the static API) + - The dimension standard names of each variable (for call-site expression construction) + - Which variables are `type = control` (for the runtime argument layer) +2. At cap generation time, the static API's `use` statements are emitted from the module + names — no runtime flexibility, no CLI data routing. +3. Call-site subsetting for every variable is constructed purely from its declared + dimension standard names: registered dimensions use the Category 1 rules; arbitrary + dimensions are resolved to Fortran expressions via the host metadata. +4. The only runtime inputs to the cap are the `type = control` variables. Their values + are supplied by the host driver for each `ccpp_physics_*` call. diff --git a/doc/redesign_prompt - original 20260505T2044.md b/doc/redesign_prompt - original 20260505T2044.md new file mode 100644 index 00000000..8a3c029e --- /dev/null +++ b/doc/redesign_prompt - original 20260505T2044.md @@ -0,0 +1,814 @@ +# CCPP Framework Code Generator — Redesign Specification + +## Purpose + +This document is a complete implementation specification for a new CCPP Framework code +generator (`ccpp-capgen-ng`). It supersedes both `ccpp-prebuild` and `ccpp-capgen`. An +implementer should be able to build the new generator from scratch using this document +alone, supplemented by the real-world examples in `redesign_analysis.md`. + +--- + +## 1. Background and Motivation + +The CCPP Framework couples host NWP models (UFS Weather Model, NEPTUNE, CCPP-SCM, +CAM-SIMA) to physics parameterization schemes by auto-generating Fortran interface +("cap") code. Two generators exist today: + +- **`ccpp-prebuild`** — simple, procedural Python; fast; DDT arguments; used in + production by UFS, NEPTUNE, SCM. Does not support framework-owned variables. +- **`ccpp-capgen`** — complex OO Python; flat-field arguments; used in CAM-SIMA. The + deep class hierarchy (`VarDictionary`, `VarCompatObj`, `CCPPDatabaseObj`, etc.) makes + it unmaintainable. Three developers spent considerable time trying to add DDT argument + passing and could not succeed. + +The redesign starts fresh, drawing lessons from both. The guiding principle is: +**simplicity of prebuild, feature set of capgen**. + +The primary failures that triggered the redesign: +1. capgen passes flat fields to group caps — infeasible at UFS/NEPTUNE scale (1200+ + variables), breaks under compiler debug flags for optional variables. +2. capgen's scope-chain variable promotion is the source of most complexity. +3. Nobody on the team fully understands capgen. + +--- + +## 2. Toolchain Structure + +The redesign produces **two separate tools** that share the same metadata parsing library: + +### 2.1 Validator (`ccpp_validator.py`) + +Parses both Fortran source files and metadata files, compares them, and reports +discrepancies. Run by developers before invoking the generator — e.g., during scheme +development or in CI. Does **not** generate any Fortran output. + +### 2.2 Code Generator (`ccpp_capgen_ng.py`) + +Parses metadata only. Assumes metadata correctly describes the Fortran source — performs +no Fortran parsing. Generates all cap files and supporting modules. + +**Both tools import the same metadata parsing module.** No duplication of metadata +parsing logic between the two tools. + +--- + +## 3. Metadata Format + +### 3.1 File format + +The existing ini-file format is preserved unchanged. Every metadata file consists of +`[ccpp-table-properties]` header blocks followed by `[ccpp-arg-table]` variable listing +blocks, exactly as in the current framework. + +The `[ccpp-table-properties]` + `[ccpp-arg-table]` pair is redundant for non-scheme +tables (the distinction is vestigial for host/DDT/suite tables) but is preserved for +symmetry with scheme metadata tables. + +### 3.2 Table types (`type =` in `[ccpp-table-properties]`) + +Five table types are supported: + +| `type =` | Ownership | Import mechanism | +|---|---|---| +| `scheme` | Physics scheme | Intent args on scheme subroutines | +| `host` | Host model | Module USE (direct or via DDT member) | +| `control` | Framework runtime layer | Explicit args to `ccpp_physics_*` entry points | +| `suite` | Generated suite cap | Module USE of generated suite data module | +| `ddt` | Type definition | Structural — describes DDT fields, no instance info | + +Notes: +- `type = module` from capgen is renamed to `type = host`. Breaking change, intentional. +- `type = suite` tables are **written by the generator** (never hand-authored). They + appear on disk for inspection and debugging only. +- `type = ddt` describes the structure of a Fortran derived type. It contains no + instance information — only field definitions. + +### 3.3 Per-variable attributes + +All existing per-variable attributes are preserved: `standard_name`, `long_name`, +`units`, `dimensions`, `type`, `kind`, `intent`, `optional`, `active`, `protected`. + +`protected = True` means: any scheme that declares `intent` other than `in` for this +variable is a metadata error, caught at generation time. This is how constants are +handled — a constants DDT is declared `type = host` with all fields `protected = True`. +No separate `type = constants` is needed. + +### 3.4 DDT type definitions + +A DDT type definition uses `type = ddt`: + +```ini +[ccpp-table-properties] + name = gfs_statein_type + type = ddt + +[ccpp-arg-table] + name = gfs_statein_type + type = ddt + +[phii] + standard_name = geopotential_at_interface + long_name = geopotential at model layer interfaces + units = m2 s-2 + dimensions = (horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys +``` + +### 3.5 DDT instances + +A DDT instance is declared as a regular variable entry inside a `type = host` table. +The enclosing `[ccpp-table-properties]` block's `name` attribute identifies the Fortran +module from which the instance is imported via `use`. No separate `module` attribute is +needed on the variable entry. + +```ini +[ccpp-table-properties] + name = CCPP_data + type = host + dependencies = CCPP_typedefs.F90,GFS_typedefs.F90 + +[ccpp-arg-table] + name = CCPP_data + type = host + +[gfs_statein] + standard_name = gfs_statein + long_name = GFS state input for all instances + units = mixed + dimensions = (number_of_instances) + type = gfs_statein_type +``` + +The generator looks up the `gfs_statein_type` DDT table, traverses its fields, and +constructs access paths of the form +`gfs_statein(instance_number)%fieldname(loop_begin:loop_end, 1:nlevs)`. + +For scalar DDT instances (no instance dimension), dimensions is `()`. +For nested DDTs, the same mechanism applies recursively. + +### 3.6 Control variable declarations + +Control variables are declared in a `type = control` table. The generator resolves +each control variable by its standard name and uses whatever local Fortran name the +host declared: + +```ini +[ccpp-table-properties] + name = my_host_control_module + type = control + +[ccpp-arg-table] + name = my_host_control_module + type = control + +[loop_begin] + standard_name = horizontal_loop_begin + long_name = start of horizontal loop + units = index + dimensions = () + type = integer +``` + +--- + +## 4. Control Variables + +The generator recognizes the following standard names for control variables. Local +Fortran names are host-defined (resolved from the `type = control` metadata table). + +### 4.1 Entry point arguments (non-register phases) + +| Standard name | Role | Conditional? | +|---|---|---| +| `suite_name` | Suite name for runtime dispatch | No | +| `group_name` | Group name for runtime dispatch | No | +| `horizontal_loop_begin` | Start of horizontal chunk/domain | No | +| `horizontal_loop_end` | End of horizontal chunk/domain | No | +| `thread_number` | Current thread index (1..number_of_threads) | No | +| `number_of_threads` | Host blocking loop thread count; allocation bound for thread-dimensioned suite data | No | +| `number_of_physics_threads` | Thread budget for physics-internal OpenMP use | No | +| `ccpp_error_message` | Error message string | No | +| `ccpp_error_code` | Integer error return code | No | +| `instance_number` | Current model instance index | **Conditional** | + +`instance_number` is included only if `instance_dimension` appears anywhere in the +parsed host metadata. Single-instance models omit it from all entry point signatures. +The generator detects this automatically at parse time. + +### 4.2 Loop-generated control variables (subcycles only) + +| Standard name | Role | +|---|---| +| `ccpp_loop_counter` | Current subcycle iteration (1..ccpp_loop_extent) | +| `ccpp_loop_extent` | Total subcycle iterations; value comes from the `loop=N` attribute on the `` element in the suite XML definition file | + +These are **not** passed as `ccpp_physics_*` arguments from the host. They are set by +the generated `do` loop inside the group cap and are available to any scheme called +within that loop. Outside a subcycle loop, these variables are not in scope. + +### 4.3 Registered dimension standard names + +The generator has built-in semantic knowledge of these dimension standard names: + +| Standard name | Indexing semantic | +|---|---| +| `instance_dimension` | Scalar extraction: `var(instance_number)` — `instance_number` is the control variable | +| `horizontal_dimension` | Slice: `horizontal_loop_begin:horizontal_loop_end` (run phase) or `1:` (non-run) | +| `vertical_*` | Slice: `1:` | + +`horizontal_dimension` and `vertical_*` are "registered" because the generator knows +their slicing semantics, but they are resolved to local names the same way as arbitrary +dimensions — by looking up the variable with that standard name in the host metadata. +There is no special-casing in the resolution mechanism, only in the indexing expression +emitted. + +All other dimension standard names are resolved identically: look up the variable with +that standard name, get its local Fortran name, emit `1:local_name`. + +The timing of `instance_dimension` substitution — whether at parse time (when building +the flat dict access path) or at call-string generation time (like other registered +dimensions) — is an implementation decision left to the developer. Either is correct; +choose whichever is easier to implement, understand, and maintain. + +--- + +## 5. Entry Points + +Eight entry points are generated in the static API. Two tiers: + +### 5.1 Framework lifecycle (no group_name dispatch) + +These operate on the entire suite at once. They take `suite_name` plus +`ccpp_error_message` and `ccpp_error_code`. No scheme `_run/_init/_final` calls. + +| Entry point | Purpose | +|---|---| +| `ccpp_register(suite_name, constituents, errmsg, errcode)` | Calls each scheme's `_register` entrypoint; allocates and populates the `ccpp_model_constituents_t` object passed by the host | +| `ccpp_init(suite_name, errmsg, errcode)` | Allocates integer state arrays in all group caps; allocates suite-owned interstitial data; no scheme calls | +| `ccpp_final(suite_name, errmsg, errcode)` | Deallocates integer state arrays and suite-owned data; no scheme calls | + +`constituents` in `ccpp_register` is `intent(inout)`: unallocated on entry, allocated +and populated on exit. The host declares and owns this object (imports +`ccpp_model_constituents_t` from the framework library). The argument is mandatory. + +### 5.2 Physics group invocation (dispatched by suite_name + group_name) + +| Entry point | Calls scheme phase | +|---|---| +| `ccpp_physics_init(...)` | `_init` | +| `ccpp_physics_timestep_init(...)` | `_timestep_init` | +| `ccpp_physics_run(...)` | `_run` | +| `ccpp_physics_timestep_final(...)` | `_timestep_final` | +| `ccpp_physics_final(...)` | `_final` | + +All five take the full control variable argument list (Section 4.1). `group_name` is +optional — if omitted, the suite cap calls all groups in declared order. If specified, +only that group is invoked. + +Non-run phases pass `horizontal_loop_begin=1` and +`horizontal_loop_end=` — the cap code is uniform across +phases, with no special-casing for run vs. non-run. + +### 5.3 Naming note + +`finalize` is renamed to `final` throughout (e.g., `ccpp_physics_final`, not +`ccpp_physics_finalize`). Breaking change, intentional for symmetry: +`ccpp_init`/`ccpp_final`, `ccpp_physics_init`/`ccpp_physics_final`, +`ccpp_physics_timestep_init`/`ccpp_physics_timestep_final`. + +--- + +## 6. Cap Hierarchy + +All three levels are fully auto-generated. No hand-written components in the cap layer. + +### 6.1 Static API (`ccpp_static_api.F90`) + +- Imports all host DDTs and flat fields via `module use` (resolved from host metadata) +- Imports `ccpp_kinds` from `ccpp_kinds.F90` +- Dispatches all eight entry points by `suite_name` to the appropriate suite cap +- Passes `ccpp_model_constituents_t` through as an explicit argument (does not own it) +- Holds no physics state + +### 6.2 Suite cap (`ccpp__cap.F90`) + +- Imports the generated suite data module (`ccpp__data.F90`) +- Contains the suite-level integer state array: `integer, allocatable :: ccpp_suite_state(:)` indexed by instance +- Implements the suite-level state machine (see Section 7) +- On `ccpp_register`: calls all scheme `_register` entrypoints across the suite +- On `ccpp_init`: allocates suite-level state array; allocates `ccpp_group_state(:)` in + all group caps for this suite; allocates suite-owned interstitial data +- On `ccpp_final`: deallocates all of the above +- Routes `ccpp_physics_*` calls by `group_name` to the appropriate group cap function + +### 6.3 Group cap (`ccpp___cap.F90`) + +- Imports `ccpp__data` (suite-owned interstitial data) +- Imports `ccpp__types` (shared wrapper types) +- Contains the group-level integer state array: `integer, allocatable :: ccpp_group_state(:)` indexed by instance +- Implements the group-level state machine (see Section 7) +- Contains the actual scheme call sites for each phase: + - Loop bound locals + - Optional variable pointer arrays (thread-dimensioned) + - Fixed-index extraction locals + - Unit/kind conversion locals + - Subcycle `do` loops + - Scheme calls with full argument lists + +--- + +## 7. State Machine + +Integer state parameters are defined in a shared framework library module (not +generated). Two levels, both indexed by `instance_number`. + +### 7.1 Suite-level state (in suite cap) + +```fortran +integer, parameter :: CCPP_SUITE_UNREGISTERED = 0 +integer, parameter :: CCPP_SUITE_REGISTERED = 1 +integer, parameter :: CCPP_SUITE_FRAMEWORK_INITIALIZED = 2 +``` + +| Entry point | Required state | State after | +|---|---|---| +| `ccpp_register` | `== UNREGISTERED` | `REGISTERED` | +| `ccpp_init` | `== REGISTERED` | `FRAMEWORK_INITIALIZED` | +| `ccpp_physics_*` | `== FRAMEWORK_INITIALIZED` | (unchanged) | +| `ccpp_final` | `== FRAMEWORK_INITIALIZED` | `REGISTERED` | + +### 7.2 Group-level state (in each group cap) + +```fortran +integer, parameter :: CCPP_GROUP_UNINITIALIZED = 0 +integer, parameter :: CCPP_GROUP_INITIALIZED = 1 +integer, parameter :: CCPP_GROUP_IN_TIMESTEP = 2 +``` + +| Entry point | Required state | State after | +|---|---|---| +| `ccpp_physics_init` | `< INITIALIZED` (idempotent if `== INITIALIZED`) | `INITIALIZED` | +| `ccpp_physics_timestep_init` | `== INITIALIZED` | `IN_TIMESTEP` | +| `ccpp_physics_run` | `== IN_TIMESTEP` | `IN_TIMESTEP` | +| `ccpp_physics_timestep_final` | `== IN_TIMESTEP` | `INITIALIZED` | +| `ccpp_physics_final` | `>= INITIALIZED` | `UNINITIALIZED` | + +The idempotency rule for `ccpp_physics_init`: if the group is already in state +`INITIALIZED`, return immediately without calling any scheme `_init` routines. This +allows the host to call `ccpp_physics_init` multiple times safely. Any further call +after the first must result in no change (idempotency is a scheme contract). + +These two integer arrays replace both the boolean `initialized(:)` array from prebuild +and the string-based `ccpp_suite_state` from CAM-SIMA. + +--- + +## 8. Scheme Metadata and Variable Matching + +### 8.1 Scheme metadata structure + +Each scheme source file has a companion `.meta` file with `type = scheme` tables — one +table per public phase subroutine (`scheme_name_init`, `scheme_name_run`, +`scheme_name_timestep_init`, etc.). The section header for each variable entry is the +**local variable name** as it appears in the scheme's Fortran subroutine argument list. + +The internal metadata store is keyed as: + +``` +metadata[scheme_name][phase][standard_name] → {local_name, units, kind, dimensions, + intent, optional, active, ...} +``` + +Keying by `scheme_name` then `phase` enables cross-phase queries ("does this scheme +have a register phase?", "what are all variables of scheme X?") and matches the +conceptual model of the suite XML. + +### 8.2 Reading order + +All metadata files (host + scheme + DDT) are read in one pass without resolving DDT +type references. After the full read, the generator builds the known DDT list, then +resolves all type references. This avoids ordering dependencies between metadata files. + +### 8.3 Known DDT list + +After reading all metadata, the generator assembles the set of known DDT types from +`type = ddt` tables. Two categories: + +**Framework-defined DDTs**: declared in `type = ddt` metadata tables. The generator +knows their fields, dimensions, and access paths. + +**External DDTs**: types from external libraries (MPI, ESMF, etc.) that the generator +cannot introspect. These are declared in variable entries using an extended `type` +syntax: + +```ini +[mycomm] + standard_name = mpi_communicator + type = external:mpi_f08:mpi_comm + ... +``` + +The format is `external::`. The generator emits +`use mpi_f08, only: mpi_comm` and treats the variable as an opaque type — no field +traversal, no dimension indexing beyond what the metadata declares. + +### 8.4 Variable matching: scheme vs. host + +For each argument in a scheme's phase function (looked up by standard name): + +1. **Found in host+control flat dict** → use the resolved access path. If `units` or + `kind` differ from what the scheme declares, generate a transformation (Section 9). +2. **Not found, first use is `intent(out)`** → suite-owned variable. Add to suite data, + generate declaration in `ccpp__data.F90`. +3. **Not found, first use is `intent(in)` or `intent(inout)`** → **error**: variable + used before it is provided by any scheme or host. +4. **Found in suite data (from a prior scheme)** → use the suite data access path. + Apply transformation if needed. + +### 8.5 Cap call argument construction + +The generator builds the argument list for each scheme call from the scheme's metadata +argument order. For each argument: + +- **Direct pass-through** (no transformation, not optional): inline host access + expression — no local variable declared +- **Transformation**: cap-local temporary named after the scheme's local variable name + (from scheme metadata section header); see Section 10.2 for naming rules +- **Optional**: cap-local pointer array named `_p`; see Section 10.3 +- **Optional + transformation**: combined in the `if (active) then` block + +The generator does not parse Fortran source. All local names, types, kinds, dimensions, +and intents come exclusively from metadata. + +--- + +## 9. Variable Resolution and Access Path Construction + +### 9.1 Flat storage model (host+control+suite) + +The generator flattens the DDT hierarchy at parse time. After parsing, all host, +control, and suite variables are stored in a flat dictionary keyed by standard name. +Each entry contains: +- The Fortran local name +- The fully-qualified access path (e.g., `gfs_statein(instance_number)%phii`) +- The module to USE +- Dimension information with registered/arbitrary classification + +The DDT hierarchy is discarded after the flat dict is built. No live DDT object tree +is maintained during code generation. + +### 9.2 Access path construction + +For each variable, the generator constructs the call-site expression by applying +dimension rules to each dimension in order: + +1. **`instance_dimension`** → substitute `instance_number` (scalar extraction) +2. **`horizontal_dimension`** → substitute `horizontal_loop_begin:horizontal_loop_end` + (run phase) or `1:local_horizontal_dimension` (non-run) +3. **`vertical_*`** → substitute `1:local_vertical_dimension` +4. **Arbitrary dimension** → resolve to local name via its own metadata entry, emit + `1:local_name` +5. **`active` condition** → generate optional pointer-association guard (see Section 10.3) + +### 9.3 Module USE + +For each variable used in a group cap, the generator emits a `use module, only: varname` +statement. The module name comes from the enclosing `[ccpp-table-properties]` block name. +For suite-owned variables, the module is the generated `ccpp__data`. + +### 9.4 Eliminating TYPEDEFS_NEW_METADATA + +The manually-maintained `TYPEDEFS_NEW_METADATA` Python dict from prebuild is eliminated. +All information previously in that dict is now in metadata: +- The DDT type structure → `type = ddt` table +- The module-level instance → variable entry in the `type = host` table, module + implied by enclosing table name + +--- + +## 10. Variable Transformations and Optional Variables + +Variable transformations (unit/kind conversions) and optional variable handling are +combined — both occur within the same `if (active) then` block when a variable is +optional. + +### 10.1 Supported transformations + +- **Unit conversions** (e.g., Pa → hPa): formula from built-in conversion table (shared + with validator), keyed on source/target unit pair from metadata `units` attribute +- **Kind conversions** (e.g., r8 → kind_phys): from `kind` metadata attribute comparison + +The transformation framework is generic and pluggable — additional transformation types +(e.g., vertical flipping) can be added without restructuring the generator. + +### 10.2 Local variable naming + +The local variable name for a transformation temporary or optional pointer is derived +from the **scheme's local variable name** as declared in the scheme's metadata section +header (e.g., `[phii]` → local name is `phii`). The generator has this name without +parsing any Fortran. + +- Transformation temporary: scheme's local name + `_l` (e.g., `phii_l`) +- Optional pointer: scheme's local name + `_p` (e.g., `phii_p`) +- Conflict resolution: if two schemes in the same group cap use the same local name for + different standard names, append a numeric suffix before the suffix (e.g., `phii_2_l`) + +The generator validates that all generated local variable names and generated subroutine +names stay within Fortran's 63-character identifier limit. Violations are code-generation +errors — the developer must use a shorter local name in their metadata. + +The `active` expression in metadata is a Fortran logical expression written using +**CCPP standard names** (not local names). The generator translates all standard names +in the expression to their local Fortran names before emitting. + +Transformations **always** use a local temporary variable. The host variable is never +modified in-place — required for bit-for-bit reproducibility and to leave host data +uncorrupted if an exception occurs. Every conversion line carries an inline Fortran +comment (e.g., `! unit conversion: Pa to hPa`). Transformation mismatches (unknown +unit pair, unknown kind pair) are code-generation errors — no stdout/stderr from +generated code. + +### 10.3 The four cases + +The generator handles exactly four combinations per variable: + +**Case 1: No pointer, no transformation** (not optional, no unit/kind mismatch) + +No local variable is declared. The host access expression is used inline at the call +site: +```fortran +call scheme_run(..., gfs_statein(instance_number)%phii(lb:ub,1:nlevs), ...) +``` + +**Case 2: Pointer only** (optional, no transformation) + +Pointer array declared at function top; conditional association in `if (active)` block: +```fortran +! declaration: +type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) + +! before call: +if () then + phii_p(thread_number)%ptr => gfs_statein(instance_number)%phii(lb:ub,1:nlevs) +else + nullify(phii_p(thread_number)%ptr) +end if + +call scheme_run(..., phii_p(thread_number)%ptr, ...) + +! after call: +nullify(phii_p(thread_number)%ptr) +``` + +**Case 3: Transformation only** (not optional, unit/kind mismatch) + +Local temporary `phii_l` (scheme's local name + `_l`); intent-driven emission: + +| `intent` | Pre-call | Post-call | +|---|---|---| +| `in` | `phii_l = host_phii(...) * factor ! unit conversion: X to Y` | nothing | +| `out` | nothing | `host_phii(...) = phii_l / factor ! unit conversion: Y to X` | +| `inout` | pre-call as above | post-call as above | + +```fortran +! declaration: +real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) ! or appropriate rank/kind + +! before call (intent in/inout): +phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa + +call scheme_run(..., phii_l, ...) + +! after call (intent inout/out): +gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa +``` + +**Case 4: Pointer and transformation** (optional + unit/kind mismatch) + +Two local variables: `phii_l` (transformation temporary) and `phii_p` (pointer array). +Sequence: (1) apply transformation to `phii_l` depending on intent, (2) assign pointer +to `phii_l`, (3) call scheme, (4) nullify pointer, (5) apply back-transformation from +`phii_l` to host depending on intent. All within the `if (active)` block: + +```fortran +! declarations: +real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) +type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) + +! before call: +if () then + ! step 1: apply forward transformation (intent in/inout) + phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa + ! step 2: assign pointer to transformed local + phii_p(thread_number)%ptr => phii_l +else + nullify(phii_p(thread_number)%ptr) +end if + +call scheme_run(..., phii_p(thread_number)%ptr, ...) + +! after call: +if () then + ! step 4: nullify pointer + nullify(phii_p(thread_number)%ptr) + ! step 5: apply back-transformation (intent inout/out) + gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa +end if +``` + +The wrapper types (`real_kind_phys_rank1_ptr_type` etc.) are defined once in the +generated shared types module (`ccpp__types.F90`), not re-declared inside every +group cap function. Passing an unassociated pointer is safe under all compiler modes. + +--- + +## 11. Subcycle Loops + +When a group in the suite XML contains ``, the generator emits a +Fortran `do` loop in the group cap: + +```fortran +do = 1, N ! subcycle: N iterations from suite XML + ! ... scheme calls ... +end do +``` + +`ccpp_loop_counter` and `ccpp_loop_extent` are **loop-context variables** — a special +class that does not fit any of the five table types: +- NOT `type = control`: not passed as `ccpp_physics_*` arguments from the host +- NOT `type = host`: not from host module USE +- NOT `type = suite`: not persistent allocated data + +The generator has built-in knowledge of these two standard names. `ccpp_loop_extent` +value comes from the `loop=N` attribute in the suite XML definition file. +`ccpp_loop_counter` is the do loop induction variable. Both exist only within the +generated loop scope — they are not in scope outside a subcycle block. + +Any scheme that requests `ccpp_loop_counter` or `ccpp_loop_extent` by standard name +receives the loop variables at the call site. Their local names in the cap are derived +from the scheme's metadata section headers as for any other variable (Section 10.2). + +--- + +## 12. Init/Finalize Deduplication + +If the same scheme appears more than once within a single group (e.g., via subcycles), +having its `_init` called multiple times is a **code generator bug** — the generator +**errors out** rather than silently deduplicating. The suite XML must not list the same +scheme multiple times in the same group for non-run phases. + +If the same scheme appears in multiple groups, its `_init` is called once per group. +This is acceptable — idempotency is a contract all scheme `_init` routines must satisfy. + +The same rule applies to `_timestep_init`, `_timestep_final`, and `_final`. + +--- + +## 13. Suite-Owned Data + +### 13.1 Discovery + +The generator identifies suite-owned variables during variable resolution: variables +requested by schemes that are not satisfied by host metadata (`type = host` or +`type = control`). + +**Error condition**: if a variable is determined to be suite-owned (not provided by the +host) and the first scheme that uses it does not have it as `intent(out)`, the generator +errors out. A suite-owned variable that is first read before it is written would be +used uninitialized. + +### 13.2 Generated files and allocation + +Suite-owned variables are declared in a generated suite data module +(`ccpp__data.F90`) as fields of a Fortran DDT. The suite cap and all group caps +`use` this module. + +Allocation happens in `ccpp_init` (suite cap). Deallocation happens in `ccpp_final`. +Suite-owned arrays are allocated for the **full `horizontal_dimension`** — threads +access their respective horizontal chunk (`horizontal_loop_begin:horizontal_loop_end`) +at scheme call sites. This avoids per-call allocation overhead. If this proves to +consume too much memory at scale, a future revision may move allocation to per-phase +with chunk-sized arrays; start with the full-dimension approach. + +Subsetting (applying horizontal loop bounds, instance index) happens at scheme call +sites in the group cap, not in the suite cap. + +### 13.3 Metadata + +The generator also writes a `type = suite` metadata table (`ccpp_.meta`) as a +byproduct. This file is for inspection and debugging — it is not consumed by the +generator on subsequent runs. + +--- + +## 14. Constituent API + +### 14.1 Type definition + +`ccpp_model_constituents_t` is unchanged from CAM-SIMA. The type definition lives in +the framework library (`ccpp_constituent_prop_mod`), not in generated code. + +### 14.2 Ownership and lifecycle + +The **host model** declares and owns the constituent object: + +```fortran +use ccpp_constituent_prop_mod, only: ccpp_model_constituents_t +type(ccpp_model_constituents_t) :: constituents ! unallocated initially +``` + +The host passes it to `ccpp_register`, which allocates and populates it. After +`ccpp_register` returns, the host holds a fully allocated object ready for `lock_table`, +`const_index`, `copy_in`, `copy_out`. + +The `constituents` argument to `ccpp_register` is mandatory. + +### 14.3 Register phase mechanics + +The suite cap's register routine: +1. Iterates over all constituent-providing schemes in the suite +2. Calls each scheme's `_register` entrypoint, which returns a + `ccpp_constituent_properties_t` array +3. Collects these arrays and populates the constituent object + +No `group_name` dispatch is needed for register — it operates on the whole suite. + +--- + +## 15. Generated Output Files + +All files are written to `--output-root`. + +| File | Contents | +|---|---| +| `ccpp_kinds.F90` | Kind parameter definitions from `--kind-type` CLI args | +| `ccpp_static_api.F90` | Static API — host imports, suite_name dispatch | +| `ccpp__cap.F90` | Suite cap — suite data import, state machine, group dispatch | +| `ccpp___cap.F90` | Group cap — scheme call sites, state array, optionals, transformations | +| `ccpp__data.F90` | Suite data module — framework-owned interstitial DDT | +| `ccpp__types.F90` | Shared cap types — optional pointer wrapper types, transformation locals | +| `ccpp_.meta` | Generated `type = suite` metadata table (output-only, for inspection) | +| `datatable.xml` | Generator database for `ccpp_datafile.py` queries | + +`ccpp_kinds.F90` is a dependency of all other generated Fortran files. + +--- + +## 16. CLI Invocation + +``` +ccpp_capgen_ng.py + --host-name + --host-files + --scheme-files + --suites + --output-root + --kind-type KIND=PRECISION # repeatable, e.g. --kind-type kind_phys=REAL64 + --verbose # once = info; twice = debug +``` + +The generator also supports programmatic Python invocation (import and call directly), +using the same internal code paths as the CLI. + +### 16.1 Kind specifications + +Kind mappings are passed at the CLI level, not in metadata. The generator substitutes +kind names in all generated Fortran declarations. Example: +`--kind-type kind_phys=REAL64 --kind-type kind_dyn=REAL32`. + +### 16.2 datatable.xml and ccpp_datafile.py + +The generator emits `datatable.xml` encoding the full relationships between suites, +groups, schemes, and variables. A separate query utility (`ccpp_datafile.py`) provides +a rich query interface used by CMake and other build systems. The full query surface of +the existing `ccpp_datafile.py` is preserved — the simplified `--ccpp-files` query is +one of many. + +### 16.3 CMake integration pattern + +The generator runs at CMake configure time via `execute_process`. Generated sources are +discovered by querying `ccpp_datafile.py --ccpp-files`. Host and scheme Fortran sources +are found by replacing `.meta` with `.F90` (same base name convention). + +--- + +## 17. Design Decisions Not Carried Forward + +The following patterns from prebuild or capgen are explicitly **not** carried forward: + +| Pattern | Reason | +|---|---| +| `ccpp_t` / `cdata` struct | Replaced by explicit named control variable arguments | +| `TYPEDEFS_NEW_METADATA` Python dict | Replaced by DDT instance declarations in metadata | +| String-based `ccpp_suite_state` | Replaced by integer state arrays with named parameters | +| Boolean `initialized(:)` array | Replaced by integer state arrays | +| Flat-field arguments to group caps | DDT arguments are used instead (as in prebuild) | +| Scope-chain variable promotion | Suite-owned variables explicitly discovered and declared | +| Fortran-vs-metadata validation in generator | Moved to standalone validator tool | +| Re-declaration of pointer wrapper types per function | Declared once in shared types module | +| `ccpp_physics_suite_init/finalize` | Replaced by `ccpp_init`/`ccpp_final` | +| `type = module` (capgen) | Renamed to `type = host` | +| `finalize` phase name | Renamed to `final` | +| Array size checks in caps | Not generated by default; rely on compiler bounds checking | diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md new file mode 100644 index 00000000..c8e1ab3b --- /dev/null +++ b/doc/redesign_prompt.md @@ -0,0 +1,1172 @@ +# CCPP Framework Code Generator — Redesign Specification + +## Purpose + +This document is a complete implementation specification for a new CCPP Framework code +generator (`ccpp-capgen-ng`). It supersedes both `ccpp-prebuild` and `ccpp-capgen`. An +implementer should be able to build the new generator from scratch using this document +alone, supplemented by the real-world examples in `redesign_analysis.md`. + +--- + +## 1. Background and Motivation + +The CCPP Framework couples host NWP models (UFS Weather Model, NEPTUNE, CCPP-SCM, +CAM-SIMA) to physics parameterization schemes by auto-generating Fortran interface +("cap") code. Two generators exist today: + +- **`ccpp-prebuild`** — simple, procedural Python; fast; DDT arguments; used in + production by UFS, NEPTUNE, SCM. Does not support framework-owned variables. +- **`ccpp-capgen`** — complex OO Python; flat-field arguments; used in CAM-SIMA. The + deep class hierarchy (`VarDictionary`, `VarCompatObj`, `CCPPDatabaseObj`, etc.) makes + it unmaintainable. Three developers spent considerable time trying to add DDT argument + passing and could not succeed. + +The redesign starts fresh, drawing lessons from both. The guiding principle is: +**simplicity of prebuild, feature set of capgen**. + +The primary failures that triggered the redesign: +1. capgen passes flat fields to group caps — infeasible at UFS/NEPTUNE scale (1200+ + variables), breaks under compiler debug flags for optional variables. +2. capgen's scope-chain variable promotion is the source of most complexity. +3. Nobody on the team fully understands capgen. + +--- + +## 2. Toolchain Structure + +The redesign produces **two separate tools** that share the same metadata parsing library: + +### 2.1 Validator (`ccpp_validator.py`) + +Parses both Fortran source files and metadata files, compares them, and reports +discrepancies. Run by developers before invoking the generator — e.g., during scheme +development or in CI. Does **not** generate any Fortran output. + +For each scheme phase declared in a `.meta` file, the validator checks that the +corresponding Fortran subroutine: (1) exists in the source tree, (2) has the same number +of dummy arguments, and (3) the argument names match the `local_name` values in the +metadata (order-insensitive). + +Fortran source files can be supplied explicitly on the CLI (`--source-files`). When +omitted, the validator auto-discovers the Fortran source for each scheme table using the +`source_path` table-level property (Section 3.5): it looks for a `.F90` file with the +same base name as the `.meta` file, in the directory given by `source_path`. + +### 2.2 Code Generator (`ccpp_capgen_ng.py`) + +Parses metadata only. Assumes metadata correctly describes the Fortran source — performs +no Fortran parsing. Generates all cap files and supporting modules. + +**Both tools import the same metadata parsing module.** No duplication of metadata +parsing logic between the two tools. + +--- + +## 3. Metadata Format + +### 3.1 File format + +The existing ini-file format is preserved unchanged. Every metadata file consists of +`[ccpp-table-properties]` header blocks followed by `[ccpp-arg-table]` variable listing +blocks, exactly as in the current framework. + +The `[ccpp-table-properties]` + `[ccpp-arg-table]` pair is redundant for non-scheme +tables (the distinction is vestigial for host/DDT/suite tables) but is preserved for +symmetry with scheme metadata tables. + +### 3.2 Table types (`type =` in `[ccpp-table-properties]`) + +Five table types are supported: + +| `type =` | Ownership | Import mechanism | +|---|---|---| +| `scheme` | Physics scheme | Intent args on scheme subroutines | +| `host` | Host model | Module USE (direct or via DDT member) | +| `control` | Framework runtime layer | Explicit args to `ccpp_physics_*` entry points | +| `suite` | Generated suite cap | Module USE of generated suite data module | +| `ddt` | Type definition | Structural — describes DDT fields, no instance info | + +Notes: +- `type = module` from capgen is renamed to `type = host`. Breaking change, intentional. +- `type = suite` tables are **written by the generator** (never hand-authored). They + appear on disk for inspection and debugging only. +- `type = ddt` describes the structure of a Fortran derived type. It contains no + instance information — only field definitions. + +### 3.3 Per-variable attributes + +All existing per-variable attributes are preserved: `standard_name`, `long_name`, +`units`, `dimensions`, `type`, `kind`, `intent`, `optional`, `active`, `protected`. + +`protected = True` means: any scheme that declares `intent` other than `in` for this +variable is a metadata error, caught at generation time. This is how constants are +handled — a constants DDT is declared `type = host` with all fields `protected = True`. +No separate `type = constants` is needed. + +### 3.4 DDT type definitions + +A DDT type definition uses `type = ddt`: + +```ini +[ccpp-table-properties] + name = gfs_statein_type + type = ddt + +[ccpp-arg-table] + name = gfs_statein_type + type = ddt + +[phii] + standard_name = geopotential_at_interface + long_name = geopotential at model layer interfaces + units = m2 s-2 + dimensions = (horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys +``` + +### 3.5 Table-level properties + +The `[ccpp-table-properties]` block supports the following table-level keys beyond `name` +and `type`: + +| Key | Applies to | Purpose | +|---|---|---| +| `source_path` | `scheme` | Relative path from the `.meta` file directory to the directory containing the corresponding Fortran `.F90` source file. Defaults to the `.meta` file's own directory if absent. Used by the validator for auto-discovery of Fortran source. | +| `dependencies` | `scheme`, `host` | Comma-separated list of dependency file names or relative paths. Resolved to absolute paths using `dependencies_path` as a base directory (or the `.meta` file's directory if `dependencies_path` is absent). | +| `dependencies_path` | `scheme`, `host` | Optional subdirectory (relative to the `.meta` file's directory) used as the base when resolving entries in `dependencies`. Has no effect if `dependencies` is absent or `none`. | + +Example: + +```ini +[ccpp-table-properties] + name = my_scheme + type = scheme + source_path = ../src + dependencies_path = ../deps + dependencies = utility_module.F90, shared_constants.F90 +``` + +The resolved `dependencies` paths are collected across all scheme tables and written to the +`` section of `datatable.xml`. The validator uses `source_path` (not +`dependencies`) for locating the Fortran `.F90` corresponding to each `.meta` file. + +**Parser implementation note:** The INI parser applies these table-level properties to +the `MetadataTable` object before transitioning to any `[ccpp-arg-table]` section. A +`flush_table_props()` call must happen at every parser-state transition (new table +header, first arg-table header, end-of-file) to avoid silently discarding the properties. + +### 3.6 DDT instances + +A DDT instance is declared as a regular variable entry inside a `type = host` table. +The enclosing `[ccpp-table-properties]` block's `name` attribute identifies the Fortran +module from which the instance is imported via `use`. No separate `module` attribute is +needed on the variable entry. + +```ini +[ccpp-table-properties] + name = CCPP_data + type = host + dependencies = CCPP_typedefs.F90,GFS_typedefs.F90 + +[ccpp-arg-table] + name = CCPP_data + type = host + +[gfs_statein] + standard_name = gfs_statein + long_name = GFS state input for all instances + units = mixed + dimensions = (number_of_instances) + type = gfs_statein_type +``` + +The generator looks up the `gfs_statein_type` DDT table, traverses its fields, and +constructs access paths of the form +`gfs_statein(instance_number)%fieldname(loop_begin:loop_end, 1:nlevs)`. + +For scalar DDT instances (no instance dimension), dimensions is `()`. +For nested DDTs, the same mechanism applies recursively. + +### 3.7 Control variable declarations + +Control variables are declared in a `type = control` table. The generator resolves +each control variable by its standard name and uses whatever local Fortran name the +host declared: + +```ini +[ccpp-table-properties] + name = my_host_control_module + type = control + +[ccpp-arg-table] + name = my_host_control_module + type = control + +[loop_begin] + standard_name = horizontal_loop_begin + long_name = start of horizontal loop + units = index + dimensions = () + type = integer +``` + +--- + +## 4. Control Variables + +The generator recognizes the following standard names for control variables. Local +Fortran names are host-defined (resolved from the `type = control` metadata table). + +### 4.1 Entry point arguments (non-register phases) + +All required control variables are unconditional — every host must declare all of them. +Models that don't use a variable pass the neutral value: `1` for integers, `''` for +character arguments. + +| Standard name | Expected type | Role | +|---|---|---| +| `suite_name` | `character` | Suite name for runtime dispatch | +| `horizontal_loop_begin` | `integer` | Start of horizontal slice (chunk bounds for `ccpp_physics_run`; `1` for all other phases) | +| `horizontal_loop_end` | `integer` | End of horizontal slice (chunk bounds for `ccpp_physics_run`; `ncols` for all other phases) | +| `thread_number` | `integer` | Current thread index (1..number_of_threads); pass `1` if single-threaded | +| `number_of_threads` | `integer` | Host blocking loop thread count; pass `1` if single-threaded | +| `number_of_physics_threads` | `integer` | Thread budget for physics-internal OpenMP; pass `1` if none | +| `ccpp_error_message` | `character` | Error message string | +| `ccpp_error_code` | `integer` | Integer error return code | +| `instance_number` | `integer` | Current model instance index; pass `1` if single-instance | + +`group_name` is **not** in the required set. It is included in the static API signature +only if the host declares it in their `type=control` table. When absent: the static API +calls all groups in declared order; no dispatch argument is generated; the generator +warns (not errors) if any loaded suite has more than one group. When present: it is a +required (non-optional) `character` argument; the value `''` (empty string) or `'all'` +calls all groups in order; any other value dispatches to the named group only. + +The generator validates the required set at startup (after host metadata is parsed): +every required standard name must be present with the expected Fortran type (rank-0 +scalar). All failures are collected and reported together before halting. + +### 4.2 Loop-generated control variables (subcycles only) + +| Standard name | Role | +|---|---| +| `ccpp_loop_counter` | Current subcycle iteration (1..ccpp_loop_extent) | +| `ccpp_loop_extent` | Total subcycle iterations; value comes from the `loop=N` attribute on the `` element in the suite XML definition file | + +These are **not** passed as `ccpp_physics_*` arguments from the host. They are set by +the generated `do` loop inside the group cap and are available to any scheme called +within that loop. Outside a subcycle loop, these variables are not in scope. + +### 4.3 Registered dimension standard names + +The generator has built-in semantic knowledge of these dimension standard names: + +| Standard name | Indexing semantic | +|---|---| +| `instance_dimension` | Scalar extraction: `var(instance_number)` — `instance_number` is the control variable | +| `horizontal_dimension` | **At scheme call sites**: always `horizontal_loop_begin:horizontal_loop_end` (using control variable local names), for all phases. **For suite-owned array allocation sizing**: local name of `horizontal_dimension` from the host `type=host` table (accessed via module USE, not the control variable). | +| `vertical_*` | Slice: `1:` | + +`horizontal_dimension` and `vertical_*` are "registered" because the generator knows +their slicing semantics, but they are resolved to local names the same way as arbitrary +dimensions — by looking up the variable with that standard name in the host metadata. +**Scheme metadata always uses `horizontal_dimension`** for the horizontal extent, regardless of which phase the entrypoint belongs to. The standard name `horizontal_loop_extent` does not exist in the new design. The distinction between a chunk call and a full-domain call is handled entirely by what the host passes for `horizontal_loop_begin` and `horizontal_loop_end` — invisible to scheme developers and to the cap generator's slicing logic. + +All other dimension standard names are resolved identically: look up the variable with +that standard name, get its local Fortran name, emit `1:local_name`. + +The timing of `instance_dimension` substitution — whether at parse time (when building +the flat dict access path) or at call-string generation time (like other registered +dimensions) — is an implementation decision left to the developer. Either is correct; +choose whichever is easier to implement, understand, and maintain. + +--- + +## 5. Entry Points + +Eight entry points are generated in the static API. Two tiers: + +### 5.1 Framework lifecycle (no group_name dispatch) + +These operate on the entire suite at once. They take `suite_name` plus +`ccpp_error_message` and `ccpp_error_code`. No scheme `_run/_init/_final` calls. + +| Entry point | Purpose | +|---|---| +| `ccpp_register(suite_name, constituents, errmsg, errcode)` | Calls each scheme's `_register` entrypoint; allocates and populates the `ccpp_model_constituents_t` object passed by the host | +| `ccpp_init(suite_name, [number_of_instances,] errmsg, errcode)` | Allocates integer state arrays in all group caps; allocates suite-owned interstitial data; no scheme calls | +| `ccpp_final(suite_name, errmsg, errcode)` | Deallocates integer state arrays and suite-owned data; no scheme calls | + +`constituents` in `ccpp_register` is `intent(inout)`: unallocated on entry, allocated +and populated on exit. The host declares and owns this object (imports +`ccpp_model_constituents_t` from the framework library). The argument is mandatory. + +`number_of_instances` in `ccpp_init` is **conditional**: included as an explicit +`intent(in)` integer argument when the host metadata declares a variable with standard +name `number_of_instances`; omitted entirely for single-instance models. The generator +detects this automatically at parse time (same conditionality rule as `instance_number`). +When present it is passed through the call chain: +`ccpp_init` → `_init` → each group's `state_alloc`. + +### 5.2 Physics group invocation (dispatched by suite_name + group_name) + +| Entry point | Calls scheme phase | +|---|---| +| `ccpp_physics_init(...)` | `_init` | +| `ccpp_physics_timestep_init(...)` | `_timestep_init` | +| `ccpp_physics_run(...)` | `_run` | +| `ccpp_physics_timestep_final(...)` | `_timestep_final` | +| `ccpp_physics_final(...)` | `_final` | + +All five take the full required control variable argument list (Section 4.1) — a uniform +signature across all phases. If `group_name` is declared in the host's `type=control` +table, it is also included; `''` or `'all'` calls all groups in order, any other value +dispatches to the named group only. If `group_name` is absent from the control table, +no dispatch argument is generated and all groups are called in order. + +The host is responsible for passing appropriate horizontal bounds: actual chunk bounds +for `ccpp_physics_run`; `1` and `ncols` (full domain) for all other phases. The cap +always uses `(horizontal_loop_begin:horizontal_loop_end)` for array slices — no +phase-specific special-casing. + +### 5.3 Naming note + +`finalize` is renamed to `final` throughout (e.g., `ccpp_physics_final`, not +`ccpp_physics_finalize`). Breaking change, intentional for symmetry: +`ccpp_init`/`ccpp_final`, `ccpp_physics_init`/`ccpp_physics_final`, +`ccpp_physics_timestep_init`/`ccpp_physics_timestep_final`. + +The SDF likewise accepts only the canonical short element names: +`` and `` (§5.5). The legacy long spellings — `` +(old typo), `` (correct long form), `` — are +rejected at parse time with a clear error pointing at the short form. + +### 5.5 Suite-level lifecycle hooks (`` / ``) + +The SDF root may declare a **single** scheme that runs at suite-init +and/or suite-final time: + +```xml + + my_init_scheme + … + my_final_scheme + +``` + +The named scheme's `init` (resp. `final`) phase is resolved from the +scheme metadata and called from inside `_init` (resp. +`_final`). Ordering: + +- `_init`: after all group `state_alloc` and + `suite_data_init_fields`, **before** the `CCPP_SUITE_FRAMEWORK_INITIALIZED` + state transition. An errflg from the init scheme prevents the + state transition. +- `_final`: before the `CCPP_SUITE_UNREGISTERED` transition. + +Constraints: + +- One scheme per `` / ``. Multiple `` children + inside (the "group" shape) is a schema violation. +- The named scheme must have the matching phase in its metadata. + Missing-phase metadata is a generator error. + +### 5.4 Suite introspection routines (planned) + +In addition to the eight entry points above, the static API will expose four +**suite-introspection** subroutines that let a host query, at runtime, what is +compiled into the API. These mirror the equivalent routines in the original +capgen (`scripts/ccpp_suite.py` — `write_inspection_routines`) and are required +for CMake integration and host-side build glue. They are **planned but not yet +implemented**; signatures below are the proposed shape and may be refined when +the original capgen sources are reviewed for adoption. + +| Entry point | Purpose | +|---|---| +| `ccpp_physics_suite_list(suites)` | Return all suite names compiled into the API | +| `ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errcode)` | Return the list of group ("part") names for a given suite | +| `ccpp_physics_suite_variables(suite_name, variable_list, errmsg, errcode, [input_vars], [output_vars], [struct_elements])` | Return the standard-name list a suite consumes/produces; optional flags filter by intent and whether DDT sub-fields are flattened | +| `ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errcode)` | Return the list of scheme module names that compose a suite | + +These routines do not advance the state machine and do not call any scheme +entrypoints. All inputs derive from generator-time data already held in +`SuiteResolution` plus the host/scheme metadata; no new metadata is required. + +--- + +## 6. Cap Hierarchy + +All three levels are fully auto-generated. No hand-written components in the cap layer. + +### 6.1 Static API (`ccpp_static_api.F90`) + +- Imports all host DDTs and flat fields via `module use` (resolved from host metadata) +- Does not USE `ccpp_kinds` directly: the static API has no kind-typed declarations of + its own (it dispatches by `suite_name` and forwards control args). `ccpp_kinds` is + USEd only by files that declare kind-typed variables: group caps, the suite types + module, and the suite data module. +- Dispatches all eight entry points by `suite_name` to the appropriate suite cap +- Passes `ccpp_model_constituents_t` through as an explicit argument (does not own it) +- Holds no physics state + +### 6.2 Suite cap (`ccpp__cap.F90`) + +- Imports the generated suite data module (`ccpp__data.F90`) +- Contains the suite-level integer state array: `integer, allocatable :: ccpp_suite_state(:)` indexed by instance +- Implements the suite-level state machine (see Section 7) +- On `ccpp_register`: calls all scheme `_register` entrypoints across the suite +- On `_init(number_of_instances, errmsg, errflg)`: calls `state_alloc` for every + group, passing `number_of_instances` (or literal `1` for single-instance hosts); + also allocates suite-owned interstitial data. The `number_of_instances` argument is + conditional on the host declaring it (Section 7.2.1). +- On `ccpp_final`: deallocates all of the above +- Routes `ccpp_physics_*` calls by `group_name` to the appropriate group cap function; + passes `instance_number` (if present) through to group cap `_init` and `_final` subs + +### 6.3 Group cap (`ccpp___cap.F90`) + +- Imports `ccpp__data` (suite-owned interstitial data) +- Imports `ccpp__types` (shared wrapper types) +- Contains the group-level integer state array: `integer, allocatable :: ccpp_group_state(:)` indexed by instance +- Implements the group-level state machine (see Section 7) +- Contains the actual scheme call sites for each phase: + - Loop bound locals + - Optional variable pointer arrays (thread-dimensioned) + - Fixed-index extraction locals + - Unit/kind conversion locals + - Subcycle `do` loops + - Scheme calls with full argument lists + +--- + +## 7. State Machine + +Integer state parameters are defined as **private named parameters directly inside each +generated group cap module** — they are NOT imported from a shared framework library +module. Each group cap file declares: + +```fortran +integer, parameter, private :: CCPP_GROUP_UNINITIALIZED = 0 +integer, parameter, private :: CCPP_GROUP_INITIALIZED = 1 +integer, parameter, private :: CCPP_GROUP_IN_TIMESTEP = 2 +``` + +This means the integer values are replicated across generated files (acceptable — the +names are the contract, not the values). No generated file USEs a framework state module. + +Two levels, both indexed by `instance_number`. + +### 7.1 Suite-level state (in suite cap) + +```fortran +integer, parameter :: CCPP_SUITE_UNREGISTERED = 0 +integer, parameter :: CCPP_SUITE_REGISTERED = 1 +integer, parameter :: CCPP_SUITE_FRAMEWORK_INITIALIZED = 2 +``` + +| Entry point | Required state | State after | +|---|---|---| +| `ccpp_register` | `== UNREGISTERED` | `REGISTERED` | +| `ccpp_init` | `== REGISTERED` | `FRAMEWORK_INITIALIZED` | +| `ccpp_physics_*` | `== FRAMEWORK_INITIALIZED` | (unchanged) | +| `ccpp_final` | `== FRAMEWORK_INITIALIZED` | `REGISTERED` | + +### 7.2 Group-level state (in each group cap) + +```fortran +integer, parameter :: CCPP_GROUP_UNINITIALIZED = 0 +integer, parameter :: CCPP_GROUP_INITIALIZED = 1 +integer, parameter :: CCPP_GROUP_IN_TIMESTEP = 2 +``` + +| Entry point | Required state | State after | +|---|---|---| +| `ccpp_physics_init` | `< INITIALIZED` (idempotent if `== INITIALIZED`) | `INITIALIZED` | +| `ccpp_physics_timestep_init` | `== INITIALIZED` | `IN_TIMESTEP` | +| `ccpp_physics_run` | `== IN_TIMESTEP` | `IN_TIMESTEP` | +| `ccpp_physics_timestep_final` | `== IN_TIMESTEP` | `INITIALIZED` | +| `ccpp_physics_final` | `>= INITIALIZED` | `UNINITIALIZED` | + +The idempotency rule for `ccpp_physics_init`: if the group is already in state +`INITIALIZED`, return immediately without calling any scheme `_init` routines. This +allows the host to call `ccpp_physics_init` multiple times safely. Any further call +after the first must result in no change (idempotency is a scheme contract). + +#### 7.2.1 State array allocation and instance indexing + +Each group cap declares an allocatable module-level array: + +```fortran +integer, private, allocatable :: ccpp_group_state(:) +``` + +Two generated subroutines manage it: + +```fortran +! Always takes number_of_instances as an explicit arg — never USEs a host module. +subroutine ccpp___state_alloc(number_of_instances, errmsg, errflg) + integer, intent(in) :: number_of_instances + ... + allocate(ccpp_group_state(number_of_instances)) + ccpp_group_state(:) = CCPP_GROUP_UNINITIALIZED + +subroutine ccpp___state_dealloc(errmsg, errflg) + ... + if (allocated(ccpp_group_state)) deallocate(ccpp_group_state) +``` + +`state_alloc` is called from the suite cap's `_init` subroutine. The count is +passed as an explicit argument: the local name of `number_of_instances` from host +metadata (multi-instance), or the integer literal `1` (single-instance): + +```fortran +! Multi-instance (host provides number_of_instances with local name ninstances): +subroutine test_suite_init(ninstances, errmsg, errflg) + integer, intent(in) :: ninstances + ... + call ccpp_test_suite_physics_state_alloc(ninstances, errmsg, errflg) + +! Single-instance (no number_of_instances in host metadata): +subroutine test_suite_init(errmsg, errflg) + ... + call ccpp_test_suite_physics_state_alloc(1, errmsg, errflg) +``` + +State array **indexing** in the phase subroutines uses the local name of +`instance_number` (e.g. `inst_num`) when the host provides it, otherwise the literal +`1`: + +```fortran +subroutine ccpp___init(inst_num, ...) + if (ccpp_group_state(inst_num) >= CCPP_GROUP_INITIALIZED) return + ... + ccpp_group_state(inst_num) = CCPP_GROUP_INITIALIZED +``` + +`instance_number` is injected into the `_init` and `_final` phase subroutine signatures +even when no scheme in those phases uses it directly — the state guard and state +transition require it. It does **not** appear in `_run`, `_timestep_init`, or +`_timestep_final` unless a scheme in those phases explicitly requests it. + +These two integer arrays replace both the boolean `initialized(:)` array from prebuild +and the string-based `ccpp_suite_state` from CAM-SIMA. + +--- + +## 8. Scheme Metadata and Variable Matching + +### 8.1 Scheme metadata structure + +Each scheme source file has a companion `.meta` file with `type = scheme` tables — one +table per public phase subroutine (`scheme_name_init`, `scheme_name_run`, +`scheme_name_timestep_init`, etc.). The section header for each variable entry is the +**local variable name** as it appears in the scheme's Fortran subroutine argument list. + +The internal metadata store is keyed as: + +``` +metadata[scheme_name][phase][standard_name] → {local_name, units, kind, dimensions, + intent, optional, active, ...} +``` + +Keying by `scheme_name` then `phase` enables cross-phase queries ("does this scheme +have a register phase?", "what are all variables of scheme X?") and matches the +conceptual model of the suite XML. + +### 8.2 Reading order + +All metadata files (host + scheme + DDT) are read in one pass without resolving DDT +type references. After the full read, the generator builds the known DDT list, then +resolves all type references. This avoids ordering dependencies between metadata files. + +### 8.3 Known DDT list + +After reading all metadata, the generator assembles the set of known DDT types from +`type = ddt` tables. Two categories: + +**Framework-defined DDTs**: declared in `type = ddt` metadata tables. The generator +knows their fields, dimensions, and access paths. + +**External DDTs**: types from external libraries (MPI, ESMF, etc.) that the generator +cannot introspect. These are declared in variable entries using an extended `type` +syntax: + +```ini +[mycomm] + standard_name = mpi_communicator + type = external:mpi_f08:mpi_comm + ... +``` + +The format is `external::`. The generator emits +`use mpi_f08, only: mpi_comm` and treats the variable as an opaque type — no field +traversal, no dimension indexing beyond what the metadata declares. + +### 8.4 Variable matching: scheme vs. host + +For each argument in a scheme's phase function (looked up by standard name): + +1. **Found in host+control flat dict** → use the resolved access path. If `units` or + `kind` differ from what the scheme declares, generate a transformation (Section 9). +2. **Not found, first use is `intent(out)`** → suite-owned variable. Add to suite data, + generate declaration in `ccpp__data.F90`. +3. **Not found, first use is `intent(in)` or `intent(inout)`** → **error**: variable + used before it is provided by any scheme or host. +4. **Found in suite data (from a prior scheme)** → use the suite data access path. + Apply transformation if needed. + +### 8.5 Cap call argument construction + +The generator builds the argument list for each scheme call from the scheme's metadata +argument order. For each argument: + +- **Direct pass-through** (no transformation, not optional): inline host access + expression — no local variable declared +- **Transformation**: cap-local temporary named after the scheme's local variable name + (from scheme metadata section header); see Section 10.2 for naming rules +- **Optional**: cap-local pointer array named `_p`; see Section 10.3 +- **Optional + transformation**: combined in the `if (active) then` block + +The generator does not parse Fortran source. All local names, types, kinds, dimensions, +and intents come exclusively from metadata. + +--- + +## 9. Variable Resolution and Access Path Construction + +### 9.1 Flat storage model (host+control+suite) + +The generator flattens the DDT hierarchy at parse time. After parsing, all host, +control, and suite variables are stored in a flat dictionary keyed by standard name. +Each entry contains: +- The Fortran local name +- The fully-qualified access path (e.g., `gfs_statein(instance_number)%phii`) +- The module to USE +- Dimension information with registered/arbitrary classification + +The DDT hierarchy is discarded after the flat dict is built. No live DDT object tree +is maintained during code generation. + +### 9.2 Access path construction + +For each variable, the generator constructs the call-site expression by applying +dimension rules to each dimension in order: + +1. **`instance_dimension`** → substitute `instance_number` (scalar extraction) +2. **`horizontal_dimension`** → always substitute `horizontal_loop_begin:horizontal_loop_end` + (using control variable local names) at scheme call sites. For suite-owned array + allocation sizing, `horizontal_dimension` from the host `type=host` table is used directly. +3. **`vertical_*`** → substitute `1:local_vertical_dimension` +4. **Arbitrary dimension** → resolve to local name via its own metadata entry, emit + `1:local_name` +5. **`active` condition** → generate optional pointer-association guard (see Section 10.3) + +### 9.3 Module USE + +For each variable used in a group cap, the generator emits a `use module, only: varname` +statement. The module name comes from the enclosing `[ccpp-table-properties]` block name. +For suite-owned variables, the module is the generated `ccpp__data`. + +### 9.4 Eliminating TYPEDEFS_NEW_METADATA + +The manually-maintained `TYPEDEFS_NEW_METADATA` Python dict from prebuild is eliminated. +All information previously in that dict is now in metadata: +- The DDT type structure → `type = ddt` table +- The module-level instance → variable entry in the `type = host` table, module + implied by enclosing table name + +--- + +## 10. Variable Transformations and Optional Variables + +Variable transformations (unit/kind conversions) and optional variable handling are +combined — both occur within the same `if (active) then` block when a variable is +optional. + +### 10.1 Supported transformations + +- **Unit conversions** (e.g., Pa → hPa): formula from built-in conversion table (shared + with validator), keyed on source/target unit pair from metadata `units` attribute +- **Kind conversions** (e.g., r8 → kind_phys): from `kind` metadata attribute comparison + +The transformation framework is generic and pluggable — additional transformation types +(e.g., vertical flipping) can be added without restructuring the generator. + +### 10.2 Local variable naming + +The local variable name for a transformation temporary or optional pointer is derived +from the **scheme's local variable name** as declared in the scheme's metadata section +header (e.g., `[phii]` → local name is `phii`). The generator has this name without +parsing any Fortran. + +- Transformation temporary: scheme's local name + `_l` (e.g., `phii_l`) +- Optional pointer: scheme's local name + `_p` (e.g., `phii_p`) +- Conflict resolution: if two schemes in the same group cap use the same local name for + different standard names, append a numeric suffix before the suffix (e.g., `phii_2_l`) + +The generator validates that all generated local variable names and generated subroutine +names stay within Fortran's 63-character identifier limit. Violations are code-generation +errors — the developer must use a shorter local name in their metadata. + +The `active` expression in metadata is a Fortran logical expression written using +**CCPP standard names** (not local names). The generator translates all standard names +in the expression to their local Fortran names before emitting. + +Transformations **always** use a local temporary variable. The host variable is never +modified in-place — required for bit-for-bit reproducibility and to leave host data +uncorrupted if an exception occurs. Every conversion line carries an inline Fortran +comment (e.g., `! unit conversion: Pa to hPa`). Transformation mismatches (unknown +unit pair, unknown kind pair) are code-generation errors — no stdout/stderr from +generated code. + +### 10.3 The four cases + +The generator handles exactly four combinations per variable: + +**Case 1: No pointer, no transformation** (not optional, no unit/kind mismatch) + +No local variable is declared. The host access expression is used inline at the call +site: +```fortran +call scheme_run(..., gfs_statein(instance_number)%phii(lb:ub,1:nlevs), ...) +``` + +**Case 2: Pointer only** (optional, no transformation) + +Pointer array declared at function top; conditional association in `if (active)` block: +```fortran +! declaration: +type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) + +! before call: +if () then + phii_p(thread_number)%ptr => gfs_statein(instance_number)%phii(lb:ub,1:nlevs) +else + nullify(phii_p(thread_number)%ptr) +end if + +call scheme_run(..., phii_p(thread_number)%ptr, ...) + +! after call: +nullify(phii_p(thread_number)%ptr) +``` + +**Case 3: Transformation only** (not optional, unit/kind mismatch) + +Local temporary `phii_l` (scheme's local name + `_l`); intent-driven emission: + +| `intent` | Pre-call | Post-call | +|---|---|---| +| `in` | `phii_l = host_phii(...) * factor ! unit conversion: X to Y` | nothing | +| `out` | nothing | `host_phii(...) = phii_l / factor ! unit conversion: Y to X` | +| `inout` | pre-call as above | post-call as above | + +```fortran +! declaration: +real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) ! or appropriate rank/kind + +! before call (intent in/inout): +phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa + +call scheme_run(..., phii_l, ...) + +! after call (intent inout/out): +gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa +``` + +**Case 4: Pointer and transformation** (optional + unit/kind mismatch) + +Two local variables: `phii_l` (transformation temporary) and `phii_p` (pointer array). +Sequence: (1) apply transformation to `phii_l` depending on intent, (2) assign pointer +to `phii_l`, (3) call scheme, (4) nullify pointer, (5) apply back-transformation from +`phii_l` to host depending on intent. All within the `if (active)` block: + +```fortran +! declarations: +real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) +type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) + +! before call: +if () then + ! step 1: apply forward transformation (intent in/inout) + phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa + ! step 2: assign pointer to transformed local + phii_p(thread_number)%ptr => phii_l +else + nullify(phii_p(thread_number)%ptr) +end if + +call scheme_run(..., phii_p(thread_number)%ptr, ...) + +! after call: +if () then + ! step 4: nullify pointer + nullify(phii_p(thread_number)%ptr) + ! step 5: apply back-transformation (intent inout/out) + gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa +end if +``` + +The wrapper types (`real_kind_phys_rank1_ptr_type` etc.) are defined once in the +generated shared types module (`ccpp__types.F90`), not re-declared inside every +group cap function. Passing an unassociated pointer is safe under all compiler modes. + +--- + +## 11. Subcycle Loops + +When a group in the suite XML contains ``, the generator emits a +Fortran `do` loop in the group cap: + +```fortran +do = 1, N ! subcycle: N iterations from suite XML + ! ... scheme calls ... +end do +``` + +`ccpp_loop_counter` and `ccpp_loop_extent` are **loop-context variables** — a special +class that does not fit any of the five table types: +- NOT `type = control`: not passed as `ccpp_physics_*` arguments from the host +- NOT `type = host`: not from host module USE +- NOT `type = suite`: not persistent allocated data + +The generator has built-in knowledge of these two standard names. `ccpp_loop_extent` +value comes from the `loop=N` attribute in the suite XML definition file. +`ccpp_loop_counter` is the do loop induction variable. Both exist only within the +generated loop scope — they are not in scope outside a subcycle block. + +Any scheme that requests `ccpp_loop_counter` or `ccpp_loop_extent` by standard name +receives the loop variables at the call site. Their local names in the cap are derived +from the scheme's metadata section headers as for any other variable (Section 10.2). + +--- + +## 12. Init/Finalize Deduplication + +If the same scheme appears more than once within a single group (e.g., via subcycles), +having its `_init` called multiple times is a **code generator bug** — the generator +**errors out** rather than silently deduplicating. The suite XML must not list the same +scheme multiple times in the same group for non-run phases. + +If the same scheme appears in multiple groups, its `_init` is called once per group. +This is acceptable — idempotency is a contract all scheme `_init` routines must satisfy. + +The same rule applies to `_timestep_init`, `_timestep_final`, and `_final`. + +--- + +## 13. Suite-Owned Data + +### 13.1 Discovery + +The generator identifies suite-owned variables during variable resolution: variables +requested by schemes that are not satisfied by host metadata (`type = host` or +`type = control`). + +**Error condition**: if a variable is determined to be suite-owned (not provided by the +host) and the first scheme that uses it does not have it as `intent(out)`, the generator +errors out. A suite-owned variable that is first read before it is written would be +used uninitialized. + +### 13.2 Generated files and allocation + +Suite-owned variables are declared in a generated suite data module +(`ccpp__data.F90`) as fields of a Fortran DDT. The suite cap and all group caps +`use` this module. + +Allocation happens in `ccpp_init` (suite cap). Deallocation happens in `ccpp_final`. +Suite-owned arrays are allocated for the **full `horizontal_dimension`** — threads +access their respective horizontal chunk (`horizontal_loop_begin:horizontal_loop_end`) +at scheme call sites. This avoids per-call allocation overhead. If this proves to +consume too much memory at scale, a future revision may move allocation to per-phase +with chunk-sized arrays; start with the full-dimension approach. + +Subsetting (applying horizontal loop bounds, instance index) happens at scheme call +sites in the group cap, not in the suite cap. + +### 13.3 Metadata + +The generator also writes a `type = suite` metadata table (`ccpp_.meta`) as a +byproduct. This file is for inspection and debugging — it is not consumed by the +generator on subsequent runs. + +--- + +## 14. Constituent API + +> **Status (2026-05-12).** The constituent API in capgen-ng has evolved past +> the sketch below. The current implementation is: +> +> - One `ccpp_model_constituents_obj(:)` array (sized to +> `number_of_instances`), declared and owned by the **generator** in +> `ccpp_host_constituents.F90` — not by the host. +> - Host-facing API: `ccpp_register_constituents(host_constituents, +> instance_number, ...)`, `ccpp_initialize_constituents`, +> `ccpp_number_constituents`, `ccpp_const_get_index`, +> `ccpp_constituents_array(instance_number)`, etc. All per-instance. +> - Schemes follow four rules: register-phase +> `ccpp_constituent_properties_t(:), intent=out, allocatable`; +> physics-phase consume via `advected=true intent=in/inout`; tendency +> produce via `constituent=true intent=out` + `tendency_of_` +> std_name; mismatched combos are codegen errors. +> - **Authoritative reference**: `doc/constituents.md` (full lifecycle + +> API + examples). +> - **Architecture review and proposed reforms**: `doc/constituents_overhaul.md` +> (2026-05-12, meeting-quality discussion of original capgen vs +> capgen-ng vs cam-sima needs, bugs/flaws, class-A/B property +> classification, three proposals A/B/C). +> +> The historic text below is retained for context but does not describe +> the live system. + +### 14.1 Type definition (historic) + +`ccpp_model_constituents_t` is unchanged from CAM-SIMA. The type definition lives in +the framework library (`ccpp_constituent_prop_mod`), not in generated code. + +### 14.2 Ownership and lifecycle (historic — superseded) + +The **host model** declares and owns the constituent object: + +```fortran +use ccpp_constituent_prop_mod, only: ccpp_model_constituents_t +type(ccpp_model_constituents_t) :: constituents ! unallocated initially +``` + +The host passes it to `ccpp_register`, which allocates and populates it. After +`ccpp_register` returns, the host holds a fully allocated object ready for `lock_table`, +`const_index`, `copy_in`, `copy_out`. + +The `constituents` argument to `ccpp_register` is mandatory. + +### 14.3 Register phase mechanics (historic — superseded) + +The suite cap's register routine: +1. Iterates over all constituent-providing schemes in the suite +2. Calls each scheme's `_register` entrypoint, which returns a + `ccpp_constituent_properties_t` array +3. Collects these arrays and populates the constituent object + +No `group_name` dispatch is needed for register — it operates on the whole suite. + +--- + +## 15. Generated Output Files + +All files are written to `--output-root`. + +| File | Contents | +|---|---| +| `ccpp_kinds.F90` | Kind parameter definitions. **Always generated.** Re-exports specs from `iso_fortran_env` (default) or host-supplied modules as `integer, parameter, public :: = `. If no `--kind-type` is supplied, `kind_phys=iso_fortran_env:REAL64` is injected automatically (logged at INFO). | +| `ccpp_static_api.F90` | Static API — host imports, suite_name dispatch | +| `ccpp__cap.F90` | Suite cap — suite data import, state machine, group dispatch | +| `ccpp___cap.F90` | Group cap — scheme call sites, state array, optionals, transformations. USEs `ccpp_kinds` for any kind referenced in transformation temporaries. | +| `ccpp__data.F90` | Suite data module — framework-owned interstitial DDT. USEs `ccpp_kinds` for any kind referenced in suite-var declarations. | +| `ccpp__types.F90` | Shared cap types — optional pointer wrapper types, transformation locals. USEs `ccpp_kinds` for any kind referenced in pointer wrappers. | +| `ccpp_.meta` | Generated `type = suite` metadata table (output-only, for inspection) | +| `datatable.xml` | Generator database for `ccpp_datafile.py` queries. `ccpp_kinds.F90` and `ccpp_static_api.F90` appear in `...` (matches original capgen). | + +`ccpp_kinds.F90` is a dependency of all generated Fortran files that reference any kind parameter (group cap, suite types, suite data). The static API and suite cap have no kind references and do not USE it. + +--- + +## 16. CLI Invocation + +``` +ccpp_capgen_ng.py + --host-name + --host-files + --scheme-files + --suites + --output-root + --kind-type NAME=[MODULE:]SPEC # repeatable, see § 16.1 + --verbose # once = info; twice = debug +``` + +The generator also supports programmatic Python invocation (import and call directly), +using the same internal code paths as the CLI. + +### 16.1 Kind specifications + +Each `--kind-type` maps a CCPP-visible kind name to a Fortran precision constant. Syntax: + +``` +--kind-type =[:] +``` + +* `` — kind name as published in `ccpp_kinds` and referenced in scheme metadata + (e.g. `kind_phys`). +* `` — name of a precision constant (kind parameter) defined in some Fortran + module. +* `` — Fortran module that defines ``. **Optional**: when omitted, + `` must be a standard `ISO_FORTRAN_ENV` constant (`REAL32`, `REAL64`, `INT32`, + ...) and the module defaults to `iso_fortran_env`. If `` is not a known ISO + constant, omitting `` is an error. + +Examples: + +* `--kind-type kind_phys=REAL64` → + `use iso_fortran_env, only: REAL64; integer, parameter, public :: kind_phys = REAL64` +* `--kind-type kind_phys=my_host_kinds:kind_r8` → + `use my_host_kinds, only: kind_r8; integer, parameter, public :: kind_phys = kind_r8` + +The flag may be specified multiple times. `ccpp_kinds.F90` is **always generated**. If +no `--kind-type` is supplied (or `kind_phys` is omitted from a non-empty list), the +generator injects `kind_phys=iso_fortran_env:REAL64` and logs an INFO message. + +### 16.2 datatable.xml and ccpp_datafile.py + +The generator emits `datatable.xml` encoding the full relationships between suites, +groups, schemes, and variables. A separate query utility (`ccpp_datafile.py`) provides +a rich query interface used by CMake and other build systems. The full query surface of +the existing `ccpp_datafile.py` is preserved — the simplified `--ccpp-files` query is +one of many. + +The XML structure is: + +```xml + + + + /abs/path/ccpp_kinds.F90 + /abs/path/ccpp_static_api.F90 + + + /abs/path/ccpp__cap.F90 + ... + + + + + + + + ... + + + + + ... + + + + + + + ... + + + + + + /abs/path/to/dep.F90 + ... + + +``` + +The `` section is populated from the `dependencies` table-level property +of all scheme metadata files (Section 3.5). Paths are resolved to absolute paths at +generation time, then sorted and deduplicated before writing. + +### 16.3 CMake integration pattern + +The generator runs at CMake configure time via `execute_process`. Generated sources are +discovered by querying `ccpp_datafile.py --ccpp-files`. Host and scheme Fortran sources +are found by replacing `.meta` with `.F90` (same base name convention). + +--- + +## 17. Design Decisions Not Carried Forward + +The following patterns from prebuild or capgen are explicitly **not** carried forward: + +| Pattern | Reason | +|---|---| +| `ccpp_t` / `cdata` struct | Replaced by explicit named control variable arguments | +| `TYPEDEFS_NEW_METADATA` Python dict | Replaced by DDT instance declarations in metadata | +| String-based `ccpp_suite_state` | Replaced by integer state arrays with named parameters | +| Boolean `initialized(:)` array | Replaced by integer state arrays | +| Flat-field arguments to group caps | DDT arguments are used instead (as in prebuild) | +| Scope-chain variable promotion | Suite-owned variables explicitly discovered and declared | +| Fortran-vs-metadata validation in generator | Moved to standalone validator tool | +| Re-declaration of pointer wrapper types per function | Declared once in shared types module | +| `ccpp_physics_suite_init/finalize` | Replaced by `ccpp_init`/`ccpp_final` | +| `type = module` (capgen) | Renamed to `type = host` | +| `finalize` phase name | Renamed to `final` | +| Array size checks in caps | Not generated by default; rely on compiler bounds checking | +| Auto-clone of `is_constituent` scheme args into framework `%instantiate` calls | Replaced by explicit registration (host_constituents arg + register-phase `ccpp_constituent_properties_t`) | +| `ConstituentVarDict` synthetic scope between suite and host | Removed; constituents are a `source='constituent'` classification on `ResolvedArg` | + +--- + +## 18. Outstanding Work + +See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` +(deferred items) for the canonical list. Snapshot as of 2026-05-12 +(end of session): + +### Landed this session + +- **`instance_number` / `number_of_instances` paired opt-in** — hosts + may omit both for a single-instance API. +- **Module-name override** — `module_name = ` on + `[ccpp-table-properties]` for scheme/host/ddt. +- **Vertical-flip transform** — `top_at_one` per-var attribute; + composes with unit/kind transforms. +- **Multiple `dependencies = …` lines** per `[ccpp-table-properties]`. +- **Sliced local names** with long subscript-token CCPP standard names + no longer trip the 63-char Fortran-id limit. +- **Unit normalisation** — `m2` ≡ `m+2` (and friends). +- **Subcycle bound = CCPP std name** — including DDT-component access + paths (`phys_state%num_subcycles`). +- **Nested ``** — preserved end-to-end as nested `do` loops. +- **Active-expression + subcycle bounds** included in introspection + inputs. +- **TARGET on `ccpp_suite_data(:)`** module-level array. +- **Group-state alloc idempotency** (matches suite-state alloc). +- **Framework PR**: `ccpt_deallocate` ownership tracking via + `framework_owns_me` flag. Backward-compatible. Needs upstream + merge to ccpp-framework + ccpp-capgen. +- **Identity unit conversions** no longer emit misleading "unit + conversion: kind_phys to kind_phys" comment. +- **Improved duplicate-standard-name error** lists both colliding + access paths. +- **Suite-level `` / ``** SDF elements consumed: named + scheme's init/final phase emitted inside `_init` / + `_final`. Single scheme only; long-form spellings + (``, ``, ``) rejected. +- **Test count**: 1070 passing. + +### Still deferred + +- **End-to-end integration tests** — user-driven; off-limits for in-session + edits. +- **Constituents overhaul** — discussion doc at + `doc/constituents_overhaul.md` (2026-05-12). Three proposals on the + table (A bugfix-only / B class-A/B split + setters / C host-only + registration). Pending decision in upcoming meeting. +- **Framework setter additions** — `set_advected`, `set_diagnostic_name`, + `set_default_value`, possibly `set_mixing_ratio_type`. Coordinated with + the overhaul. +- **Validator host-metadata check** — `ccpp_validator.py` is currently + scheme-only; revisit after the e2e test suite settles. See + `project_validator_host_check_deferred.md` (memory). +- **Codegen-time scheme-registration cross-check** — new metadata attr + `registers_std_names = a, b, c` on register-phase tables; replaces + current runtime `int_unassigned` check with codegen-time error. +- **Capgen-ng cleanup**: replace `_FRAMEWORK_CONST_DIM_INPUTS` frozenset + in `generator/static_api.py` with `used_const_dim_std_names: Set[str]` + on `ResolvedArg`. +- **Nested subcycle `ccpp_loop_counter` semantics**: currently a scheme + inside a nested subcycle requesting `ccpp_loop_counter` would get + the OUTERMOST counter, not the innermost. None of the cam-sima + schemes use this — revisit if a real scheme needs the innermost. + +### Where to find the migration summary + +`doc/migration.md` (created 2026-05-12) — user-facing single-page +summary of metadata + SDF + host-Fortran requirements after all the +above changes. Read it first when porting a host model. diff --git a/doc/redesign_prompt.md - updated 20260513T0733.md b/doc/redesign_prompt.md - updated 20260513T0733.md new file mode 100644 index 00000000..c8e1ab3b --- /dev/null +++ b/doc/redesign_prompt.md - updated 20260513T0733.md @@ -0,0 +1,1172 @@ +# CCPP Framework Code Generator — Redesign Specification + +## Purpose + +This document is a complete implementation specification for a new CCPP Framework code +generator (`ccpp-capgen-ng`). It supersedes both `ccpp-prebuild` and `ccpp-capgen`. An +implementer should be able to build the new generator from scratch using this document +alone, supplemented by the real-world examples in `redesign_analysis.md`. + +--- + +## 1. Background and Motivation + +The CCPP Framework couples host NWP models (UFS Weather Model, NEPTUNE, CCPP-SCM, +CAM-SIMA) to physics parameterization schemes by auto-generating Fortran interface +("cap") code. Two generators exist today: + +- **`ccpp-prebuild`** — simple, procedural Python; fast; DDT arguments; used in + production by UFS, NEPTUNE, SCM. Does not support framework-owned variables. +- **`ccpp-capgen`** — complex OO Python; flat-field arguments; used in CAM-SIMA. The + deep class hierarchy (`VarDictionary`, `VarCompatObj`, `CCPPDatabaseObj`, etc.) makes + it unmaintainable. Three developers spent considerable time trying to add DDT argument + passing and could not succeed. + +The redesign starts fresh, drawing lessons from both. The guiding principle is: +**simplicity of prebuild, feature set of capgen**. + +The primary failures that triggered the redesign: +1. capgen passes flat fields to group caps — infeasible at UFS/NEPTUNE scale (1200+ + variables), breaks under compiler debug flags for optional variables. +2. capgen's scope-chain variable promotion is the source of most complexity. +3. Nobody on the team fully understands capgen. + +--- + +## 2. Toolchain Structure + +The redesign produces **two separate tools** that share the same metadata parsing library: + +### 2.1 Validator (`ccpp_validator.py`) + +Parses both Fortran source files and metadata files, compares them, and reports +discrepancies. Run by developers before invoking the generator — e.g., during scheme +development or in CI. Does **not** generate any Fortran output. + +For each scheme phase declared in a `.meta` file, the validator checks that the +corresponding Fortran subroutine: (1) exists in the source tree, (2) has the same number +of dummy arguments, and (3) the argument names match the `local_name` values in the +metadata (order-insensitive). + +Fortran source files can be supplied explicitly on the CLI (`--source-files`). When +omitted, the validator auto-discovers the Fortran source for each scheme table using the +`source_path` table-level property (Section 3.5): it looks for a `.F90` file with the +same base name as the `.meta` file, in the directory given by `source_path`. + +### 2.2 Code Generator (`ccpp_capgen_ng.py`) + +Parses metadata only. Assumes metadata correctly describes the Fortran source — performs +no Fortran parsing. Generates all cap files and supporting modules. + +**Both tools import the same metadata parsing module.** No duplication of metadata +parsing logic between the two tools. + +--- + +## 3. Metadata Format + +### 3.1 File format + +The existing ini-file format is preserved unchanged. Every metadata file consists of +`[ccpp-table-properties]` header blocks followed by `[ccpp-arg-table]` variable listing +blocks, exactly as in the current framework. + +The `[ccpp-table-properties]` + `[ccpp-arg-table]` pair is redundant for non-scheme +tables (the distinction is vestigial for host/DDT/suite tables) but is preserved for +symmetry with scheme metadata tables. + +### 3.2 Table types (`type =` in `[ccpp-table-properties]`) + +Five table types are supported: + +| `type =` | Ownership | Import mechanism | +|---|---|---| +| `scheme` | Physics scheme | Intent args on scheme subroutines | +| `host` | Host model | Module USE (direct or via DDT member) | +| `control` | Framework runtime layer | Explicit args to `ccpp_physics_*` entry points | +| `suite` | Generated suite cap | Module USE of generated suite data module | +| `ddt` | Type definition | Structural — describes DDT fields, no instance info | + +Notes: +- `type = module` from capgen is renamed to `type = host`. Breaking change, intentional. +- `type = suite` tables are **written by the generator** (never hand-authored). They + appear on disk for inspection and debugging only. +- `type = ddt` describes the structure of a Fortran derived type. It contains no + instance information — only field definitions. + +### 3.3 Per-variable attributes + +All existing per-variable attributes are preserved: `standard_name`, `long_name`, +`units`, `dimensions`, `type`, `kind`, `intent`, `optional`, `active`, `protected`. + +`protected = True` means: any scheme that declares `intent` other than `in` for this +variable is a metadata error, caught at generation time. This is how constants are +handled — a constants DDT is declared `type = host` with all fields `protected = True`. +No separate `type = constants` is needed. + +### 3.4 DDT type definitions + +A DDT type definition uses `type = ddt`: + +```ini +[ccpp-table-properties] + name = gfs_statein_type + type = ddt + +[ccpp-arg-table] + name = gfs_statein_type + type = ddt + +[phii] + standard_name = geopotential_at_interface + long_name = geopotential at model layer interfaces + units = m2 s-2 + dimensions = (horizontal_dimension, vertical_interface_dimension) + type = real + kind = kind_phys +``` + +### 3.5 Table-level properties + +The `[ccpp-table-properties]` block supports the following table-level keys beyond `name` +and `type`: + +| Key | Applies to | Purpose | +|---|---|---| +| `source_path` | `scheme` | Relative path from the `.meta` file directory to the directory containing the corresponding Fortran `.F90` source file. Defaults to the `.meta` file's own directory if absent. Used by the validator for auto-discovery of Fortran source. | +| `dependencies` | `scheme`, `host` | Comma-separated list of dependency file names or relative paths. Resolved to absolute paths using `dependencies_path` as a base directory (or the `.meta` file's directory if `dependencies_path` is absent). | +| `dependencies_path` | `scheme`, `host` | Optional subdirectory (relative to the `.meta` file's directory) used as the base when resolving entries in `dependencies`. Has no effect if `dependencies` is absent or `none`. | + +Example: + +```ini +[ccpp-table-properties] + name = my_scheme + type = scheme + source_path = ../src + dependencies_path = ../deps + dependencies = utility_module.F90, shared_constants.F90 +``` + +The resolved `dependencies` paths are collected across all scheme tables and written to the +`` section of `datatable.xml`. The validator uses `source_path` (not +`dependencies`) for locating the Fortran `.F90` corresponding to each `.meta` file. + +**Parser implementation note:** The INI parser applies these table-level properties to +the `MetadataTable` object before transitioning to any `[ccpp-arg-table]` section. A +`flush_table_props()` call must happen at every parser-state transition (new table +header, first arg-table header, end-of-file) to avoid silently discarding the properties. + +### 3.6 DDT instances + +A DDT instance is declared as a regular variable entry inside a `type = host` table. +The enclosing `[ccpp-table-properties]` block's `name` attribute identifies the Fortran +module from which the instance is imported via `use`. No separate `module` attribute is +needed on the variable entry. + +```ini +[ccpp-table-properties] + name = CCPP_data + type = host + dependencies = CCPP_typedefs.F90,GFS_typedefs.F90 + +[ccpp-arg-table] + name = CCPP_data + type = host + +[gfs_statein] + standard_name = gfs_statein + long_name = GFS state input for all instances + units = mixed + dimensions = (number_of_instances) + type = gfs_statein_type +``` + +The generator looks up the `gfs_statein_type` DDT table, traverses its fields, and +constructs access paths of the form +`gfs_statein(instance_number)%fieldname(loop_begin:loop_end, 1:nlevs)`. + +For scalar DDT instances (no instance dimension), dimensions is `()`. +For nested DDTs, the same mechanism applies recursively. + +### 3.7 Control variable declarations + +Control variables are declared in a `type = control` table. The generator resolves +each control variable by its standard name and uses whatever local Fortran name the +host declared: + +```ini +[ccpp-table-properties] + name = my_host_control_module + type = control + +[ccpp-arg-table] + name = my_host_control_module + type = control + +[loop_begin] + standard_name = horizontal_loop_begin + long_name = start of horizontal loop + units = index + dimensions = () + type = integer +``` + +--- + +## 4. Control Variables + +The generator recognizes the following standard names for control variables. Local +Fortran names are host-defined (resolved from the `type = control` metadata table). + +### 4.1 Entry point arguments (non-register phases) + +All required control variables are unconditional — every host must declare all of them. +Models that don't use a variable pass the neutral value: `1` for integers, `''` for +character arguments. + +| Standard name | Expected type | Role | +|---|---|---| +| `suite_name` | `character` | Suite name for runtime dispatch | +| `horizontal_loop_begin` | `integer` | Start of horizontal slice (chunk bounds for `ccpp_physics_run`; `1` for all other phases) | +| `horizontal_loop_end` | `integer` | End of horizontal slice (chunk bounds for `ccpp_physics_run`; `ncols` for all other phases) | +| `thread_number` | `integer` | Current thread index (1..number_of_threads); pass `1` if single-threaded | +| `number_of_threads` | `integer` | Host blocking loop thread count; pass `1` if single-threaded | +| `number_of_physics_threads` | `integer` | Thread budget for physics-internal OpenMP; pass `1` if none | +| `ccpp_error_message` | `character` | Error message string | +| `ccpp_error_code` | `integer` | Integer error return code | +| `instance_number` | `integer` | Current model instance index; pass `1` if single-instance | + +`group_name` is **not** in the required set. It is included in the static API signature +only if the host declares it in their `type=control` table. When absent: the static API +calls all groups in declared order; no dispatch argument is generated; the generator +warns (not errors) if any loaded suite has more than one group. When present: it is a +required (non-optional) `character` argument; the value `''` (empty string) or `'all'` +calls all groups in order; any other value dispatches to the named group only. + +The generator validates the required set at startup (after host metadata is parsed): +every required standard name must be present with the expected Fortran type (rank-0 +scalar). All failures are collected and reported together before halting. + +### 4.2 Loop-generated control variables (subcycles only) + +| Standard name | Role | +|---|---| +| `ccpp_loop_counter` | Current subcycle iteration (1..ccpp_loop_extent) | +| `ccpp_loop_extent` | Total subcycle iterations; value comes from the `loop=N` attribute on the `` element in the suite XML definition file | + +These are **not** passed as `ccpp_physics_*` arguments from the host. They are set by +the generated `do` loop inside the group cap and are available to any scheme called +within that loop. Outside a subcycle loop, these variables are not in scope. + +### 4.3 Registered dimension standard names + +The generator has built-in semantic knowledge of these dimension standard names: + +| Standard name | Indexing semantic | +|---|---| +| `instance_dimension` | Scalar extraction: `var(instance_number)` — `instance_number` is the control variable | +| `horizontal_dimension` | **At scheme call sites**: always `horizontal_loop_begin:horizontal_loop_end` (using control variable local names), for all phases. **For suite-owned array allocation sizing**: local name of `horizontal_dimension` from the host `type=host` table (accessed via module USE, not the control variable). | +| `vertical_*` | Slice: `1:` | + +`horizontal_dimension` and `vertical_*` are "registered" because the generator knows +their slicing semantics, but they are resolved to local names the same way as arbitrary +dimensions — by looking up the variable with that standard name in the host metadata. +**Scheme metadata always uses `horizontal_dimension`** for the horizontal extent, regardless of which phase the entrypoint belongs to. The standard name `horizontal_loop_extent` does not exist in the new design. The distinction between a chunk call and a full-domain call is handled entirely by what the host passes for `horizontal_loop_begin` and `horizontal_loop_end` — invisible to scheme developers and to the cap generator's slicing logic. + +All other dimension standard names are resolved identically: look up the variable with +that standard name, get its local Fortran name, emit `1:local_name`. + +The timing of `instance_dimension` substitution — whether at parse time (when building +the flat dict access path) or at call-string generation time (like other registered +dimensions) — is an implementation decision left to the developer. Either is correct; +choose whichever is easier to implement, understand, and maintain. + +--- + +## 5. Entry Points + +Eight entry points are generated in the static API. Two tiers: + +### 5.1 Framework lifecycle (no group_name dispatch) + +These operate on the entire suite at once. They take `suite_name` plus +`ccpp_error_message` and `ccpp_error_code`. No scheme `_run/_init/_final` calls. + +| Entry point | Purpose | +|---|---| +| `ccpp_register(suite_name, constituents, errmsg, errcode)` | Calls each scheme's `_register` entrypoint; allocates and populates the `ccpp_model_constituents_t` object passed by the host | +| `ccpp_init(suite_name, [number_of_instances,] errmsg, errcode)` | Allocates integer state arrays in all group caps; allocates suite-owned interstitial data; no scheme calls | +| `ccpp_final(suite_name, errmsg, errcode)` | Deallocates integer state arrays and suite-owned data; no scheme calls | + +`constituents` in `ccpp_register` is `intent(inout)`: unallocated on entry, allocated +and populated on exit. The host declares and owns this object (imports +`ccpp_model_constituents_t` from the framework library). The argument is mandatory. + +`number_of_instances` in `ccpp_init` is **conditional**: included as an explicit +`intent(in)` integer argument when the host metadata declares a variable with standard +name `number_of_instances`; omitted entirely for single-instance models. The generator +detects this automatically at parse time (same conditionality rule as `instance_number`). +When present it is passed through the call chain: +`ccpp_init` → `_init` → each group's `state_alloc`. + +### 5.2 Physics group invocation (dispatched by suite_name + group_name) + +| Entry point | Calls scheme phase | +|---|---| +| `ccpp_physics_init(...)` | `_init` | +| `ccpp_physics_timestep_init(...)` | `_timestep_init` | +| `ccpp_physics_run(...)` | `_run` | +| `ccpp_physics_timestep_final(...)` | `_timestep_final` | +| `ccpp_physics_final(...)` | `_final` | + +All five take the full required control variable argument list (Section 4.1) — a uniform +signature across all phases. If `group_name` is declared in the host's `type=control` +table, it is also included; `''` or `'all'` calls all groups in order, any other value +dispatches to the named group only. If `group_name` is absent from the control table, +no dispatch argument is generated and all groups are called in order. + +The host is responsible for passing appropriate horizontal bounds: actual chunk bounds +for `ccpp_physics_run`; `1` and `ncols` (full domain) for all other phases. The cap +always uses `(horizontal_loop_begin:horizontal_loop_end)` for array slices — no +phase-specific special-casing. + +### 5.3 Naming note + +`finalize` is renamed to `final` throughout (e.g., `ccpp_physics_final`, not +`ccpp_physics_finalize`). Breaking change, intentional for symmetry: +`ccpp_init`/`ccpp_final`, `ccpp_physics_init`/`ccpp_physics_final`, +`ccpp_physics_timestep_init`/`ccpp_physics_timestep_final`. + +The SDF likewise accepts only the canonical short element names: +`` and `` (§5.5). The legacy long spellings — `` +(old typo), `` (correct long form), `` — are +rejected at parse time with a clear error pointing at the short form. + +### 5.5 Suite-level lifecycle hooks (`` / ``) + +The SDF root may declare a **single** scheme that runs at suite-init +and/or suite-final time: + +```xml + + my_init_scheme + … + my_final_scheme + +``` + +The named scheme's `init` (resp. `final`) phase is resolved from the +scheme metadata and called from inside `_init` (resp. +`_final`). Ordering: + +- `_init`: after all group `state_alloc` and + `suite_data_init_fields`, **before** the `CCPP_SUITE_FRAMEWORK_INITIALIZED` + state transition. An errflg from the init scheme prevents the + state transition. +- `_final`: before the `CCPP_SUITE_UNREGISTERED` transition. + +Constraints: + +- One scheme per `` / ``. Multiple `` children + inside (the "group" shape) is a schema violation. +- The named scheme must have the matching phase in its metadata. + Missing-phase metadata is a generator error. + +### 5.4 Suite introspection routines (planned) + +In addition to the eight entry points above, the static API will expose four +**suite-introspection** subroutines that let a host query, at runtime, what is +compiled into the API. These mirror the equivalent routines in the original +capgen (`scripts/ccpp_suite.py` — `write_inspection_routines`) and are required +for CMake integration and host-side build glue. They are **planned but not yet +implemented**; signatures below are the proposed shape and may be refined when +the original capgen sources are reviewed for adoption. + +| Entry point | Purpose | +|---|---| +| `ccpp_physics_suite_list(suites)` | Return all suite names compiled into the API | +| `ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errcode)` | Return the list of group ("part") names for a given suite | +| `ccpp_physics_suite_variables(suite_name, variable_list, errmsg, errcode, [input_vars], [output_vars], [struct_elements])` | Return the standard-name list a suite consumes/produces; optional flags filter by intent and whether DDT sub-fields are flattened | +| `ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errcode)` | Return the list of scheme module names that compose a suite | + +These routines do not advance the state machine and do not call any scheme +entrypoints. All inputs derive from generator-time data already held in +`SuiteResolution` plus the host/scheme metadata; no new metadata is required. + +--- + +## 6. Cap Hierarchy + +All three levels are fully auto-generated. No hand-written components in the cap layer. + +### 6.1 Static API (`ccpp_static_api.F90`) + +- Imports all host DDTs and flat fields via `module use` (resolved from host metadata) +- Does not USE `ccpp_kinds` directly: the static API has no kind-typed declarations of + its own (it dispatches by `suite_name` and forwards control args). `ccpp_kinds` is + USEd only by files that declare kind-typed variables: group caps, the suite types + module, and the suite data module. +- Dispatches all eight entry points by `suite_name` to the appropriate suite cap +- Passes `ccpp_model_constituents_t` through as an explicit argument (does not own it) +- Holds no physics state + +### 6.2 Suite cap (`ccpp__cap.F90`) + +- Imports the generated suite data module (`ccpp__data.F90`) +- Contains the suite-level integer state array: `integer, allocatable :: ccpp_suite_state(:)` indexed by instance +- Implements the suite-level state machine (see Section 7) +- On `ccpp_register`: calls all scheme `_register` entrypoints across the suite +- On `_init(number_of_instances, errmsg, errflg)`: calls `state_alloc` for every + group, passing `number_of_instances` (or literal `1` for single-instance hosts); + also allocates suite-owned interstitial data. The `number_of_instances` argument is + conditional on the host declaring it (Section 7.2.1). +- On `ccpp_final`: deallocates all of the above +- Routes `ccpp_physics_*` calls by `group_name` to the appropriate group cap function; + passes `instance_number` (if present) through to group cap `_init` and `_final` subs + +### 6.3 Group cap (`ccpp___cap.F90`) + +- Imports `ccpp__data` (suite-owned interstitial data) +- Imports `ccpp__types` (shared wrapper types) +- Contains the group-level integer state array: `integer, allocatable :: ccpp_group_state(:)` indexed by instance +- Implements the group-level state machine (see Section 7) +- Contains the actual scheme call sites for each phase: + - Loop bound locals + - Optional variable pointer arrays (thread-dimensioned) + - Fixed-index extraction locals + - Unit/kind conversion locals + - Subcycle `do` loops + - Scheme calls with full argument lists + +--- + +## 7. State Machine + +Integer state parameters are defined as **private named parameters directly inside each +generated group cap module** — they are NOT imported from a shared framework library +module. Each group cap file declares: + +```fortran +integer, parameter, private :: CCPP_GROUP_UNINITIALIZED = 0 +integer, parameter, private :: CCPP_GROUP_INITIALIZED = 1 +integer, parameter, private :: CCPP_GROUP_IN_TIMESTEP = 2 +``` + +This means the integer values are replicated across generated files (acceptable — the +names are the contract, not the values). No generated file USEs a framework state module. + +Two levels, both indexed by `instance_number`. + +### 7.1 Suite-level state (in suite cap) + +```fortran +integer, parameter :: CCPP_SUITE_UNREGISTERED = 0 +integer, parameter :: CCPP_SUITE_REGISTERED = 1 +integer, parameter :: CCPP_SUITE_FRAMEWORK_INITIALIZED = 2 +``` + +| Entry point | Required state | State after | +|---|---|---| +| `ccpp_register` | `== UNREGISTERED` | `REGISTERED` | +| `ccpp_init` | `== REGISTERED` | `FRAMEWORK_INITIALIZED` | +| `ccpp_physics_*` | `== FRAMEWORK_INITIALIZED` | (unchanged) | +| `ccpp_final` | `== FRAMEWORK_INITIALIZED` | `REGISTERED` | + +### 7.2 Group-level state (in each group cap) + +```fortran +integer, parameter :: CCPP_GROUP_UNINITIALIZED = 0 +integer, parameter :: CCPP_GROUP_INITIALIZED = 1 +integer, parameter :: CCPP_GROUP_IN_TIMESTEP = 2 +``` + +| Entry point | Required state | State after | +|---|---|---| +| `ccpp_physics_init` | `< INITIALIZED` (idempotent if `== INITIALIZED`) | `INITIALIZED` | +| `ccpp_physics_timestep_init` | `== INITIALIZED` | `IN_TIMESTEP` | +| `ccpp_physics_run` | `== IN_TIMESTEP` | `IN_TIMESTEP` | +| `ccpp_physics_timestep_final` | `== IN_TIMESTEP` | `INITIALIZED` | +| `ccpp_physics_final` | `>= INITIALIZED` | `UNINITIALIZED` | + +The idempotency rule for `ccpp_physics_init`: if the group is already in state +`INITIALIZED`, return immediately without calling any scheme `_init` routines. This +allows the host to call `ccpp_physics_init` multiple times safely. Any further call +after the first must result in no change (idempotency is a scheme contract). + +#### 7.2.1 State array allocation and instance indexing + +Each group cap declares an allocatable module-level array: + +```fortran +integer, private, allocatable :: ccpp_group_state(:) +``` + +Two generated subroutines manage it: + +```fortran +! Always takes number_of_instances as an explicit arg — never USEs a host module. +subroutine ccpp___state_alloc(number_of_instances, errmsg, errflg) + integer, intent(in) :: number_of_instances + ... + allocate(ccpp_group_state(number_of_instances)) + ccpp_group_state(:) = CCPP_GROUP_UNINITIALIZED + +subroutine ccpp___state_dealloc(errmsg, errflg) + ... + if (allocated(ccpp_group_state)) deallocate(ccpp_group_state) +``` + +`state_alloc` is called from the suite cap's `_init` subroutine. The count is +passed as an explicit argument: the local name of `number_of_instances` from host +metadata (multi-instance), or the integer literal `1` (single-instance): + +```fortran +! Multi-instance (host provides number_of_instances with local name ninstances): +subroutine test_suite_init(ninstances, errmsg, errflg) + integer, intent(in) :: ninstances + ... + call ccpp_test_suite_physics_state_alloc(ninstances, errmsg, errflg) + +! Single-instance (no number_of_instances in host metadata): +subroutine test_suite_init(errmsg, errflg) + ... + call ccpp_test_suite_physics_state_alloc(1, errmsg, errflg) +``` + +State array **indexing** in the phase subroutines uses the local name of +`instance_number` (e.g. `inst_num`) when the host provides it, otherwise the literal +`1`: + +```fortran +subroutine ccpp___init(inst_num, ...) + if (ccpp_group_state(inst_num) >= CCPP_GROUP_INITIALIZED) return + ... + ccpp_group_state(inst_num) = CCPP_GROUP_INITIALIZED +``` + +`instance_number` is injected into the `_init` and `_final` phase subroutine signatures +even when no scheme in those phases uses it directly — the state guard and state +transition require it. It does **not** appear in `_run`, `_timestep_init`, or +`_timestep_final` unless a scheme in those phases explicitly requests it. + +These two integer arrays replace both the boolean `initialized(:)` array from prebuild +and the string-based `ccpp_suite_state` from CAM-SIMA. + +--- + +## 8. Scheme Metadata and Variable Matching + +### 8.1 Scheme metadata structure + +Each scheme source file has a companion `.meta` file with `type = scheme` tables — one +table per public phase subroutine (`scheme_name_init`, `scheme_name_run`, +`scheme_name_timestep_init`, etc.). The section header for each variable entry is the +**local variable name** as it appears in the scheme's Fortran subroutine argument list. + +The internal metadata store is keyed as: + +``` +metadata[scheme_name][phase][standard_name] → {local_name, units, kind, dimensions, + intent, optional, active, ...} +``` + +Keying by `scheme_name` then `phase` enables cross-phase queries ("does this scheme +have a register phase?", "what are all variables of scheme X?") and matches the +conceptual model of the suite XML. + +### 8.2 Reading order + +All metadata files (host + scheme + DDT) are read in one pass without resolving DDT +type references. After the full read, the generator builds the known DDT list, then +resolves all type references. This avoids ordering dependencies between metadata files. + +### 8.3 Known DDT list + +After reading all metadata, the generator assembles the set of known DDT types from +`type = ddt` tables. Two categories: + +**Framework-defined DDTs**: declared in `type = ddt` metadata tables. The generator +knows their fields, dimensions, and access paths. + +**External DDTs**: types from external libraries (MPI, ESMF, etc.) that the generator +cannot introspect. These are declared in variable entries using an extended `type` +syntax: + +```ini +[mycomm] + standard_name = mpi_communicator + type = external:mpi_f08:mpi_comm + ... +``` + +The format is `external::`. The generator emits +`use mpi_f08, only: mpi_comm` and treats the variable as an opaque type — no field +traversal, no dimension indexing beyond what the metadata declares. + +### 8.4 Variable matching: scheme vs. host + +For each argument in a scheme's phase function (looked up by standard name): + +1. **Found in host+control flat dict** → use the resolved access path. If `units` or + `kind` differ from what the scheme declares, generate a transformation (Section 9). +2. **Not found, first use is `intent(out)`** → suite-owned variable. Add to suite data, + generate declaration in `ccpp__data.F90`. +3. **Not found, first use is `intent(in)` or `intent(inout)`** → **error**: variable + used before it is provided by any scheme or host. +4. **Found in suite data (from a prior scheme)** → use the suite data access path. + Apply transformation if needed. + +### 8.5 Cap call argument construction + +The generator builds the argument list for each scheme call from the scheme's metadata +argument order. For each argument: + +- **Direct pass-through** (no transformation, not optional): inline host access + expression — no local variable declared +- **Transformation**: cap-local temporary named after the scheme's local variable name + (from scheme metadata section header); see Section 10.2 for naming rules +- **Optional**: cap-local pointer array named `_p`; see Section 10.3 +- **Optional + transformation**: combined in the `if (active) then` block + +The generator does not parse Fortran source. All local names, types, kinds, dimensions, +and intents come exclusively from metadata. + +--- + +## 9. Variable Resolution and Access Path Construction + +### 9.1 Flat storage model (host+control+suite) + +The generator flattens the DDT hierarchy at parse time. After parsing, all host, +control, and suite variables are stored in a flat dictionary keyed by standard name. +Each entry contains: +- The Fortran local name +- The fully-qualified access path (e.g., `gfs_statein(instance_number)%phii`) +- The module to USE +- Dimension information with registered/arbitrary classification + +The DDT hierarchy is discarded after the flat dict is built. No live DDT object tree +is maintained during code generation. + +### 9.2 Access path construction + +For each variable, the generator constructs the call-site expression by applying +dimension rules to each dimension in order: + +1. **`instance_dimension`** → substitute `instance_number` (scalar extraction) +2. **`horizontal_dimension`** → always substitute `horizontal_loop_begin:horizontal_loop_end` + (using control variable local names) at scheme call sites. For suite-owned array + allocation sizing, `horizontal_dimension` from the host `type=host` table is used directly. +3. **`vertical_*`** → substitute `1:local_vertical_dimension` +4. **Arbitrary dimension** → resolve to local name via its own metadata entry, emit + `1:local_name` +5. **`active` condition** → generate optional pointer-association guard (see Section 10.3) + +### 9.3 Module USE + +For each variable used in a group cap, the generator emits a `use module, only: varname` +statement. The module name comes from the enclosing `[ccpp-table-properties]` block name. +For suite-owned variables, the module is the generated `ccpp__data`. + +### 9.4 Eliminating TYPEDEFS_NEW_METADATA + +The manually-maintained `TYPEDEFS_NEW_METADATA` Python dict from prebuild is eliminated. +All information previously in that dict is now in metadata: +- The DDT type structure → `type = ddt` table +- The module-level instance → variable entry in the `type = host` table, module + implied by enclosing table name + +--- + +## 10. Variable Transformations and Optional Variables + +Variable transformations (unit/kind conversions) and optional variable handling are +combined — both occur within the same `if (active) then` block when a variable is +optional. + +### 10.1 Supported transformations + +- **Unit conversions** (e.g., Pa → hPa): formula from built-in conversion table (shared + with validator), keyed on source/target unit pair from metadata `units` attribute +- **Kind conversions** (e.g., r8 → kind_phys): from `kind` metadata attribute comparison + +The transformation framework is generic and pluggable — additional transformation types +(e.g., vertical flipping) can be added without restructuring the generator. + +### 10.2 Local variable naming + +The local variable name for a transformation temporary or optional pointer is derived +from the **scheme's local variable name** as declared in the scheme's metadata section +header (e.g., `[phii]` → local name is `phii`). The generator has this name without +parsing any Fortran. + +- Transformation temporary: scheme's local name + `_l` (e.g., `phii_l`) +- Optional pointer: scheme's local name + `_p` (e.g., `phii_p`) +- Conflict resolution: if two schemes in the same group cap use the same local name for + different standard names, append a numeric suffix before the suffix (e.g., `phii_2_l`) + +The generator validates that all generated local variable names and generated subroutine +names stay within Fortran's 63-character identifier limit. Violations are code-generation +errors — the developer must use a shorter local name in their metadata. + +The `active` expression in metadata is a Fortran logical expression written using +**CCPP standard names** (not local names). The generator translates all standard names +in the expression to their local Fortran names before emitting. + +Transformations **always** use a local temporary variable. The host variable is never +modified in-place — required for bit-for-bit reproducibility and to leave host data +uncorrupted if an exception occurs. Every conversion line carries an inline Fortran +comment (e.g., `! unit conversion: Pa to hPa`). Transformation mismatches (unknown +unit pair, unknown kind pair) are code-generation errors — no stdout/stderr from +generated code. + +### 10.3 The four cases + +The generator handles exactly four combinations per variable: + +**Case 1: No pointer, no transformation** (not optional, no unit/kind mismatch) + +No local variable is declared. The host access expression is used inline at the call +site: +```fortran +call scheme_run(..., gfs_statein(instance_number)%phii(lb:ub,1:nlevs), ...) +``` + +**Case 2: Pointer only** (optional, no transformation) + +Pointer array declared at function top; conditional association in `if (active)` block: +```fortran +! declaration: +type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) + +! before call: +if () then + phii_p(thread_number)%ptr => gfs_statein(instance_number)%phii(lb:ub,1:nlevs) +else + nullify(phii_p(thread_number)%ptr) +end if + +call scheme_run(..., phii_p(thread_number)%ptr, ...) + +! after call: +nullify(phii_p(thread_number)%ptr) +``` + +**Case 3: Transformation only** (not optional, unit/kind mismatch) + +Local temporary `phii_l` (scheme's local name + `_l`); intent-driven emission: + +| `intent` | Pre-call | Post-call | +|---|---|---| +| `in` | `phii_l = host_phii(...) * factor ! unit conversion: X to Y` | nothing | +| `out` | nothing | `host_phii(...) = phii_l / factor ! unit conversion: Y to X` | +| `inout` | pre-call as above | post-call as above | + +```fortran +! declaration: +real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) ! or appropriate rank/kind + +! before call (intent in/inout): +phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa + +call scheme_run(..., phii_l, ...) + +! after call (intent inout/out): +gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa +``` + +**Case 4: Pointer and transformation** (optional + unit/kind mismatch) + +Two local variables: `phii_l` (transformation temporary) and `phii_p` (pointer array). +Sequence: (1) apply transformation to `phii_l` depending on intent, (2) assign pointer +to `phii_l`, (3) call scheme, (4) nullify pointer, (5) apply back-transformation from +`phii_l` to host depending on intent. All within the `if (active)` block: + +```fortran +! declarations: +real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) +type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) + +! before call: +if () then + ! step 1: apply forward transformation (intent in/inout) + phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa + ! step 2: assign pointer to transformed local + phii_p(thread_number)%ptr => phii_l +else + nullify(phii_p(thread_number)%ptr) +end if + +call scheme_run(..., phii_p(thread_number)%ptr, ...) + +! after call: +if () then + ! step 4: nullify pointer + nullify(phii_p(thread_number)%ptr) + ! step 5: apply back-transformation (intent inout/out) + gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa +end if +``` + +The wrapper types (`real_kind_phys_rank1_ptr_type` etc.) are defined once in the +generated shared types module (`ccpp__types.F90`), not re-declared inside every +group cap function. Passing an unassociated pointer is safe under all compiler modes. + +--- + +## 11. Subcycle Loops + +When a group in the suite XML contains ``, the generator emits a +Fortran `do` loop in the group cap: + +```fortran +do = 1, N ! subcycle: N iterations from suite XML + ! ... scheme calls ... +end do +``` + +`ccpp_loop_counter` and `ccpp_loop_extent` are **loop-context variables** — a special +class that does not fit any of the five table types: +- NOT `type = control`: not passed as `ccpp_physics_*` arguments from the host +- NOT `type = host`: not from host module USE +- NOT `type = suite`: not persistent allocated data + +The generator has built-in knowledge of these two standard names. `ccpp_loop_extent` +value comes from the `loop=N` attribute in the suite XML definition file. +`ccpp_loop_counter` is the do loop induction variable. Both exist only within the +generated loop scope — they are not in scope outside a subcycle block. + +Any scheme that requests `ccpp_loop_counter` or `ccpp_loop_extent` by standard name +receives the loop variables at the call site. Their local names in the cap are derived +from the scheme's metadata section headers as for any other variable (Section 10.2). + +--- + +## 12. Init/Finalize Deduplication + +If the same scheme appears more than once within a single group (e.g., via subcycles), +having its `_init` called multiple times is a **code generator bug** — the generator +**errors out** rather than silently deduplicating. The suite XML must not list the same +scheme multiple times in the same group for non-run phases. + +If the same scheme appears in multiple groups, its `_init` is called once per group. +This is acceptable — idempotency is a contract all scheme `_init` routines must satisfy. + +The same rule applies to `_timestep_init`, `_timestep_final`, and `_final`. + +--- + +## 13. Suite-Owned Data + +### 13.1 Discovery + +The generator identifies suite-owned variables during variable resolution: variables +requested by schemes that are not satisfied by host metadata (`type = host` or +`type = control`). + +**Error condition**: if a variable is determined to be suite-owned (not provided by the +host) and the first scheme that uses it does not have it as `intent(out)`, the generator +errors out. A suite-owned variable that is first read before it is written would be +used uninitialized. + +### 13.2 Generated files and allocation + +Suite-owned variables are declared in a generated suite data module +(`ccpp__data.F90`) as fields of a Fortran DDT. The suite cap and all group caps +`use` this module. + +Allocation happens in `ccpp_init` (suite cap). Deallocation happens in `ccpp_final`. +Suite-owned arrays are allocated for the **full `horizontal_dimension`** — threads +access their respective horizontal chunk (`horizontal_loop_begin:horizontal_loop_end`) +at scheme call sites. This avoids per-call allocation overhead. If this proves to +consume too much memory at scale, a future revision may move allocation to per-phase +with chunk-sized arrays; start with the full-dimension approach. + +Subsetting (applying horizontal loop bounds, instance index) happens at scheme call +sites in the group cap, not in the suite cap. + +### 13.3 Metadata + +The generator also writes a `type = suite` metadata table (`ccpp_.meta`) as a +byproduct. This file is for inspection and debugging — it is not consumed by the +generator on subsequent runs. + +--- + +## 14. Constituent API + +> **Status (2026-05-12).** The constituent API in capgen-ng has evolved past +> the sketch below. The current implementation is: +> +> - One `ccpp_model_constituents_obj(:)` array (sized to +> `number_of_instances`), declared and owned by the **generator** in +> `ccpp_host_constituents.F90` — not by the host. +> - Host-facing API: `ccpp_register_constituents(host_constituents, +> instance_number, ...)`, `ccpp_initialize_constituents`, +> `ccpp_number_constituents`, `ccpp_const_get_index`, +> `ccpp_constituents_array(instance_number)`, etc. All per-instance. +> - Schemes follow four rules: register-phase +> `ccpp_constituent_properties_t(:), intent=out, allocatable`; +> physics-phase consume via `advected=true intent=in/inout`; tendency +> produce via `constituent=true intent=out` + `tendency_of_` +> std_name; mismatched combos are codegen errors. +> - **Authoritative reference**: `doc/constituents.md` (full lifecycle + +> API + examples). +> - **Architecture review and proposed reforms**: `doc/constituents_overhaul.md` +> (2026-05-12, meeting-quality discussion of original capgen vs +> capgen-ng vs cam-sima needs, bugs/flaws, class-A/B property +> classification, three proposals A/B/C). +> +> The historic text below is retained for context but does not describe +> the live system. + +### 14.1 Type definition (historic) + +`ccpp_model_constituents_t` is unchanged from CAM-SIMA. The type definition lives in +the framework library (`ccpp_constituent_prop_mod`), not in generated code. + +### 14.2 Ownership and lifecycle (historic — superseded) + +The **host model** declares and owns the constituent object: + +```fortran +use ccpp_constituent_prop_mod, only: ccpp_model_constituents_t +type(ccpp_model_constituents_t) :: constituents ! unallocated initially +``` + +The host passes it to `ccpp_register`, which allocates and populates it. After +`ccpp_register` returns, the host holds a fully allocated object ready for `lock_table`, +`const_index`, `copy_in`, `copy_out`. + +The `constituents` argument to `ccpp_register` is mandatory. + +### 14.3 Register phase mechanics (historic — superseded) + +The suite cap's register routine: +1. Iterates over all constituent-providing schemes in the suite +2. Calls each scheme's `_register` entrypoint, which returns a + `ccpp_constituent_properties_t` array +3. Collects these arrays and populates the constituent object + +No `group_name` dispatch is needed for register — it operates on the whole suite. + +--- + +## 15. Generated Output Files + +All files are written to `--output-root`. + +| File | Contents | +|---|---| +| `ccpp_kinds.F90` | Kind parameter definitions. **Always generated.** Re-exports specs from `iso_fortran_env` (default) or host-supplied modules as `integer, parameter, public :: = `. If no `--kind-type` is supplied, `kind_phys=iso_fortran_env:REAL64` is injected automatically (logged at INFO). | +| `ccpp_static_api.F90` | Static API — host imports, suite_name dispatch | +| `ccpp__cap.F90` | Suite cap — suite data import, state machine, group dispatch | +| `ccpp___cap.F90` | Group cap — scheme call sites, state array, optionals, transformations. USEs `ccpp_kinds` for any kind referenced in transformation temporaries. | +| `ccpp__data.F90` | Suite data module — framework-owned interstitial DDT. USEs `ccpp_kinds` for any kind referenced in suite-var declarations. | +| `ccpp__types.F90` | Shared cap types — optional pointer wrapper types, transformation locals. USEs `ccpp_kinds` for any kind referenced in pointer wrappers. | +| `ccpp_.meta` | Generated `type = suite` metadata table (output-only, for inspection) | +| `datatable.xml` | Generator database for `ccpp_datafile.py` queries. `ccpp_kinds.F90` and `ccpp_static_api.F90` appear in `...` (matches original capgen). | + +`ccpp_kinds.F90` is a dependency of all generated Fortran files that reference any kind parameter (group cap, suite types, suite data). The static API and suite cap have no kind references and do not USE it. + +--- + +## 16. CLI Invocation + +``` +ccpp_capgen_ng.py + --host-name + --host-files + --scheme-files + --suites + --output-root + --kind-type NAME=[MODULE:]SPEC # repeatable, see § 16.1 + --verbose # once = info; twice = debug +``` + +The generator also supports programmatic Python invocation (import and call directly), +using the same internal code paths as the CLI. + +### 16.1 Kind specifications + +Each `--kind-type` maps a CCPP-visible kind name to a Fortran precision constant. Syntax: + +``` +--kind-type =[:] +``` + +* `` — kind name as published in `ccpp_kinds` and referenced in scheme metadata + (e.g. `kind_phys`). +* `` — name of a precision constant (kind parameter) defined in some Fortran + module. +* `` — Fortran module that defines ``. **Optional**: when omitted, + `` must be a standard `ISO_FORTRAN_ENV` constant (`REAL32`, `REAL64`, `INT32`, + ...) and the module defaults to `iso_fortran_env`. If `` is not a known ISO + constant, omitting `` is an error. + +Examples: + +* `--kind-type kind_phys=REAL64` → + `use iso_fortran_env, only: REAL64; integer, parameter, public :: kind_phys = REAL64` +* `--kind-type kind_phys=my_host_kinds:kind_r8` → + `use my_host_kinds, only: kind_r8; integer, parameter, public :: kind_phys = kind_r8` + +The flag may be specified multiple times. `ccpp_kinds.F90` is **always generated**. If +no `--kind-type` is supplied (or `kind_phys` is omitted from a non-empty list), the +generator injects `kind_phys=iso_fortran_env:REAL64` and logs an INFO message. + +### 16.2 datatable.xml and ccpp_datafile.py + +The generator emits `datatable.xml` encoding the full relationships between suites, +groups, schemes, and variables. A separate query utility (`ccpp_datafile.py`) provides +a rich query interface used by CMake and other build systems. The full query surface of +the existing `ccpp_datafile.py` is preserved — the simplified `--ccpp-files` query is +one of many. + +The XML structure is: + +```xml + + + + /abs/path/ccpp_kinds.F90 + /abs/path/ccpp_static_api.F90 + + + /abs/path/ccpp__cap.F90 + ... + + + + + + + + ... + + + + + ... + + + + + + + ... + + + + + + /abs/path/to/dep.F90 + ... + + +``` + +The `` section is populated from the `dependencies` table-level property +of all scheme metadata files (Section 3.5). Paths are resolved to absolute paths at +generation time, then sorted and deduplicated before writing. + +### 16.3 CMake integration pattern + +The generator runs at CMake configure time via `execute_process`. Generated sources are +discovered by querying `ccpp_datafile.py --ccpp-files`. Host and scheme Fortran sources +are found by replacing `.meta` with `.F90` (same base name convention). + +--- + +## 17. Design Decisions Not Carried Forward + +The following patterns from prebuild or capgen are explicitly **not** carried forward: + +| Pattern | Reason | +|---|---| +| `ccpp_t` / `cdata` struct | Replaced by explicit named control variable arguments | +| `TYPEDEFS_NEW_METADATA` Python dict | Replaced by DDT instance declarations in metadata | +| String-based `ccpp_suite_state` | Replaced by integer state arrays with named parameters | +| Boolean `initialized(:)` array | Replaced by integer state arrays | +| Flat-field arguments to group caps | DDT arguments are used instead (as in prebuild) | +| Scope-chain variable promotion | Suite-owned variables explicitly discovered and declared | +| Fortran-vs-metadata validation in generator | Moved to standalone validator tool | +| Re-declaration of pointer wrapper types per function | Declared once in shared types module | +| `ccpp_physics_suite_init/finalize` | Replaced by `ccpp_init`/`ccpp_final` | +| `type = module` (capgen) | Renamed to `type = host` | +| `finalize` phase name | Renamed to `final` | +| Array size checks in caps | Not generated by default; rely on compiler bounds checking | +| Auto-clone of `is_constituent` scheme args into framework `%instantiate` calls | Replaced by explicit registration (host_constituents arg + register-phase `ccpp_constituent_properties_t`) | +| `ConstituentVarDict` synthetic scope between suite and host | Removed; constituents are a `source='constituent'` classification on `ResolvedArg` | + +--- + +## 18. Outstanding Work + +See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` +(deferred items) for the canonical list. Snapshot as of 2026-05-12 +(end of session): + +### Landed this session + +- **`instance_number` / `number_of_instances` paired opt-in** — hosts + may omit both for a single-instance API. +- **Module-name override** — `module_name = ` on + `[ccpp-table-properties]` for scheme/host/ddt. +- **Vertical-flip transform** — `top_at_one` per-var attribute; + composes with unit/kind transforms. +- **Multiple `dependencies = …` lines** per `[ccpp-table-properties]`. +- **Sliced local names** with long subscript-token CCPP standard names + no longer trip the 63-char Fortran-id limit. +- **Unit normalisation** — `m2` ≡ `m+2` (and friends). +- **Subcycle bound = CCPP std name** — including DDT-component access + paths (`phys_state%num_subcycles`). +- **Nested ``** — preserved end-to-end as nested `do` loops. +- **Active-expression + subcycle bounds** included in introspection + inputs. +- **TARGET on `ccpp_suite_data(:)`** module-level array. +- **Group-state alloc idempotency** (matches suite-state alloc). +- **Framework PR**: `ccpt_deallocate` ownership tracking via + `framework_owns_me` flag. Backward-compatible. Needs upstream + merge to ccpp-framework + ccpp-capgen. +- **Identity unit conversions** no longer emit misleading "unit + conversion: kind_phys to kind_phys" comment. +- **Improved duplicate-standard-name error** lists both colliding + access paths. +- **Suite-level `` / ``** SDF elements consumed: named + scheme's init/final phase emitted inside `_init` / + `_final`. Single scheme only; long-form spellings + (``, ``, ``) rejected. +- **Test count**: 1070 passing. + +### Still deferred + +- **End-to-end integration tests** — user-driven; off-limits for in-session + edits. +- **Constituents overhaul** — discussion doc at + `doc/constituents_overhaul.md` (2026-05-12). Three proposals on the + table (A bugfix-only / B class-A/B split + setters / C host-only + registration). Pending decision in upcoming meeting. +- **Framework setter additions** — `set_advected`, `set_diagnostic_name`, + `set_default_value`, possibly `set_mixing_ratio_type`. Coordinated with + the overhaul. +- **Validator host-metadata check** — `ccpp_validator.py` is currently + scheme-only; revisit after the e2e test suite settles. See + `project_validator_host_check_deferred.md` (memory). +- **Codegen-time scheme-registration cross-check** — new metadata attr + `registers_std_names = a, b, c` on register-phase tables; replaces + current runtime `int_unassigned` check with codegen-time error. +- **Capgen-ng cleanup**: replace `_FRAMEWORK_CONST_DIM_INPUTS` frozenset + in `generator/static_api.py` with `used_const_dim_std_names: Set[str]` + on `ResolvedArg`. +- **Nested subcycle `ccpp_loop_counter` semantics**: currently a scheme + inside a nested subcycle requesting `ccpp_loop_counter` would get + the OUTERMOST counter, not the innermost. None of the cam-sima + schemes use this — revisit if a real scheme needs the innermost. + +### Where to find the migration summary + +`doc/migration.md` (created 2026-05-12) — user-facing single-page +summary of metadata + SDF + host-Fortran requirements after all the +above changes. Read it first when porting a host model. From f72cb2993a87aff24eb89571c47f07507f3f1dd5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 08:52:26 -0600 Subject: [PATCH 05/74] Update GitHub workflows --- .github/workflows/end-to-end-tests.yaml | 8 ++++++-- .github/workflows/unit-tests.yaml | 20 ++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index ea6bee2a..3af5eff3 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -17,8 +17,12 @@ jobs: fortran-compiler: [gfortran-9, gfortran-10, gfortran-11, gfortran-12] runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 - - name: update repos and install dependencies + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: 3.12 + - name: Update repos and install dependencies run: | sudo apt-get update sudo apt-get install -y \ diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index cd97315e..e827ed37 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -19,15 +19,31 @@ jobs: python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + #- name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # pip install pytest + # which pytest + - name: Update repos and install dependencies run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + libopenmpi-dev \ + ${{matrix.fortran-compiler}} \ + cmake \ + python3 \ + git \ + libxml2-utils python -m pip install --upgrade pip pip install pytest + which xmllint + xmllint --version which pytest - name: Run capgen-ng unit tests run: | From eaaea49a211db750d6f30f2f643cd9b2dd0e37f5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 09:13:02 -0600 Subject: [PATCH 06/74] .github/workflows/unit-tests.yaml: drop Python 3.8, EOL Oct 2024 --- .github/workflows/unit-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index e827ed37..eabc01ad 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4 From 9912ba4ee68e8f92105f2487e3f402e1fb29d302 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 09:13:32 -0600 Subject: [PATCH 07/74] Minor bug fixes in capgen-ng code and unit tests from ccpp-scm integration --- capgen-ng/generator/suite_resolver.py | 24 +- capgen-ng/metadata/metadata_table.py | 6 +- unit-tests/test_metadata_table.py | 32 +++ unit-tests/test_suite_resolver.py | 386 ++++++++++++++++++++++++++ 4 files changed, 444 insertions(+), 4 deletions(-) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 185eb854..dcad5a35 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -328,7 +328,14 @@ def _resolve_single_bound( entry = host_dict.get(bound) if entry is not None: used.add(bound) - return entry.local_name + # Use ``access_path``, not ``local_name``: for plain module- + # level host vars these are identical, but for DDT-component + # vars (e.g. ``physics%Model%levs`` with std_name + # ``vertical_layer_dimension``) the full DDT walk is required + # so the emitted subscript references the actual storage and + # the USE statement (which walks back to the root via + # ``_root_symbol``) imports the right top-level symbol. + return entry.access_path if suite_vars: sv = suite_vars.get(bound) if sv is not None: @@ -558,7 +565,12 @@ def _build_merged_subscript( key = token.lower() entry = host_dict.get(key) if entry is not None: - parts.append(entry.local_name) + # Use ``access_path`` so DDT-component subscript indices + # (e.g. ``q(:,:,index_of_)`` where index_of_X lives + # on a DDT) resolve to the full DDT walk, not the bare + # leaf name. Identical to ``local_name`` for plain + # module-level host vars. + parts.append(entry.access_path) used.add(key) elif suite_vars and key in suite_vars: parts.append(suite_vars[key].access_path) @@ -2010,7 +2022,13 @@ def _collect_dim_uses( entry = host_dict.get(dim_std) if entry is not None and entry.module_name is not None: mod = entry.module_name - sym = entry.local_name + # Walk back to the access-path root so DDT- + # component dims (e.g. ``physics%Model%levs``) + # USE the top-level instance (``physics``) and + # not the leaf (``levs``, which doesn't exist + # as a module symbol). Equivalent to + # ``entry.local_name`` for plain host vars. + sym = _root_symbol(entry.access_path) dim_uses.setdefault(mod, set()).add(sym) elif suite_vars and dim_std in suite_vars: sv = suite_vars[dim_std] diff --git a/capgen-ng/metadata/metadata_table.py b/capgen-ng/metadata/metadata_table.py index 6cc621fd..631e636a 100644 --- a/capgen-ng/metadata/metadata_table.py +++ b/capgen-ng/metadata/metadata_table.py @@ -587,7 +587,11 @@ def set_attr(self, key: str, value: str, context: ParseContext) -> None: elif key == 'optional': self.optional = _parse_bool(value, context) elif key == 'active': - self.active = value.strip() + # Standard names elsewhere are canonicalised to lowercase by + # check_cf_standard_name; an active expression references those + # same names, so normalise here too. Fortran is case-insensitive, + # so embedded logical operators/literals are unaffected. + self.active = value.strip().lower() elif key == 'protected': self.protected = _parse_bool(value, context) elif key == 'allocatable': diff --git a/unit-tests/test_metadata_table.py b/unit-tests/test_metadata_table.py index b4aa8000..622e46c7 100644 --- a/unit-tests/test_metadata_table.py +++ b/unit-tests/test_metadata_table.py @@ -1093,6 +1093,38 @@ def test_active_in_host_allowed(self): tables = _parse_text(text) self.assertEqual(len(tables), 1) + def test_active_expression_is_lowercased(self): + """Mixed-case identifiers in 'active' must be normalised to lowercase + so they match the canonical lowercase standard names stored on host + variables (see check_cf_standard_name). + """ + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ flag ] + standard_name = flag_for_aerosol_input_mg_radiation + units = flag + dimensions = () + type = logical + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + active = (flag_for_aerosol_input_MG_radiation) + """ + tables = _parse_text(text) + x_var = next( + v for v in tables[0].sections()[0].variables + if v.local_name == 'x' + ) + self.assertEqual(x_var.active, '(flag_for_aerosol_input_mg_radiation)') + def test_optional_in_scheme_allowed(self): """'optional' attribute in scheme metadata is valid.""" text = """ diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 8490ce0e..d8753d97 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -2864,6 +2864,392 @@ def test_unclaimed_index_of_still_routes_to_constituents(self): 'index_of_some_other_constituent_not_in_host') +class TestDimDDTComponentResolution(unittest.TestCase): + """When a dimension standard name maps to a DDT-component host + entry (e.g. ``vertical_layer_dimension`` is ``physics%Model%levs``, + a two-level DDT walk inside the SCM/UFS host), the resolver must: + + 1. Emit the full ``access_path`` in subscript expressions + (``1:physics%Model%levs``), NOT the bare leaf (``1:levs``). + 2. Walk back to the access-path *root* for the USE statement + (``use scm_host_mod, only: physics``), NOT the leaf + (``use scm_host_mod, only: levs`` — undefined symbol). + 3. Behave identically to today for plain module-level host vars + (``access_path == local_name``). + + Historical: this was a long-standing pain point in the original + capgen → SCM migration. The pre-2026-05-13 capgen-ng emitted + ``use scm_type_defs, only: levs`` and similar bogus imports for + every DDT-component dim, producing many ``Symbol referenced ... + not found in module`` errors at compile time. + """ + + # Mimics SCM: ``physics`` is host-level (module ``scm_host_mod``), + # of type ``physics_t`` (a DDT) which has component ``Model`` of + # type ``gfs_control_t`` (a DDT) which has scalar ``levs`` and + # ``ncols`` declared on it. + _DDT_SRC = ( + '[ccpp-table-properties]\n' + ' name = gfs_control_t\n' + ' type = ddt\n' + '[ccpp-arg-table]\n' + ' name = gfs_control_t\n' + ' type = ddt\n' + '[levs]\n' + ' standard_name = vertical_layer_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ncol]\n' + ' standard_name = horizontal_dimension_total\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '\n' + '[ccpp-table-properties]\n' + ' name = physics_t\n' + ' type = ddt\n' + '[ccpp-arg-table]\n' + ' name = physics_t\n' + ' type = ddt\n' + '[Model]\n' + ' standard_name = gfs_control_instance\n' + ' units = DDT\n' + ' dimensions = ()\n' + ' type = gfs_control_t\n' + ) + + _HOST_SRC = ( + '[ccpp-table-properties]\n' + ' name = scm_host_mod\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = scm_host_mod\n' + ' type = host\n' + # Plain (non-DDT) horizontal dim — for the mixed-dim test. + '[ncols]\n' + ' standard_name = horizontal_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + # The DDT instance. Resolver should produce + # access_path == 'physics' for the instance, and + # 'physics%Model%levs' for the leaf dim. + '[physics]\n' + ' standard_name = physics_state_instance\n' + ' units = DDT\n' + ' dimensions = ()\n' + ' type = physics_t\n' + ) + + def _host_dict(self): + return build_flat_host_dict( + _parse(self._HOST_SRC), [], _parse(self._DDT_SRC), + ) + + # ---- Unit-level: host_dict entry shape (the prerequisite) ----------- + + def test_host_dict_levs_has_ddt_walk_access_path(self): + hd = self._host_dict() + entry = hd.get('vertical_layer_dimension') + self.assertIsNotNone(entry) + self.assertEqual(entry.local_name, 'levs') + self.assertEqual(entry.access_path, 'physics%Model%levs') + self.assertEqual(entry.module_name, 'scm_host_mod') + + # ---- Unit-level: _resolve_single_bound (the primary fix site) ------- + + def test_resolve_single_bound_returns_access_path_for_ddt_dim(self): + from generator.suite_resolver import _resolve_single_bound + hd = self._host_dict() + used = set() + result = _resolve_single_bound('vertical_layer_dimension', hd, used) + self.assertEqual(result, 'physics%Model%levs') + self.assertIn('vertical_layer_dimension', used) + + def test_resolve_single_bound_plain_var_unchanged(self): + # ncols is a plain host var (access_path == local_name == 'ncols'). + from generator.suite_resolver import _resolve_single_bound + hd = self._host_dict() + used = set() + result = _resolve_single_bound('horizontal_dimension', hd, used) + self.assertEqual(result, 'ncols') + + # ---- Unit-level: _one_dim_part wrapping ----------------------------- + + def test_one_dim_part_bare_form_uses_ddt_walk(self): + from generator.suite_resolver import _one_dim_part + hd = self._host_dict() + # Bare std name normalises to ``ccpp_constant_one:``. + part, used = _one_dim_part( + 'vertical_layer_dimension', 'run', hd, + ) + self.assertEqual(part, '1:physics%Model%levs') + self.assertIn('vertical_layer_dimension', used) + + def test_one_dim_part_range_form_uses_ddt_walk(self): + from generator.suite_resolver import _one_dim_part + hd = self._host_dict() + part, used = _one_dim_part( + 'ccpp_constant_one:vertical_layer_dimension', 'run', hd, + ) + self.assertEqual(part, '1:physics%Model%levs') + + def test_one_dim_part_ddt_dim_as_lower_bound(self): + # Stress: DDT-component appears as the LOWER bound of a range. + from generator.suite_resolver import _one_dim_part + hd = self._host_dict() + part, used = _one_dim_part( + 'vertical_layer_dimension:horizontal_dimension_total', 'run', hd, + ) + # Both bounds walked: physics%Model%levs : physics%Model%ncol + self.assertEqual(part, 'physics%Model%levs:physics%Model%ncol') + + # ---- Unit-level: _build_call_subscript composition ----------------- + + def test_build_call_subscript_two_ddt_dims(self): + # Two DDT-component dims side by side: both must walk. + from generator.suite_resolver import _build_call_subscript + hd = self._host_dict() + sub, used = _build_call_subscript( + ['horizontal_dimension_total', 'vertical_layer_dimension'], + 'run', hd, + ) + self.assertEqual( + sub, '(1:physics%Model%ncol, 1:physics%Model%levs)', + ) + + def test_build_call_subscript_pure_ddt_dims(self): + from generator.suite_resolver import _build_call_subscript + hd = self._host_dict() + sub, used = _build_call_subscript( + ['vertical_layer_dimension', 'vertical_layer_dimension'], + 'run', hd, + ) + self.assertEqual(sub, '(1:physics%Model%levs, 1:physics%Model%levs)') + + # ---- Sliced-subscript path: _build_merged_subscript --------------- + + def test_build_merged_subscript_ddt_index_token(self): + # Mirrors host metadata like ``q(:,:,vertical_layer_dimension)`` + # but here we exercise the helper directly with a synthetic + # local_subscript carrying a std-name token whose target lives + # on a DDT. The third token must be the full DDT walk, not + # the bare leaf — bug pre-2026-05-13 produced ``levs`` instead + # of ``physics%Model%levs`` and the generated cap then failed + # to compile against the host module. + from generator.suite_resolver import _build_merged_subscript + hd = self._host_dict() + # Use ``horizontal_dimension_total`` for the leading dims so the + # fixture doesn't need loop bounds. + merged, used = _build_merged_subscript( + host_dims=['horizontal_dimension_total', + 'horizontal_dimension_total'], + local_subscript=[':', ':', 'vertical_layer_dimension'], + phase='run', host_dict=hd, suite_vars={}, + ) + # Third subscript token = full DDT walk. + self.assertTrue( + merged.rstrip(')').endswith('physics%Model%levs'), + 'merged subscript should end with the DDT walk; got {!r}' + .format(merged), + ) + # Leaf should not leak. + self.assertNotIn(', levs)', merged) + + # ---- Group-cap USE collection: _collect_dim_uses -------------------- + + @staticmethod + def _mock_arg(used_dim_std_names): + from unittest.mock import MagicMock + arg = MagicMock() + arg.used_dim_std_names = set(used_dim_std_names) + return arg + + def _mock_rg(self, arg): + from unittest.mock import MagicMock + from generator.suite_resolver import ResolvedCall + rg = MagicMock() + rg.phase_calls = { + 'run': [ResolvedCall( + scheme_name='dummy', phase='run', + args=[arg], scheme_module='dummy_mod', + )], + } + return rg + + def test_collect_dim_uses_walks_to_root_for_ddt_dim(self): + from generator.suite_resolver import _collect_dim_uses + hd = self._host_dict() + arg = self._mock_arg({'vertical_layer_dimension'}) + rg = self._mock_rg(arg) + dim_uses = _collect_dim_uses(rg, hd, suite_vars={}) + # USE clause must pull the ROOT (``physics``), not the leaf + # (``levs``, which is not a module symbol). + self.assertIn('scm_host_mod', dim_uses) + self.assertIn('physics', dim_uses['scm_host_mod']) + self.assertNotIn('levs', dim_uses['scm_host_mod']) + self.assertNotIn('Model', dim_uses['scm_host_mod']) + + def test_collect_dim_uses_plain_var_unchanged(self): + # Regression: behaviour identical for plain host vars. + from generator.suite_resolver import _collect_dim_uses + hd = self._host_dict() + arg = self._mock_arg({'horizontal_dimension'}) + rg = self._mock_rg(arg) + dim_uses = _collect_dim_uses(rg, hd, suite_vars={}) + self.assertEqual(dim_uses.get('scm_host_mod'), {'ncols'}) + + def test_collect_dim_uses_two_ddt_dims_dedupe_root(self): + # Both ``vertical_layer_dimension`` and + # ``horizontal_dimension_total`` live on the same ``physics`` + # instance — USE clause emits ``physics`` ONCE, not twice. + from generator.suite_resolver import _collect_dim_uses + hd = self._host_dict() + arg = self._mock_arg({'vertical_layer_dimension', + 'horizontal_dimension_total'}) + rg = self._mock_rg(arg) + dim_uses = _collect_dim_uses(rg, hd, suite_vars={}) + self.assertEqual(dim_uses.get('scm_host_mod'), {'physics'}) + + # ---- End-to-end: resolve_suite + group cap output ------------------ + + def test_end_to_end_ddt_dim_in_group_cap(self): + """Scheme arg with a DDT-component dim should compile. + + Builds the full pipeline: parse scheme + host metadata, resolve + the suite, and generate the group cap text. Asserts: + + - The scheme call subscript contains ``1:physics%Model%levs`` + (NOT ``1:levs``). + - The group cap's ``use scm_host_mod, only: ...`` clause + contains ``physics`` (the DDT root) and NOT ``levs`` (the + leaf, which would be an undefined symbol). + """ + # Add control vars so the static API can build. + control_src = ( + '[ccpp-table-properties]\n' + ' name = control\n' + ' type = control\n' + '[ccpp-arg-table]\n' + ' name = control\n' + ' type = control\n' + '[errmsg]\n' + ' standard_name = ccpp_error_message\n' + ' units = none\n' + ' dimensions = ()\n' + ' type = character | kind = len=512\n' + '[errflg]\n' + ' standard_name = ccpp_error_code\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ilb]\n' + ' standard_name = horizontal_loop_begin\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[iub]\n' + ' standard_name = horizontal_loop_end\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + ) + scheme_src = ( + '[ccpp-table-properties]\n' + ' name = ddt_dim_user\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = ddt_dim_user_run\n' + ' type = scheme\n' + '[temp]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = inout\n' + '[errmsg]\n' + ' standard_name = ccpp_error_message\n' + ' units = none\n' + ' dimensions = ()\n' + ' type = character | kind = len=*\n' + ' intent = out\n' + '[errflg]\n' + ' standard_name = ccpp_error_code\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = out\n' + ) + # Extend the host with horizontal_dimension + air_temperature + # so the scheme's dims resolve. + host_src = self._HOST_SRC + ( + '[gt0]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ) + hd = build_flat_host_dict( + _parse(host_src), _parse(control_src), _parse(self._DDT_SRC), + ) + store = SchemeStore.build_from(_parse(scheme_src)) + + # Build a tiny suite XML and resolve it. + suite_xml = ( + '\n' + '\n' + ' ddt_dim_user\n' + '\n' + ) + with tempfile.TemporaryDirectory() as tmpdir: + xml_path = os.path.join(tmpdir, 's.xml') + with open(xml_path, 'w') as fh: + fh.write(suite_xml) + from generator.suite_xml import parse_suite_xml + import logging + logger = logging.getLogger('ddt_dim_e2e') + suite = parse_suite_xml(xml_path, tmpdir, logger, + skip_validation=True) + sr = resolve_suite(suite, store, hd) + + # Inspect the resolved scheme arg's subscript. + run_call = list(iter_phase_calls(sr.groups[0].phase_calls['run']))[0] + temp_arg = [a for a in run_call.args + if a.scheme_local_name == 'temp'][0] + self.assertIn('physics%Model%levs', temp_arg.subscript) + # Leaf name must not appear bare anywhere in the call expr. + self.assertNotIn(', 1:levs)', temp_arg.call_expr) + + # Now emit the group cap and inspect the USE clause. + from generator.group_cap import _generate_group_cap + group_lines = _generate_group_cap( + suite_name=sr.suite_name, + group_name=sr.groups[0].group_name, + rg=sr.groups[0], host_dict=hd, + ) + group_text = '\n'.join(group_lines) + # Pull the host-module USE line. Must import ``physics``, + # must NOT import the bare leaf ``levs``. + import re as _re + host_use_lines = [ + ln for ln in group_text.splitlines() + if _re.search(r'use\s+scm_host_mod\b', ln) + ] + self.assertTrue(host_use_lines, + "group cap missing ``use scm_host_mod`` line") + joined = ' '.join(host_use_lines) + self.assertIn('physics', joined) + # Word-boundary check so we don't false-match a substring like + # ``physics`` containing ``levs`` etc. + self.assertIsNone(_re.search(r'\blevs\b', joined), + "group cap leaked DDT-leaf ``levs`` into USE: " + + joined) + self.assertIsNone(_re.search(r'\bModel\b', joined), + "group cap leaked DDT-mid-component ``Model`` " + "into USE: " + joined) + + ######################################################################## # Doctest loader ######################################################################## From 837bcd1b6f763b2e96a13aa26e47b0af1ef0f177 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 09:26:42 -0600 Subject: [PATCH 08/74] Update documentation --- doc/constituents_20260513T0733.md | 1029 +++++++++++++++++ doc/constituents_overhaul_20260513T0733.md | 829 +++++++++++++ doc/migration_20260513T0733.md | 545 +++++++++ ....md => redesign_analysis_20260513T0733.md} | 0 ...esign_analysis_original_202060505T2044.md} | 0 ...33.md => redesign_prompt_20260513T0733.md} | 0 ...redesign_prompt_original_20260505T2044.md} | 0 7 files changed, 2403 insertions(+) create mode 100644 doc/constituents_20260513T0733.md create mode 100644 doc/constituents_overhaul_20260513T0733.md create mode 100644 doc/migration_20260513T0733.md rename doc/{redesign_analysis.md - updated 20260513T0733.md => redesign_analysis_20260513T0733.md} (100%) rename doc/{redesign_analysis - original 202060505T2044.md => redesign_analysis_original_202060505T2044.md} (100%) rename doc/{redesign_prompt.md - updated 20260513T0733.md => redesign_prompt_20260513T0733.md} (100%) rename doc/{redesign_prompt - original 20260505T2044.md => redesign_prompt_original_20260505T2044.md} (100%) diff --git a/doc/constituents_20260513T0733.md b/doc/constituents_20260513T0733.md new file mode 100644 index 00000000..8e5a7b77 --- /dev/null +++ b/doc/constituents_20260513T0733.md @@ -0,0 +1,1029 @@ +# CCPP capgen-ng — Constituents Reference + +*Last revised: 2026-05-11.* + +This document is the authoritative reference for **constituent variables** in +capgen-ng — what they are, how scheme authors declare them in metadata, what +the host model has to do to plumb them through, what the generator emits, and +how the per-instance lifecycle works. + +> If you are migrating a host or scheme from the original capgen, jump to +> [§9 Differences from original capgen](#9-differences-from-original-capgen) +> first. + +--- + +## Table of Contents + +1. [What is a constituent?](#1-what-is-a-constituent) +2. [The four rules (scheme-author conventions)](#2-the-four-rules-scheme-author-conventions) +3. [Required host metadata + Fortran](#3-required-host-metadata--fortran) +4. [Host-side lifecycle (call sequence)](#4-host-side-lifecycle-call-sequence) +5. [Public API reference](#5-public-api-reference) +6. [Generated code structure](#6-generated-code-structure) +7. [Multi-instance design](#7-multi-instance-design) +8. [Limitations and gotchas](#8-limitations-and-gotchas) +9. [Differences from original capgen](#9-differences-from-original-capgen) +10. [Worked example](#10-worked-example) + +--- + +## 1. What is a constituent? + +A **constituent** is a model variable owned by the host's dynamical core (or +its constituent infrastructure) that is read and updated by physics schemes — +typically a tracer / mass mixing ratio (water vapor, cloud liquid, ozone, +chemistry species) — together with its **tendency**, the rate of change that +physics writes back so the dycore can advect/integrate it forward. + +In capgen-ng, the constituent layer has three concerns: + +1. **Registration** — declaring at model startup which constituents exist + (their standard name, units, vertical layout, advection flag, …). +2. **Storage** — the framework owns one `ccpp_model_constituents_t` DDT per + host instance (see [§7](#7-multi-instance-design)) which holds the + constituent values (`%vars_layer`), tendencies (`%vars_layer_tend`), and + metadata (`%const_metadata`). +3. **Access** — schemes reference constituents by standard name in their + metadata; the resolver translates those references to + `ccpp_model_constituents_obj(inst)%vars_layer(slice, index_of_)` + subscripts at code-gen time. + +All constituent state lives in **one generated module**: +`ccpp_host_constituents.F90` (one per generator run, emitted only when at +least one suite touches constituent state). Public symbols from this module +are also re-exported by `ccpp_static_api`, so most host code only needs + +```fortran +use ccpp_static_api, only: ccpp_register_constituents, ccpp_initialize_constituents, & + ccpp_constituents_array, ccpp_const_get_index, ... +``` + +--- + +## 2. The four rules (scheme-author conventions) + +These four rules govern every scheme-arg metadata pattern related to +constituents. They derive from a 2026-05-11 audit of all 12 cam-sima scheme +metadata files that touch constituent attributes. + +### Rule 1 — Register a new constituent (register phase) + +A scheme that creates a new constituent declares it in the **register** +phase via an `intent=out, allocatable` array of +`ccpp_constituent_properties_t`: + +``` +[ccpp-arg-table] + name = my_scheme_register + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_my_scheme + long_name = per-scheme constituent array + units = none + dimensions = (:) + type = ccpp_constituent_properties_t + allocatable = True + intent = out +[ errmsg ] + ... +[ errflg ] + ... +``` + +The scheme's Fortran register routine `allocate`s this array, populates +each entry via `%instantiate(std_name=..., long_name=..., units=..., +vertical_dim=..., advected=..., ...)` and returns it. The framework +captures every register-phase scheme's array, packs them into a per-suite +buffer (`_dynamic_constituents`), and merges them into each +host-instance's constituent object during `ccpp_register_constituents`. + +This is the **only path** for declaring a new constituent. + +### Rule 2 — Consume a base constituent (any physics phase) + +A scheme that reads (or reads + writes) an existing base constituent +declares the variable with `is_constituent` set (any of `advected`, +`constituent`, or `molar_mass` non-default) and `intent=in` or `intent=inout`: + +``` +[ cldliq ] + standard_name = cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in ! or inout + advected = true +``` + +The resolver translates this scheme arg to +`ccpp_model_constituents_obj()%vars_layer(, index_of_)` +in the generated group cap. No host metadata declaration is needed for +the variable. + +### Rule 3 — Produce a tendency (any physics phase) + +A scheme that writes a constituent tendency declares the variable with +`is_constituent` set, `intent=out`, and a standard name that **starts +with `tendency_of_`**: + +``` +[ tend_cldliq ] + standard_name = tendency_of_cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = true +``` + +The resolver translates this scheme arg to +`ccpp_model_constituents_obj()%vars_layer_tend(, index_of_)` +where `` is the std_name with the `tendency_of_` prefix stripped. +The tendency variable is implicitly tied to the base constituent of the +same name. + +### Rule 4 — Mismatched combinations are hard errors + +Two combinations are explicitly rejected by the resolver at code-gen time: + +| Mismatch | Error | +|---|---| +| `is_constituent=True` + `intent=out` + std_name does NOT start with `tendency_of_` | *"Physics phases may only produce constituent tendencies; new base constituents must be declared via a `ccpp_constituent_properties_t` argument in a register-phase scheme."* | +| `is_constituent=True` + `intent in (in, inout)` + std_name starts with `tendency_of_` | *"Constituent tendency arg must be declared with intent=out; physics phases only produce tendencies, never consume them."* | + +### Direct framework-array access + +A scheme may also access the framework's bulk arrays directly by +declaring an arg with one of these standard names: + +| Standard name | Maps to | +|---|---| +| `ccpp_constituents` | `ccpp_model_constituents_obj()%vars_layer` (3D) | +| `ccpp_constituent_tendencies` | `ccpp_model_constituents_obj()%vars_layer_tend` (3D) | +| `ccpp_constituent_properties` | `ccpp_model_constituents_obj()%const_metadata` (1D of `ccpp_constituent_prop_ptr_t`) | +| `number_of_ccpp_constituents` | `ccpp_model_constituents_obj()%num_layer_vars` (scalar integer) | +| `index_of_` | module-level `integer :: index_of_` (no per-instance access — the index is identical for every instance) | + +The trailing dimension `number_of_ccpp_constituents` in a 3D scheme arg +is emitted as `:` (whole-axis slice). + +--- + +## 3. Required host metadata + Fortran + +### Host metadata (`type=host` table) + +The host **must** declare: + +``` +[ ] + standard_name = number_of_instances + units = count + dimensions = () + type = integer +``` + +… **only when the host actually wants multi-instance support**. When +absent, every per-instance allocation falls back to size `1` and the +host effectively runs single-instance. + +The host **does not** need to declare: + +- `ccpp_model_constituents_object` — the constituent object is owned + by the generator (in `ccpp_host_constituents`); the host doesn't + declare it in metadata. +- `ccpp_constituents`, `ccpp_constituent_tendencies`, + `ccpp_constituent_properties`, `number_of_ccpp_constituents`, + `index_of_` — all auto-provided by the generator. + +#### Host metadata wins over auto-provisioning + +If the host **does** declare any of the framework-named standard +names above as a regular host variable, the resolver uses the host's +declaration instead of auto-provisioning. This matters for legacy +hosts (GFS / SCM) that own their own tracer indices: + +```meta +[ ntcw ] + standard_name = index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array + units = index + type = integer + protected = True + dimensions = () +``` + +A scheme arg requesting the same standard name resolves to the host's +short local name (`ntcw`), not a parallel module-level integer in +`ccpp_host_constituents` named after the full standard name (which +would blow the Fortran 63-character identifier limit). Auto-provisioning +only fires for framework-named standard names the host has **not** +claimed. + +### Host control-table requirements + +The host's `type=control` table must declare: + +``` +[ ] + standard_name = instance_number + units = 1 + dimensions = () + type = integer +``` + +… so the framework signature knows the index for per-instance state. +Same caveat as `number_of_instances` — required only when multi-instance +is wanted. + +### Host Fortran code + +The host's Fortran code only needs to: + +1. Maintain its own `integer :: ` for `number_of_instances` + in a module that's USE'd by the generator. (Same module that owns + the metadata.) +2. Build its **host constituents** array (water vapor, ozone, etc. — + the constituents that the host model owns directly, separately from + any scheme-registered ones). Pass this to + `ccpp_register_constituents`. + +The host does **not** need to allocate or own a +`type(ccpp_model_constituents_t)` variable. + +--- + +## 4. Host-side lifecycle (call sequence) + +``` + ┌─ host startup ─┐ + │ + ▼ + ┌──────────────────────────────────────┐ + │ for each instance: │ + │ ccpp_register(suite_name, │ + │ errmsg, errflg, │ + │ instance_number) │ ─── per-instance ───┐ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ allocate host_constituents(:) │ │ + │ host_constituents(1)%instantiate( │ ─── once ─────────┘ + │ std_name='water_vapor_specific_humidity', ...) │ + │ ... │ │ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_register_constituents( │ │ + │ host_constituents, │ │ + │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_initialize_constituents( │ │ + │ ncols, num_layers, │ │ + │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_init(suite_name, │ │ + │ errmsg, errflg, │ │ + │ instance_number) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ (model time-stepping) │ + ┌──────────────────────────────────────┐ │ + │ ccpp_physics_*(...) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ (host shutdown) │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_deallocate_dynamic_constituents( │ + │ instance_number) │ ─── per-instance ──┤ + │ ccpp_final(suite_name, │ │ + │ errmsg, errflg, │ │ + │ instance_number) │ │ + └──────────────────────────────────────┘ │ + │ + ┌────────────────────────────────────────┘ + │ ◀── last-to-leave dealloc fires + │ automatically inside the per-instance + │ calls when the final instance finishes. + ▼ +``` + +### Important ordering rules + +- `ccpp_register_constituents` **must** be called *after* `ccpp_register` + (per instance). The latter populates the per-suite dynamic-constituent + buffers via `_register`; the former merges them into the + per-instance constituent object. +- `ccpp_initialize_constituents` **must** be called *after* + `ccpp_register_constituents` (per instance). It calls `%lock_data` + on the per-instance object — which can only happen once + `%lock_table` has fired (which `ccpp_register_constituents` does). +- Physics phases (`ccpp_init`, `ccpp_physics_run`, etc.) require + the constituent state to be locked + bound (i.e., + `ccpp_initialize_constituents` already called). +- `ccpp_deallocate_dynamic_constituents` is per-instance with + last-to-leave teardown. Once the last instance calls it, the shared + per-suite buffers and the constituent object array are deallocated + automatically. + +### Built-in constituents vs scheme-registered constituents + +`ccpp_register_constituents` takes one explicit argument: an array of +`ccpp_constituent_properties_t` describing the **host's own constituents** +(typically water vapor and any other tracers the dycore carries +intrinsically). The framework then merges those entries with every +suite's per-suite dynamic-constituent buffer (populated during +`ccpp_register` from each register-phase scheme's output). + +Pass an empty (zero-size) array if the host has no built-in constituents +of its own. + +--- + +## 5. Public API reference + +All routines below live in `ccpp_host_constituents` and are also +re-exported from `ccpp_static_api` for convenience. The dummy-argument +name `instance_number` is the **standard name**; the actual emitted +dummy uses the host's local name for it (typically also +`instance_number` or `inst_num`). + +### `ccpp_register_constituents(host_constituents, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `host_constituents` | `type(ccpp_constituent_properties_t), target, intent(in) :: (:)` | Host-owned constituent declarations (water vapor, etc.). May be zero-size. | +| `instance_number` | `integer, intent(in)` | Per-instance index. | +| `errflg` | `integer, intent(out)` | Error flag (0 = success). | +| `errmsg` | `character(len=*), intent(out)` | Error message. | + +**Effect**: +- On the first call across instances, allocates + `ccpp_model_constituents_obj(number_of_instances)`. +- Calls `obj(instance_number)%initialize_table(num_consts)` where + `num_consts = size(host_constituents) + sum(size(_dynamic_constituents))`. +- Iterates `host_constituents` first, then every suite's + `_dynamic_constituents` buffer, calling + `obj(instance_number)%new_field(const_prop, errcode=errflg, errmsg=errmsg)` + for each entry. +- Calls `obj(instance_number)%lock_table(...)`. + +**Preconditions**: every `_register` call (across all suites) for +this instance has already happened (so the per-suite buffers are +populated). + +### `ccpp_initialize_constituents(ncols, num_layers, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `ncols` | `integer, intent(in)` | Horizontal extent for this instance's chunk. | +| `num_layers` | `integer, intent(in)` | Vertical layer count. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +**Effect**: +- Calls `obj(instance_number)%lock_data(ncols, num_layers, ...)` — + allocates `obj(inst)%vars_layer` and `%vars_layer_tend` arrays. +- Registers a singleton pointer with + `ccpp_scheme_utils.ccpp_initialize_constituent_ptr(obj(inst))` so + cam-sima schemes that call `ccpp_constituent_index` see the + constituent table. **First instance wins** — see + [§8 Limitations](#8-limitations-and-gotchas). +- Queries `obj(instance_number)%const_index(index_of_, '', ...)` for + every constituent `` known at code-gen time; populates the + module-level integer `index_of_`. These integers are identical + across instances; the last call to set them wins (benign — the + constituent table is the same per instance). + +**Preconditions**: `ccpp_register_constituents` has been called for this +instance. + +### `ccpp_is_scheme_constituent(var_name, constituent_exists, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `var_name` | `character(len=*), intent(in)` | Standard name to query. | +| `constituent_exists` | `logical, intent(out)` | True iff *var_name* matches one of the constituent std names known to capgen at code-gen time. | +| `errflg` / `errmsg` | `intent(out)` | | + +**No `instance_number`** — the data lookup is against the module-level +`character(len=N), parameter :: ccpp_model_const_stdnames(K)` array +(compile-time constant, identical across instances). + +### `ccpp_number_constituents(num_flds, advected, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `num_flds` | `integer, intent(out)` | Constituent count returned. | +| `advected` | `logical, optional, intent(in)` | If `.true.`, count advected only. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%num_constituents(num_flds, advected=advected, ...)`. + +> Even though every `obj(i)` returns the same count (registration is +> identical across instances), `instance_number` is part of the +> signature so the caller can guarantee they're querying an +> already-locked instance. Useful for hosts that lifecycle one +> instance at a time. + +### `ccpp_gather_constituents(const_array, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `const_array` | `real(kind=kind_phys), intent(out) :: (:,:,:)` | Destination buffer for constituent values. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%copy_in(const_array, ...)`. Use this to +pull the per-instance constituent values into a host-side array. + +### `ccpp_update_constituents(const_array, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `const_array` | `real(kind=kind_phys), intent(in) :: (:,:,:)` | Source buffer with updated constituent values. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%copy_out(const_array, ...)`. Use this to +push host-side updates back into the per-instance constituent object. + +### `ccpp_const_get_index(stdname, const_index, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `stdname` | `character(len=*), intent(in)` | Constituent standard name to look up. | +| `const_index` | `integer, intent(out)` | Returned index into the constituent array (or `int_unassigned` on miss). | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%const_index(standard_name=stdname, +index=const_index, ...)`. For constituents whose std names are known +at code-gen time, prefer using the module-level `index_of_` integer +directly (no call needed; it's bound during +`ccpp_initialize_constituents`). + +### `ccpp_constituents_array(instance_number) result(const_ptr)` + +Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → +`obj(instance_number)%field_data_ptr()`. + +### `ccpp_advected_constituents_array(instance_number) result(const_ptr)` + +Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → +`obj(instance_number)%advected_constituents_ptr()`. Subset of the +full constituent array containing only those flagged `advected=.true.`. + +### `ccpp_model_const_properties(instance_number) result(const_ptr)` + +Returns `type(ccpp_constituent_prop_ptr_t), pointer :: const_ptr(:)` → +`obj(instance_number)%constituent_props_ptr()`. + +### `ccpp_deallocate_dynamic_constituents(instance_number)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `instance_number` | `integer, intent(in)` | | + +**Per-instance reset + last-to-leave teardown**: +1. `obj(instance_number)%reset()` — unlocks the table for this instance. +2. Iterates every `obj(i)` and checks `%const_props_locked()`. If any + instance is still locked, the routine returns. +3. If **every** instance has been reset (none still locked), the routine + tears down the shared state: + - Deallocates every `_dynamic_constituents` buffer. + - Deallocates `ccpp_model_constituents_obj(:)`. + - Resets every `index_of_` integer to 0. + +The host should call this for every instance that successfully called +`ccpp_register_constituents`. + +--- + +## 6. Generated code structure + +When any suite touches constituent state, capgen-ng emits one extra +module per generator run: **`ccpp_host_constituents.F90`**. + +### Module declarations + +```fortran +module ccpp_host_constituents + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: & + ccpp_model_constituents_t, & + ccpp_constituent_properties_t, & + ccpp_constituent_prop_ptr_t + + implicit none + private + + ! ----- public state ---------------------------------------------------- + public :: ccpp_model_constituents_obj + public :: index_of_ ! one per known constituent std name + public :: index_of_ + public :: ccpp_model_const_stdnames ! parameter array + + ! ----- public routines (also re-exported from ccpp_static_api) -------- + public :: ccpp_register_constituents + public :: ccpp_initialize_constituents + public :: ccpp_is_scheme_constituent + public :: ccpp_number_constituents + public :: ccpp_gather_constituents + public :: ccpp_update_constituents + public :: ccpp_const_get_index + public :: ccpp_constituents_array + public :: ccpp_advected_constituents_array + public :: ccpp_model_const_properties + public :: ccpp_deallocate_dynamic_constituents + public :: _dynamic_constituents ! one per suite with register-phase producers + public :: _dynamic_constituents + + ! ----- module-level state --------------------------------------------- + type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) + type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) + type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) + integer :: index_of_ = 0 + integer :: index_of_ = 0 + character(len=N), parameter :: ccpp_model_const_stdnames(K) = (/ & + ' ', & + ' ' /) + +contains + ! ... routines as documented in §5 ... +end module ccpp_host_constituents +``` + +### Suite-cap responsibilities + +`ccpp__cap.F90` does NOT own constituent state. Its +`_register` routine packs each register-phase scheme's +constituent array into the suite's `_dynamic_constituents` +buffer (USE'd from `ccpp_host_constituents`): + +```fortran +if (.not. allocated(_dynamic_constituents)) then + ! First-instance-only two-pass count + populate. + num_consts = 0 + call _register(scheme_consts=scheme_consts, ...) + num_consts = num_consts + size(scheme_consts, 1) + deallocate(scheme_consts) + ... + allocate(_dynamic_constituents(num_consts)) + num_consts = 0 + call _register(scheme_consts=scheme_consts, ...) + do i = 1, size(scheme_consts, 1) + _dynamic_constituents(num_consts + i) = scheme_consts(i) + end do + num_consts = num_consts + size(scheme_consts, 1) + deallocate(scheme_consts) + ... +end if +``` + +The buffer is **shared across instances** (registration is identical +per instance); only the first instance to call `_register` +populates it. The host-wide merge happens in +`ccpp_register_constituents`. + +### Group-cap call sites + +`ccpp___cap.F90` USE's the constituent symbols it needs +from `ccpp_host_constituents`: + +```fortran +use ccpp_host_constituents, only: ccpp_model_constituents_obj, & + index_of_cloud_liquid_water_mixing_ratio +``` + +… and emits scheme call sites with the per-instance access expression: + +```fortran +call cld_liq_run( & + ... + cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer(lb:ub, 1:nlev, & + index_of_cloud_liquid_water_mixing_ratio), & + tend_cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer_tend(lb:ub, 1:nlev, & + index_of_cloud_liquid_water_mixing_ratio), & + ...) +``` + +The `instance_number` dummy is auto-injected into the group-cap +subroutine signatures by `_extra_dim_ctrl_entries` because the +resolver adds `instance_number` to every constituent arg's +`used_dim_std_names`. + +### Framework F90 dependencies + +`ccpp_host_constituents.F90` and the suite caps depend on these +framework files (listed under `` in `datatable.xml`): + +| File | Why | +|---|---| +| `ccpp_constituent_prop_mod.F90` | Provides `ccpp_model_constituents_t`, `ccpp_constituent_properties_t`, `ccpp_constituent_prop_ptr_t`. | +| `ccpp_hashable.F90` | Transitive dep of `ccpp_constituent_prop_mod`. | +| `ccpp_hash_table.F90` | Transitive dep. | +| `ccpp_scheme_utils.F90` | Provides `ccpp_initialize_constituent_ptr` + `ccpp_constituent_index` (used by cam-sima rrtmgp / mmm schemes). | + +The host's CMake should query `ccpp_datafile.py --utility-files` to +get the absolute paths to these files at the right output location. + +--- + +## 7. Multi-instance design + +In capgen-ng, **per-instance state** means: each "instance" (typically +an OpenMP team / chunk-domain partition) has its own copy of the +state arrays, indexed by `instance_number ∈ [1, number_of_instances]`. + +### What's per-instance + +| State | Storage | +|---|---| +| Constituent values + tendencies | `ccpp_model_constituents_obj(:)` — one DDT per instance | +| Suite-level state machine | `ccpp_suite_state(:)` — declared in each suite cap | +| Suite-owned data | `ccpp_suite_data(:)` — declared in each suite-data module | +| Group-level state machine | `ccpp_group_state(:)` — declared in each group cap | + +### What's shared across instances + +| State | Reason | +|---|---| +| `_dynamic_constituents(:)` per-suite buffers | Registration is identical per instance — populated by the first instance to call `_register` and reused by the rest | +| `index_of_` integers | The constituent table is identical per instance, so the indices are too | +| `ccpp_model_const_stdnames` parameter array | Compile-time constant | + +### Sizing + +`number_of_instances` is the single source of truth. The host declares +it in metadata + Fortran; the generator USE's it from the host module +wherever per-instance allocation happens. See the prior memo +[*Where the total number of instances comes from*](#) for the call +chain (and matching values across all four state arrays: +`ccpp_suite_state`, `ccpp_suite_data`, `ccpp_group_state`, +`ccpp_model_constituents_obj`). + +If the host doesn't declare `number_of_instances`, every per-instance +allocation falls back to `1` and the framework runs single-instance. + +### Two host-side lifecycle patterns + +Both work; pick whichever fits your model. + +**Pattern A: all instances registered first** +``` +do isuite = 1, num_suites + do iinst = 1, num_instances + call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) + end do +end do +do iinst = 1, num_instances + call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) + call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) +end do +do isuite = 1, num_suites + do iinst = 1, num_instances + call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) + end do +end do +! ... time-stepping ... +do iinst = 1, num_instances + call ccpp_deallocate_dynamic_constituents(iinst) + ... +end do +``` + +**Pattern B: serial per instance** +``` +do iinst = 1, num_instances + do isuite = 1, num_suites + call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) + end do + call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) + call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) + do isuite = 1, num_suites + call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) + end do + ! ... per-instance time-stepping ... + call ccpp_deallocate_dynamic_constituents(iinst) +end do +``` + +### Last-to-leave teardown + +`ccpp_deallocate_dynamic_constituents(inst)`: +1. Per-instance `obj(inst)%reset()`. +2. Iterates every `obj(i)`; if any has `%const_props_locked() == .true.`, + returns early. +3. Otherwise (every instance reset): deallocates the shared per-suite + buffers, deallocates `ccpp_model_constituents_obj(:)`, and zeros every + `index_of_` integer. + +This works for both lifecycle patterns above. + +--- + +## 8. Limitations and gotchas + +> **Note (2026-05-12).** Several items in this section are under active +> discussion for an upcoming framework + generator overhaul. See +> `doc/constituents_overhaul.md` for the full architectural review and +> three reform proposals. + +### Framework property ownership (2026-05-12) + +The framework's `ccpp_constituent_properties_t` now carries a private +`framework_owns_me` flag (default `.false.`) with +`is_framework_owned()` getter and `set_framework_owned(value)` setter. +`ccpt_deallocate` only deallocates the underlying prop when the flag +is `.true.`; otherwise it just nullifies its pointer. + +Under capgen-ng's explicit-registration model, all +`ccpp_constituent_properties_t` objects are **target-owned by the +caller** (the host's `host_constituents(:)` array, or the per-suite +`_dynamic_constituents(:)` buffer). We never set the flag, so +the framework correctly skips deallocation. Hosts that hand-allocate +property objects on the heap and want the framework to free them must +call `set_framework_owned(.true.)` before passing to `%new_field`. + +### Missing setters (framework gap) + +The framework lacks setters for `advected`, `diagnostic_name`, +`default_value` (and `mixing_ratio_type`). This means once a +constituent is `%instantiate`d, those properties cannot be changed. +If your host needs to override a scheme-supplied `diagnostic_name` or +`advected` value, you currently cannot — open item in the constituents +overhaul proposal. + +### `ccpp_scheme_utils` singleton + +`ccpp_scheme_utils.ccpp_initialize_constituent_ptr` accepts only one +singleton pointer. It's a framework-level convenience used by cam-sima +schemes that call `ccpp_constituent_index` from `ccpp_scheme_utils`. + +`ccpp_initialize_constituents` calls +`ccpp_initialize_constituent_ptr(obj(inst))` on each instance, but +**only the first call across instances actually sets the pointer** +(the routine is internally guarded by an `initialized` flag). +Subsequent calls are silent no-ops. + +For multi-instance hosts, schemes that use +`ccpp_scheme_utils.ccpp_constituent_index` will see only the first +instance's object — a known limitation inherited from the framework +module's design. Schemes that use the per-instance accessors +(`obj(inst)%const_index(...)` via `ccpp_const_get_index`) are +unaffected. + +### Constituent metadata is identical across instances + +The constituent table (which constituents exist, their properties, the +`index_of_` mapping) is **identical** for every instance. Every +instance's `obj(i)` has the same hash table, populated identically by +its own `ccpp_register_constituents` call. + +This means: + +- `ccpp_number_constituents` returns the same value regardless of + `instance_number`. +- `ccpp_const_get_index` returns the same index regardless of + `instance_number`. +- The `index_of_` integers are populated identically by every + instance's `ccpp_initialize_constituents` (last-write-wins is fine + since every write is the same value). + +`instance_number` is still in the signatures of these routines — see +[§5](#5-public-api-reference) for the rationale. + +### Forbidden patterns recap + +These are rejected at code-gen time (Rule 4 of [§2](#2-the-four-rules-scheme-author-conventions)): + +- `is_constituent + intent=out + non-tendency std_name` — physics phases + may only produce tendencies, not new base constituents. +- `is_constituent + intent=in/inout + tendency_of_*` — tendencies are + write-only. + +### Subscript indices in sliced local_names must be standard names + +If a host metadata variable is declared with a sliced local name +like `q(:,:,index_of_water_vapor_specific_humidity)`, every subscript +token (other than `:` and integer literals) must be a known standard +name. Otherwise the resolver raises a `CCPPError` with a clear +message naming the offending token. + +### Open work items + +- **Unconditional `ccpp_host_constituents.F90` emission.** The + generator currently emits `ccpp_host_constituents.F90` for every + build, even when no scheme or host actually uses the constituent + system (no `ccpp_constituent_properties_t(:)` register-phase arg, + no `is_constituent`-flagged scheme arg, no framework-named + `index_of_` / `ccpp_constituents` / etc. claimed by capgen-ng). + When the host owns its own indices (SCM/GFS) and no scheme exercises + the constituent path, the generated file is dead code that should be + suppressed. Tracked as a deferred item; the `host_dict` precedence + rule above already keeps the file *correct* (empty) in that case. + +--- + +## 9. Differences from original capgen + +| Aspect | Original capgen | capgen-ng | +|---|---|---| +| Constituent object location | Generated `_ccpp_cap.F90` module | `ccpp_host_constituents.F90` (one per generator run) | +| Per-instance | No (single instance) | Yes (`obj(:)` allocatable, sized to `number_of_instances`) | +| Routine name prefix | `_ccpp_register_constituents`, etc. | `ccpp_register_constituents`, etc. (no host prefix; one set per generator run) | +| Routine signatures | No `instance_number` arg | `instance_number` in every per-instance routine | +| Host metadata for constituent obj | None (auto-created by generator) | None (still auto-created by generator) | +| Module-level pointers | `_constituents_array` etc. as functions returning pointers | Same idea, now per-instance via `instance_number` arg | +| Scheme std-name set | `_model_const_stdnames` parameter array | `ccpp_model_const_stdnames` parameter array (no host prefix) | +| Host-facing API surface | `_ccpp_register_constituents`, `_ccpp_initialize_constituents`, `_ccpp_number_constituents`, `_ccpp_is_scheme_constituent`, `_ccpp_gather_constituents`, `_ccpp_update_constituents`, `_ccpp_deallocate_dynamic_constituents`, `_constituents_array`, `_advected_constituents_array`, `_model_const_properties`, `_const_get_index` | Same surface, no `_` prefix | +| Dynamic constituent buffer dimensionality | 1D, per host | 1D, per generator run, **shared across instances** | +| Static suite constituents | Auto-cloned by `ConstituentVarDict.find_variable` and registered via `_constituents_copy_const` accessors | Tracked at code-gen time via `is_constituent` flag; included in the constituent table only if a register-phase scheme produces them (rule 1). Schemes that *consume* a base constituent (rule 2) don't trigger registration — the constituent must be registered by SOMEONE (host or another scheme's register) for the access to work at runtime. | + +### Migration notes for cam-sima hosts + +- **Scheme metadata**: no changes needed. Cam-sima follows rules 2 and + 3 already (audited 2026-05-11). The 4 schemes that register + constituents via `ccpp_constituent_properties_t` (rule 1) work + unchanged. +- **Host metadata**: drop any explicit declaration of + `ccpp_model_constituents_object` if you carried one over from a + previous capgen-ng experiment — the generator owns it now. +- **Host Fortran**: change all `_ccpp_*_constituents` calls to + the unprefixed names (`ccpp_register_constituents` etc.) and add + `instance_number` to every call site. + +--- + +## 10. Worked example + +A minimal cam-sima-style suite with one scheme that consumes a base +constituent and produces its tendency. + +### Scheme metadata (`consume_constituent.meta`) + +``` +[ccpp-table-properties] + name = consume_constituent + type = scheme + +[ccpp-arg-table] + name = consume_constituent_run + type = scheme +[ cldliq ] + standard_name = cloud_liquid_water_mixing_ratio + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in + advected = .true. +[ tend_cldliq ] + standard_name = tendency_of_cloud_liquid_water_mixing_ratio + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = .true. +[ errmsg ] + ... +[ errflg ] + ... +``` + +### Host metadata (`my_host.meta`) + +``` +[ccpp-table-properties] + name = my_host + type = host + +[ccpp-arg-table] + name = my_host + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + units = count + dimensions = () + type = integer +``` + +(Plus a `type=control` table declaring `instance_number`, +`horizontal_loop_begin`, `horizontal_loop_end`, `ccpp_error_message`, +`ccpp_error_code`, etc.) + +### Suite XML (`my_suite.xml`) + +```xml + + + + consume_constituent + + +``` + +### Generated `ccpp_host_constituents.F90` (excerpt) + +```fortran +module ccpp_host_constituents + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: & + ccpp_model_constituents_t, ccpp_constituent_properties_t, & + ccpp_constituent_prop_ptr_t + + implicit none + private + + public :: ccpp_model_constituents_obj + public :: index_of_cloud_liquid_water_mixing_ratio + public :: ccpp_register_constituents, ccpp_initialize_constituents + public :: ccpp_is_scheme_constituent, ccpp_number_constituents + public :: ccpp_gather_constituents, ccpp_update_constituents + public :: ccpp_const_get_index, ccpp_constituents_array + public :: ccpp_advected_constituents_array, ccpp_model_const_properties + public :: ccpp_deallocate_dynamic_constituents + public :: ccpp_model_const_stdnames + + type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) + integer :: index_of_cloud_liquid_water_mixing_ratio = 0 + character(len=31), parameter :: ccpp_model_const_stdnames(1) = (/ & + 'cloud_liquid_water_mixing_ratio' /) + +contains + ! ... full subroutine bodies as in §5 ... +end module ccpp_host_constituents +``` + +### Host code skeleton (single-instance illustration) + +```fortran +subroutine my_host_run() + use ccpp_static_api, only: ccpp_register, ccpp_register_constituents, & + ccpp_initialize_constituents, ccpp_init, & + ccpp_physics_run, ccpp_final, & + ccpp_deallocate_dynamic_constituents + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + type(ccpp_constituent_properties_t), allocatable :: host_consts(:) + integer :: errflg + character(len=512) :: errmsg + integer, parameter :: inst = 1 + + ! 1. Run register phase: populates per-suite dynamic-constituent buffers. + call ccpp_register('my_suite', errmsg, errflg, inst) + + ! 2. Build host's own constituent declarations (water vapor, etc.). + allocate(host_consts(1)) + call host_consts(1)%instantiate( & + std_name='water_vapor_specific_humidity', long_name='water vapor', & + units='kg kg-1', vertical_dim='vertical_layer_dimension', & + advected=.true., errcode=errflg, errmsg=errmsg) + + ! 3. Merge host + suite-side constituents into obj(inst). + call ccpp_register_constituents(host_consts, inst, errflg, errmsg) + + ! 4. Allocate vars_layer + bind cached indices. + call ccpp_initialize_constituents(ncols, nlev, inst, errflg, errmsg) + + ! 5. Framework init phase. + call ccpp_init('my_suite', errmsg, errflg, inst) + + ! 6. Time-stepping (omitted). + call ccpp_physics_run('my_suite', 'phys', col_start, col_end, & + thread_num, nthreads, nphys_threads, & + errflg, errmsg, inst) + + ! 7. Shutdown. + call ccpp_final('my_suite', errmsg, errflg, inst) + call ccpp_deallocate_dynamic_constituents(inst) + deallocate(host_consts) +end subroutine my_host_run +``` + +For multi-instance, wrap each per-instance call in +`do iinst = 1, ninstances ... end do` per the patterns in +[§7](#7-multi-instance-design). diff --git a/doc/constituents_overhaul_20260513T0733.md b/doc/constituents_overhaul_20260513T0733.md new file mode 100644 index 00000000..7d177cae --- /dev/null +++ b/doc/constituents_overhaul_20260513T0733.md @@ -0,0 +1,829 @@ +# CCPP Constituents — Architecture Review & Overhaul Discussion + +**Authors:** Dom Heinzeller (lead), Claude (assistant) +**Date drafted:** 2026-05-12 +**Intended audience:** CCPP framework team, CAM-SIMA team +**Status:** Discussion document — no decisions are final. + +--- + +## Executive summary + +CCPP's "constituent" mechanism — how schemes declare and how the framework +manages tracer species like water vapor, cloud liquid, prescribed ozone, +etc. — has grown organically over the last few years. The result works, +but it carries: + +- **A latent framework bug** in `ccpp_constituent_prop_mod` that crashes on + teardown of explicitly-registered (target-passed) constituent property + arrays. Fixed in capgen-ng's framework copy 2026-05-12; needs to land + upstream. +- **Architectural confusion** about which properties are *physics-portable* + (the scheme owns them) versus *host-configuration* (the host owns them). + Today schemes are forced to supply host-specific values (`diag_name` is + the worst offender) at `%instantiate` time. +- **Setter API gaps**: properties that the host wants to override after + scheme-side registration (`advected`, `diagnostic_name`, `default_value`) + have no setters; `is_match` is overly strict about properties hosts + should be free to change. +- **Two registration models** coexist — original capgen's auto-clone of + is_constituent scheme args, and capgen-ng's explicit register-phase + + host-side declaration. Capgen-ng deliberately dropped auto-clone. + +This document is a structured brief for a discussion this week. It does +NOT pre-commit to any decision; it lays out what exists, what's broken, +what we audited, and what proposals are on the table. + +--- + +## Table of contents + +1. [How original capgen handles constituents](#1-how-original-capgen-handles-constituents) +2. [How capgen-ng handles constituents](#2-how-capgen-ng-handles-constituents) +3. [What CAM-SIMA actually needs (audit)](#3-what-cam-sima-actually-needs-audit) +4. [Bugs and design flaws](#4-bugs-and-design-flaws) +5. [Property classification (Class A vs Class B)](#5-property-classification-class-a-vs-class-b) +6. [What to remove, replace, improve](#6-what-to-remove-replace-improve) +7. [Open design questions](#7-open-design-questions) +8. [Three proposals — minimal / clean / deep](#8-three-proposals--minimal--clean--deep) +9. [Appendix: framework setter inventory](#9-appendix-framework-setter-inventory) + +--- + +## 1. How original capgen handles constituents + +### 1.1 Mental model + +Original capgen treats constituents as a **separate scope** between +suite and host: + +``` +group → suite → ConstituentVarDict → host +``` + +A scheme arg flagged `constituent = True` in metadata is matched first +against group/suite/ConstituentVarDict, and only against host as a last +resort. The ConstituentVarDict is a synthetic dictionary whose entries +are auto-created by `find_variable()` when a scheme metadata declares a +constituent dependency. + +### 1.2 Auto-clone of `is_constituent` scheme args + +Every scheme arg with non-default `advected`, `constituent`, or +`molar_mass` is treated as a *registration*. The generator emits, into +the host cap, a routine `_constituents_ccpp_create_constituent_array` +that: + +1. Allocates a `ccpp_constituent_properties_t` pointer per scheme arg. +2. Calls `%instantiate(...)` populating fields **from the scheme + metadata directly** — `std_name`, `long_name`, `diagnostic_name`, + `units`, `default_value`, `advected`, `vertical_dim`, etc. (See + `scripts/constituents.py:565`.) +3. Adds it to the model constituents object via `%new_field`. + +After this auto-clone runs, the host's hand-written +`host_constituents(:)` array is appended, then `%lock_table` finalizes +the hash table. + +### 1.3 The host-cap-owned `ccpp_model_constituents_obj` + +Original capgen generates **one** `ccpp_model_constituents_obj` per +generator invocation, declared module-level in `_ccpp_cap.F90`. +Single global; not per-instance. (CAM-SIMA runs one host per +executable, so single-instance is fine for them.) + +### 1.4 Scheme-side `%instantiate` registration (the other path) + +A scheme may also register constituents via a register-phase argument: + +```fortran +type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) +``` + +The scheme allocates the array, calls `%instantiate` per entry, and +returns it. Original capgen wires this through a per-suite +"dynamic constituents" buffer and merges it during host-cap setup, +alongside the auto-cloned set. + +So original capgen really supports **three** registration sources: + +- Host: hand-written `host_constituents(:)` arg. +- Suite-dynamic: register-phase scheme args. +- Suite-static: auto-cloned from any `is_constituent` consumer. + +All three flow into one `%new_field` table. + +### 1.5 Lifecycle + +- `_ccpp_register_constituents(host_constituents, ...)` runs the + three-source merge. +- `ccpp_initialize_constituent_ptr(const_obj, ...)` (in + `ccpp_scheme_utils`) caches a pointer for the `ccpp_constituent_index` + lookup. +- Phase entry points access `vars_layer` / `vars_layer_tend` via cached + `index_of_` integers. + +### 1.6 What's good about original capgen's approach + +- Schemes declare a constituent dependency once in metadata; no manual + Fortran registration ever needed for "static" tracers. +- Host doesn't have to enumerate every species every scheme wants. +- Works for CAM-SIMA's current scheme catalog. + +### 1.7 What's painful about original capgen's approach + +- The auto-clone path is **invisible** to anyone reading the scheme + Fortran — the registration happens in generated code. +- `ConstituentVarDict` is a synthetic scope, conceptually subtle, and + doesn't generalize cleanly to multi-instance. +- The auto-clone path lifts `diagnostic_name` and `default_value` from + scheme metadata, but those values are often host-specific (see §4.4). +- Three sources of registration with overlap mean two registrations of + the same `std_name` may collide; original capgen relies on + `is_match` (units, advected, thermo_active, water_species) to dedup, + which means schemes accidentally diverge on `advected` and trip the + "incompatible constituent" error. + +--- + +## 2. How capgen-ng handles constituents + +### 2.1 Mental model + +No synthetic scope. Constituents are *one of four* sources for any +scheme arg: + +``` +control | host | suite | constituent +``` + +The resolver classifies each scheme arg into exactly one source. A +`constituent` source means the value will be accessed at runtime as +`ccpp_model_constituents_obj()%vars_layer(:, :, index_of_)` +(or `%vars_layer_tend(...)` for `tendency_of_` outputs). + +### 2.2 The four scheme-author rules + +(See `doc/constituents.md` for full details; this is the summary.) + +1. **Register** — register-phase scheme args of type + `ccpp_constituent_properties_t(:), intent=out, allocatable` declare + new constituents the scheme contributes. +2. **Consume** — physics-phase scheme args with `advected=true` (or + `molar_mass=...` or `constituent=true`) and `intent=in/inout`, with + the constituent's standard name, read the base species. +3. **Produce a tendency** — physics-phase scheme args with + `constituent=true`, `intent=out`, and standard name + `tendency_of_`, write the tendency. +4. **Mismatched combinations are errors** — `intent=out` on a base + constituent, or `intent=in` on a tendency, are codegen-time errors. + +### 2.3 Two registration sources (no auto-clone) + +- **Host**: hand-written `host_constituents(:)`, passed into + `ccpp_register_constituents(host_constituents, instance_number, ...)`. +- **Suite-dynamic**: register-phase scheme args, accumulated into a + per-suite buffer `_dynamic_constituents(:)` by `_register`, + drained into `ccpp_model_constituents_obj(inst)` by + `ccpp_register_constituents`. + +The auto-clone-from-metadata path is deliberately **gone**. If a scheme +declares `advected=true` on an arg but no source registers that +standard name, capgen-ng now emits a runtime check during +`ccpp_initialize_constituents` that errors with the missing name. + +### 2.4 Per-instance state + +Everything is per-instance: + +```fortran +type(ccpp_model_constituents_t), allocatable :: ccpp_model_constituents_obj(:) + ! indexed by instance_number +``` + +All host-facing entry points take `instance_number`: + +``` +ccpp_register_constituents (host_constituents, instance_number, errflg, errmsg) +ccpp_initialize_constituents (ncols, num_layers, instance_number, errflg, errmsg) +ccpp_number_constituents (num_flds, advected, instance_number, errflg, errmsg) +ccpp_gather_constituents (const_array, instance_number, errflg, errmsg) +ccpp_update_constituents (const_array, instance_number, errflg, errmsg) +ccpp_const_get_index (stdname, const_index, instance_number, errflg, errmsg) +ccpp_constituents_array (instance_number) => pointer +ccpp_advected_constituents_array (instance_number) => pointer +ccpp_model_const_properties (instance_number) => pointer +ccpp_deallocate_dynamic_constituents (instance_number, ...) +``` + +`ccpp_is_scheme_constituent(var_name, ...)` and the +`ccpp_model_const_stdnames(:)` parameter array are NOT per-instance — +the standard-name catalog is identical across instances. + +### 2.5 Lifecycle + +``` +ccpp_register(suite_name, instance_number, ...) + └─ _register → packs scheme-dynamic constituents into + _dynamic_constituents (shared buffer, + first instance wins) + ↓ +ccpp_register_constituents(host_constituents, instance_number, ...) + └─ initialize_table(num_host_consts + num_suite_consts) + └─ new_field(host_consts ...) + └─ new_field(_dynamic_constituents ...) + └─ lock_table + ↓ +ccpp_initialize_constituents(ncols, num_layers, instance_number, ...) + └─ lock_data (allocates vars_layer, vars_layer_tend, vars_minvalue) + └─ ccpp_initialize_constituent_ptr (first instance wins; documented limit) + └─ %const_index('') for each enumerated constituent + └─ post-lookup int_unassigned check → clear error message + ↓ +ccpp_init(suite_name, instance_number, ...) + └─ _init → binds module-level pointers + ↓ +... physics phases ... + ↓ +ccpp_final(suite_name, instance_number, ...) + └─ _final → nullifies + last-to-leave deallocates + ↓ +ccpp_deallocate_dynamic_constituents(instance_number, ...) + └─ ccp_model_constituents_obj(inst)%reset + ↓ (in _final, last-to-leave) + deallocate(_dynamic_constituents) +``` + +### 2.6 What's good + +- Explicit. Every constituent registration is visible in someone's + Fortran source. +- Multi-instance from day one. +- The "four rules" are small enough to fit on a slide. +- Resolver-time + codegen-time + runtime checks catch the most common + mistakes. + +### 2.7 What's still painful + +Covered in §4. + +--- + +## 3. What CAM-SIMA actually needs (audit) + +### 3.1 Scheme-side registration usage + +We audited `EXT/cam-sima/atmospheric_physics/schemes/` for use of the +register-phase `ccpp_constituent_properties_t(:)` pattern: + +| Scheme | File | Registers | +|---|---|---| +| RRTMGP constituents | `schemes/rrtmgp/rrtmgp_constituents.meta` | radiative-active species | +| MUSICA chemistry | `schemes/musica/musica_ccpp.meta` | chemical species from MUSICA | +| Prescribed aerosols | `schemes/chemistry/prescribed_aerosols.meta` | aerosol species | +| Prescribed ozone | `schemes/chemistry/prescribed_ozone.meta` | ozone | + +**Total: 4 of 128 schemes** in the atmospheric_physics tree use +scheme-side registration. The other 124 only **consume** constituents +(`advected=true` + `intent=in/inout` in metadata, accessed via the +framework's `vars_layer`). + +This is a small enough number that an alternative "host-only +registration" model is feasible: move those 4 register calls into the +host (or into helper modules the host calls), and the rest of the +catalog only consumes. + +### 3.2 Host-side patterns + +`EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` wraps +the framework setters and exposes: + +- `const_set_thermo_active(const_obj | const_ind, value)` +- `const_set_water_species(const_obj | const_ind, value)` +- `const_set_minimum(...)` + +CAM-SIMA actively **calls these setters at runtime** — schemes don't +supply `thermo_active` at instantiate time; the host configures it +afterwards. This is direct evidence that the "post-instantiation +override" pattern is real and used today, and that the framework's +setter API is load-bearing. + +### 3.3 What CAM-SIMA does **not** do + +- It does not rely on auto-clone for `diag_name`. The scheme-side + register calls in the 4 schemes do supply `diag_name`, but those + values are CAM-SIMA's; a different host would need different ones. +- It does not use `ccpp_constituent_index` (the + `ccpp_scheme_utils`-singleton-based lookup) extensively — most + access goes through the framework's `index_of_` integers. + +### 3.4 What CAM-SIMA's host-cap-owned constituent object looks like + +Because original capgen generates **one** `ccpp_model_constituents_obj` +per generator invocation, and CAM-SIMA uses one generator invocation per +executable, CAM-SIMA effectively runs single-instance today. A +multi-instance CAM-SIMA (sub-columns, ensembles) would expose the +single-global limitation immediately. + +--- + +## 4. Bugs and design flaws + +This section lists known issues across the three layers (framework, +original capgen, capgen-ng). Items marked **(FIXED)** were resolved +2026-05-12 and either are or will be PRs; items marked **(OPEN)** are +intentionally left for this discussion. + +### 4.1 Framework: `ccpt_deallocate` ownership bug (FIXED in capgen-ng tree, needs upstream PR) + +- **Location**: `src/ccpp_constituent_prop_mod.F90`, `ccpt_deallocate` + + `ccpt_set`. +- **Symptom**: `free(): invalid size` crash when + `ccp_model_const_reset` is called on a properly-locked table whose + entries came from pointer-assigned targets (the common pattern + under capgen-ng's explicit registration; also potentially under + original capgen's `host_constituents` path). +- **Root cause**: `ccpt_set` does pointer assignment (`this%prop => + const_ptr`); `ccpt_deallocate` does an unconditional + `deallocate(this%prop)`. The deallocate is correct only when the + caller allocated `const_ptr` on the heap and transferred ownership. +- **Why it didn't surface earlier**: original capgen's advection test + only calls `deallocate` once between a *failing* register and a + *successful* one — at that point `lock_table` has not populated + `const_metadata`, so the broken inner loop is skipped. Capgen-ng + triggers it because its teardown calls `reset` after a successful + lock. +- **Fix landed 2026-05-12**: added `framework_owns_me` private flag on + `ccpp_constituent_properties_t` (default `.false.`) with + `is_framework_owned()` getter and `set_framework_owned(value)` + setter; `ccpt_deallocate` now only deallocates when the flag is set. + Original capgen's auto-clone path in `scripts/constituents.py` + updated to call `set_framework_owned(.true.)` after `allocate`. + Diffs in `src/ccpp_constituent_prop_mod.F90` (and capgen-ng's + parallel copy) + `scripts/constituents.py`. +- **Status**: framework tests pass, capgen-ng tests pass (954). Needs + upstream PR to ccpp-framework + ccpp-capgen. + +### 4.2 Framework: missing setters (OPEN) + +| Property | Optional in `%instantiate`? | Has setter? | `is_match`-checked? | +|---|---|---|---| +| `std_name` | required | — | (lookup key) | +| `long_name` | required | — | no | +| `diag_name` | required | **NO** | no | +| `units` | required | — | **yes** | +| `vertical_dim` | required | — | no | +| `advected` | optional (default .false.) | **NO** | **yes** | +| `default_value` | optional | **NO** | no | +| `min_value` | optional | `set_minimum` | no | +| `molar_mass` | optional | `set_molar_mass` | no | +| `water_species` | optional (default .false.) | `set_water_species` | **yes** | +| `mixing_ratio_type`| optional | **NO** | no | +| `thermo_active` | not in instantiate | `set_thermo_active` | **yes** | +| `const_index` | internal | `set_const_index` | no | + +**Pain points**: + +- `advected` is `is_match`-checked AND has no setter. Once registered, + immutable. If a scheme and the host disagree, you get the + "incompatible constituent" error and you cannot reconcile from + Fortran. +- `diag_name` is required (cannot be omitted at instantiate) AND has + no setter. A scheme must pick a value at registration time; that + value is then frozen. +- `default_value` is silently optional. If omitted, the constituent + array initializes to `huge(real)` and downstream comparisons fail + in surprising ways (we burnt half a day on this 2026-05-12). +- `thermo_active` is the only property in the "post-instantiate-only" + shape: it has a setter but isn't a `%instantiate` arg. The + asymmetry is confusing. + +### 4.3 Framework: `is_match` is too strict (OPEN) + +`is_match` (in `ccp_is_match`) checks `units`, `advected`, +`thermo_active`, `water_species`. Three of those four (`advected`, +`thermo_active`, `water_species`) are properties the host legitimately +overrides post-registration. Two registrations of the same `std_name` +with the same `units` but different `advected` should be a +duplicate-dedup (host wins), not a hard error. + +### 4.4 Framework: `diag_name` portability problem (OPEN) + +Diagnostic output names are host-specific. CAM-SIMA names cloud +liquid mixing ratio `CLDLIQ`; UFS would call it something else. Yet +`%instantiate` makes `diag_name` a *required* arg, forcing schemes to +either: + +- Pick a host-specific value (couples the scheme to a host), or +- Pick a "neutral" default that no host's diagnostic tooling + recognizes. + +The current de-facto pattern in CAM-SIMA scheme code is to pick a +CAM-SIMA-flavoured value and ship it. Any port to UFS would need to +either monkey-patch or fork the scheme. + +A clean fix: +1. Make `diag_name` optional at `%instantiate` (default to empty + string or `std_name`). +2. Add `set_diagnostic_name(value)` setter. +3. Host overrides per-registration after `ccpp_register_constituents`. + +### 4.5 Original capgen: implicit registration (OPEN — observation) + +The auto-clone path is generator magic. Reading scheme metadata +doesn't tell you whether the scheme's args result in registration; you +have to know that `advected=true` triggers it. This is a documentation ++ comprehension problem more than a bug. + +### 4.6 Original capgen: single-instance `ccpp_model_constituents_obj` (OPEN — limitation) + +The host cap declares one global. Multi-instance hosts would need to +either generate one cap per instance or restructure. + +### 4.7 Original capgen: `ConstituentVarDict` complexity (OPEN — observation) + +The synthetic scope between suite and host serves correctness but +adds a code path that most contributors don't read. If we drop it +(capgen-ng has), the variable-matching algorithm shrinks. + +### 4.8 Capgen-ng: `_FRAMEWORK_CONST_DIM_INPUTS` is hand-curated (OPEN) + +`generator/static_api.py` carries a frozenset of standard names +(currently just `number_of_ccpp_constituents`) that introspection +treats specially. Cleaner: a dedicated `used_const_dim_std_names` +field on `ResolvedArg`. Marked REVISIT in the code. + +### 4.9 Capgen-ng: no codegen-time cross-check of scheme registration (OPEN) + +The resolver knows every `is_constituent` arg's standard name (in +`SuiteResolution.constituent_index_names`) but doesn't know what each +scheme's `_register` subroutine actually `%instantiate`s. Today's +guarantee is a runtime check (the `int_unassigned` validation we +added 2026-05-12). Stronger options: + +- (a) New metadata attribute `registers_std_names = a, b, c` on + register-phase tables; codegen errors at generation time. +- (b) Parse scheme `_register` Fortran for `%instantiate(std_name=…)` + calls and cross-check. +- (c) Keep runtime check as authoritative, document the gap. + +### 4.10 Capgen-ng: scheme-metadata `diagnostic_name` for is_constituent args is host-specific (OPEN) + +Same issue as §4.4 but in capgen-ng's metadata layer. Today's +`diagnostic_name` attribute on a scheme metadata arg flows into +`datatable.xml` and is then trusted as "the" diagnostic name. If we +adopt setter-based class-B overrides, this attribute should either be +dropped for constituent args or marked as a default-only hint. + +### 4.11 Capgen-ng: `ccpp_scheme_utils` singleton (OPEN — documented limit) + +`ccpp_initialize_constituent_ptr(const_obj)` stores a single module-level +pointer. Schemes that use `ccpp_constituent_index(stdname)` get that +pointer back. Under multi-instance, only the first instance's +pointer is retained — `ccpp_constituent_index` queries from +within a scheme will always reflect instance 1. CAM-SIMA's 4 +scheme-registering schemes don't rely on this; documented in +`doc/constituents.md` §8. Real fix requires either threading +`instance_number` through `ccpp_constituent_index` (interface +change) or maintaining a per-instance pointer table. + +--- + +## 5. Property classification (Class A vs Class B) + +Proposed in `design_constituents_mutability.md` 2026-05-12. Each +constituent property is conceptually owned by either the scheme +(physics-portable, immutable once instantiated) or the host +(host-configuration, mutable post-instantiation). + +### Class A — scheme-intrinsic (immutable) + +| Property | Why class A | +|---|---| +| `std_name` | Identity. Cannot change. | +| `long_name` | Human-readable name of the *species*. Not host-specific. | +| `units` | Physics correctness. `is_match`-checked. | +| `vertical_dim` | Scheme's structural expectation (interface vs layer). | +| `molar_mass` | Physical constant of the species. | +| `default_value` | (Debatable — see §7) Scheme-appropriate initial value. | + +### Class B — host-configuration (mutable post-instantiation) + +| Property | Why class B | +|---|---| +| `advected` | Whether the host's dycore advects this — host decision. | +| `diag_name` | Host-specific diagnostic system name. | +| `thermo_active` | Host model configuration. | +| `min_value` | Host runtime guardrail. | +| `water_species` | (Borderline — see §7) Physical classification but also host-config. | +| `mixing_ratio_type` | (Borderline — see §7) Depends on dycore convention. | + +### Consequences if adopted + +- `is_match` should check **only class A**. Today it checks 3 of 4 + class-B properties. +- Class B properties need setters. Today + `advected`, `diag_name`, (and `mixing_ratio_type` if it stays + class B) have none. +- `%instantiate` can demote class B from "required + optional" to + "all optional with sane defaults" — `diag_name=''`, + `advected=.false.`, etc. Schemes wouldn't need to set them at all. + +--- + +## 6. What to remove, replace, improve + +### Remove (or stop requiring) + +- **Scheme-metadata `diagnostic_name` on is_constituent args** — host + will override. Keep the attribute valid on non-constituent args + (where it's host tooling documentation, no portability issue). +- **`is_match` checks on advected / water_species / thermo_active** — + class B should not block dedup. +- **The `diag_name` requirement at `%instantiate`** — demote to + optional with `''` default. +- **(Not adopting)** Original capgen's auto-clone path. Already gone + in capgen-ng; this discussion does not propose bringing it back. + Listed for completeness because the option is in memory. + +### Replace + +- **`ConstituentVarDict`** as a concept — capgen-ng already runs + without it. If the framework or future generator code references + it, dropping is fine. +- **Single-global `ccpp_model_constituents_obj`** — capgen-ng's + per-instance array is the replacement. Original capgen could be + retrofitted, but the priority depends on whether multi-instance + enters the original capgen's roadmap. + +### Improve + +- **Add the missing setters**: `set_advected`, `set_diagnostic_name`, + `set_default_value` (if `default_value` becomes class B), + `set_mixing_ratio_type` (if class B). +- **Add a convenience routine** like + `ccpp_get_constituent_props_by_std_name(stdname, instance_number, prop_ptr, errflg, errmsg)` + so hosts can lookup a single constituent's property wrapper by + name without iterating. +- **Codegen-time cross-check** of scheme `_register` calls vs + metadata declarations (preferred: §4.9 option (a) — new + `registers_std_names` attr). +- **Document the lifecycle** clearly. `doc/constituents.md` is + ~960 lines; targeted additions for "register-then-override" + workflow once the new setters land. +- **Capgen-ng-internal cleanup**: replace + `_FRAMEWORK_CONST_DIM_INPUTS` with a `used_const_dim_std_names` + field on `ResolvedArg`. + +--- + +## 7. Open design questions + +These are the calls we need to make in the meeting. + +### Q1. `default_value` — class A or class B? + +- **Class A argument**: the scheme knows what the species + should be initialized to (zero for "starts empty"; small positive + for "starts at background"); the host doesn't typically override. +- **Class B argument**: hosts may want non-default starting values + (chemistry runs with prescribed initial profiles). +- **Today's reality**: framework has no setter, so it's de-facto + class A. The advection-test issue 2026-05-12 surfaced because we + removed the `default_value=0._kind_phys` from cld_liq.F90's + scheme-side register and had no way to put it back; restoring it + in the scheme fixed the test but cements the class-A treatment. +- **Recommendation**: leave class A for now. Revisit when a real + host-override use case appears. + +### Q2. `water_species` — class A or class B? + +- The current `is_match` check on `water_species` treats it as + identity-defining (class A semantics). But the actual *meaning* of + the bit is mostly host bookkeeping ("does the dycore treat this as + water?"). CAM-SIMA has a `set_water_species` wrapper and uses it. +- **Recommendation**: class B, with the caveat that schemes whose + numerics depend on a constituent *being* water should declare that + in metadata as a hard requirement (different mechanism — not the + `is_match` machinery). + +### Q3. `mixing_ratio_type` — class A or class B? + +- The scheme's calculations assume `wrt_dry` or `wrt_moist`; this + feels class A. +- But hosts using different dycores might want to interpret the + same `std_name` differently — feels class B. +- **Recommendation**: class A. The mismatch should manifest as + different `std_name`s (`cloud_liquid_dry_mixing_ratio` vs + `cloud_liquid_wet_mixing_ratio`), not the same name with a runtime + override. Need cam-sima input. + +### Q4. After `is_match` relaxation: what happens on disagreement? + +- If two registrations of the same std_name agree on class A but + disagree on class B (e.g., `advected=.false.` from a scheme, + `advected=.true.` from the host), the second registration's class + B values should win without error. Effectively: the host overrides + the scheme. +- Order matters: today the host appends *after* the dynamic + constituents. Should we reverse so the host appends *first*? + Probably not — the "first registration wins on class A; host + setters override class B" model is conceptually clearer. +- **Recommendation**: silently dedup on matching class A; for class + B disagreements, the *later* registration's class B values are + ignored. Hosts use setters to override after registration + finalizes. + +### Q5. Should `%instantiate` accept class-B args at all? + +- **Option Y**: keep `%instantiate` accepting class B args (with + defaults). Schemes can supply them as hints; hosts can override. + Backward-compatible. +- **Option N**: remove class-B args from `%instantiate`. Schemes + *must* leave them to the host. Breaks the 4 cam-sima + scheme-registering schemes. +- **Recommendation**: option Y. The cost of breaking 4 schemes for + marginal clarity isn't worth it. + +### Q6. `ccpp_scheme_utils` singleton + +- Today: `ccpp_initialize_constituent_ptr(const_obj)` stores one + pointer module-wide. First instance wins. +- Fix options: + - (a) Maintain a per-instance pointer table; threading + `instance_number` through `ccpp_constituent_index`. + - (b) Document the limitation, route around it (no scheme uses + `ccpp_constituent_index` under multi-instance — capgen-ng + already enforces `index_of_` everywhere). +- **Recommendation**: (b). It's a one-line doc note and zero code + change. + +--- + +## 8. Three proposals — minimal / clean / deep + +### Proposal A — bugfix only + +**Scope**: +- Land the `ccpt_deallocate` ownership fix (done 2026-05-12). +- Update `scripts/constituents.py` for original capgen's auto-clone + path to pass `owned=.true.` (done). +- Add the three missing setters (`set_advected`, + `set_diagnostic_name`, `set_default_value`) without changing + semantics. Doesn't touch `is_match` or `%instantiate`. +- Document the gaps in `doc/constituents.md`. + +**Cost**: ~50 lines framework code + tests. No cam-sima changes +required. + +**Benefit**: closes the immediate bug, gives hosts the override +mechanism they need today (specifically for `diag_name`), unblocks +the advection test's deferred-property pattern. + +**Limit**: leaves `is_match` strict — hosts that disagree with a +scheme on `advected` still hit the "incompatible constituent" error. + +### Proposal B — class A/B split + setters + +**Scope** (in addition to A): +- Relax `is_match` to check only class A (`units` and possibly + `mixing_ratio_type`). +- Make all class-B properties optional in `%instantiate` with sane + defaults; deprecate (but keep accepting) class-B kwargs. +- Adopt the recommendation in Q4: silently dedup; host setters + override. +- Update `doc/constituents.md` with the register-then-override + workflow. +- (capgen-ng) Reject `diagnostic_name` on `is_constituent=True` + scheme args at parse time, or downgrade it to a default-only hint. +- (capgen-ng) Replace `_FRAMEWORK_CONST_DIM_INPUTS` with a + `ResolvedArg.used_const_dim_std_names` field. + +**Cost**: ~150 lines framework + ~50 lines capgen-ng + tests. +CAM-SIMA host code can stay as-is (the 4 scheme-side registrations +continue to work with their existing class-B values; they're just +not enforced anymore). Optional: tidy the 4 schemes to pass class-A +only. + +**Benefit**: physics schemes become genuinely portable across +hosts. The class-B override pattern that CAM-SIMA already uses for +`thermo_active` and `water_species` generalizes. + +**Limit**: does not change the registration model (still +explicit-only in capgen-ng, still auto-clone in original capgen). + +### Proposal C — host-only registration + +**Scope** (in addition to B): +- Move the 4 cam-sima scheme-side register calls into a CAM-SIMA + helper module called from `cam_comp.F90`'s initialization. +- Drop register-phase `ccpp_constituent_properties_t(:)` support + from capgen-ng (and possibly original capgen). Schemes only + consume constituents; only the host registers. +- Codegen-time enforcement: any `advected=true` scheme arg whose + std_name is not in the host's enumeration → codegen error. +- Eliminates the `_dynamic_constituents` per-suite buffer + entirely. + +**Cost**: ~300 lines code total; requires coordinated PRs across +ccpp-framework, ccpp-capgen, ccpp-capgen-ng, atmospheric_physics, and +CAM-SIMA. The 4 schemes need their `_register` routines deleted (or +made no-ops); the host needs a new helper. + +**Benefit**: one source of truth for what constituents exist +(the host). Removes the auto-clone / scheme-register conceptual +overlap. Simplifies generator and runtime. + +**Limit**: changes the contract for the 4 scheme authors. Risk of +breaking yet-undiscovered downstream users of the scheme-side +registration model. + +### Comparison + +| Aspect | A | B | C | +|---|---|---|---| +| Lines changed | ~50 | ~200 | ~500+ | +| Coordination needed | framework only | framework + capgen-ng | framework + both generators + cam-sima | +| Fixes the crash | yes | yes | yes | +| Fixes `diag_name` portability | yes (host overrides) | yes | yes | +| Relaxes `is_match` | no | yes | yes | +| Removes scheme-side register | no | no | yes | +| Risk to existing CAM-SIMA workflows | none | low | medium | + +### Recommendation + +**Adopt A immediately (mostly done), aim for B over the next 4–6 +weeks, table C until the framework PR for B is in and we have a +clearer signal on whether the scheme-side register pattern is worth +keeping.** + +--- + +## 9. Appendix: framework setter inventory + +(For reference during the meeting. Reproduced from +`design_constituents_mutability.md`.) + +`ccpp_constituent_properties_t` methods (`src/ccpp_constituent_prop_mod.F90`): + +``` +Instantiation + procedure :: instantiate ! takes std_name, long_name, diag_name (REQUIRED), + ! units, vertical_dim, plus optional + ! advected, default_value, min_value, molar_mass, + ! water_species, mixing_ratio_type + procedure :: deallocate + +Getters (subset) + procedure :: standard_name + procedure :: long_name + procedure :: diagnostic_name + procedure :: units + procedure :: vertical_dimension + procedure :: is_advected + procedure :: is_thermo_active + procedure :: is_water_species + procedure :: is_mass_mixing_ratio + procedure :: is_volume_mixing_ratio + procedure :: is_number_concentration + procedure :: is_dry / is_moist / is_wet + procedure :: minimum + procedure :: molar_mass + procedure :: default_value + procedure :: has_default + procedure :: is_framework_owned ! NEW 2026-05-12 + +Setters (changes after instantiate) + procedure :: set_const_index + procedure :: set_thermo_active + procedure :: set_water_species + procedure :: set_minimum + procedure :: set_molar_mass + procedure :: set_framework_owned ! NEW 2026-05-12 + procedure :: set_advected ! GAP + procedure :: set_diagnostic_name ! GAP + procedure :: set_default_value ! GAP (or keep class A) + procedure :: set_mixing_ratio_type ! GAP (if class B) + +Identity / equality + procedure :: equivalent ! full equality + procedure :: is_match ! checks units + (class-B props ← too strict) +``` + +`ccpp_constituent_prop_ptr_t` is the pointer wrapper. Has parallel +setters that delegate to the underlying `ccpp_constituent_properties_t`. + +--- + +## Cross-references + +- `doc/constituents.md` — capgen-ng's user-facing constituents reference. +- `design_constituent_api.md` (memory) — capgen-ng's per-instance option-A design. +- `design_constituents_mutability.md` (memory) — extended design notes incl. class A/B classification. +- `project_implementation_status.md` (memory) — current implementation state and deferred items. +- `scripts/constituents.py` — original capgen's host-cap generator. +- `src/ccpp_constituent_prop_mod.F90` — framework. +- `capgen-ng/generator/host_constituents.py` — capgen-ng's host-side module emitter. +- `capgen-ng/generator/suite_resolver.py` (`_resolve_constituent_arg`) — capgen-ng's resolver routing. +- `EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` — CAM-SIMA's host-side wrappers around framework setters. + diff --git a/doc/migration_20260513T0733.md b/doc/migration_20260513T0733.md new file mode 100644 index 00000000..ce968aee --- /dev/null +++ b/doc/migration_20260513T0733.md @@ -0,0 +1,545 @@ +# Migrating from ccpp-prebuild / ccpp-capgen to capgen-ng + +This document captures the **user-facing differences** a host model author +or scheme author needs to know when moving metadata, suite XML, and host +Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to +**capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and +`doc/redesign_analysis.md` (analysis of the old systems). + +Generated as of 2026-05-13. Current unit-test suite: 1113 passing. + +**Repository layout** (post-2026-05-13 cleanup): tooling lives under +`capgen-ng/` (top-level of this repo). Unit tests live at the top +level in `unit-tests/`; end-to-end tests in `end-to-end-tests/`. Run +the unit suite from the repo root with `python -m pytest unit-tests/`. + +## Table of contents + +1. [Metadata format changes](#1-metadata-format-changes) + 1. [1.8 `horizontal_loop_extent` → `horizontal_dimension`](#18-horizontal_loop_extent--horizontal_dimension) +2. [Suite definition file (SDF) changes](#2-suite-definition-file-sdf-changes) +3. [Host Fortran requirements](#3-host-fortran-requirements) +4. [Generator CLI and build integration](#4-generator-cli-and-build-integration) +5. [Generated cap layout — what's new and what changed](#5-generated-cap-layout--whats-new-and-what-changed) +6. [Framework changes (constituents)](#6-framework-changes-constituents) + 1. [6.3 Host metadata wins over auto-provisioning](#63-host-metadata-wins-over-auto-provisioning-2026-05-12) +7. [Validator (`ccpp_validator.py`)](#7-validator) +8. [Known gaps and deferred items](#8-known-gaps-and-deferred-items) + +--- + +## 1. Metadata format changes + +### 1.1 Table types + +Four `type =` values in `[ccpp-table-properties]`: + +| Type | Contents | +|---------|---------------------------------------------------------| +| `control` | Control variables passed as ``ccpp_physics_*`` args. | +| `host` | Host-model variables imported via `use`. | +| `ddt` | Derived-type definitions. | +| `scheme` | Scheme metadata. | + +The legacy `type = module` (capgen) becomes `type = host`. The legacy +`TYPEDEFS_NEW_METADATA` Python dict (prebuild) is replaced by `type = ddt` +tables. See `doc/redesign_prompt.md` §3.2. + +### 1.2 New table-property attributes + +All optional inside the `[ccpp-table-properties]` block: + +| Attribute | Applies to | Description | +|-----------------------|-----------------------|-------------| +| `module_name` | scheme, host, ddt | Fortran module name; overrides "module name = table name" when they differ. | +| `dependencies` | any | Comma-separated list of file paths to compile. **May appear multiple times** in one block (new this session); single occurrence still accepted. | +| `dependencies_path` | any | Relative base for `dependencies` entries. Single-valued. | +| `source_path` | any | Relative path to the Fortran source. Single-valued. | +| `kind_spec` | any | `:=>spec` (or shorthand). May appear multiple times. | + +Example with multi-line dependencies (real CCPP physics pattern): + +``` +[ccpp-table-properties] + name = GFS_rrtmg_setup + type = scheme + module_name = GFS_rrtmg_setup # optional when names match + dependencies_path = ../../ + dependencies = tools/mpiutil.F90 + dependencies = hooks/machine.F + dependencies = Radiation/RRTMG/radlw_main.F90,Radiation/RRTMG/radsw_main.F90 +``` + +### 1.3 New per-variable attributes + +Inside a `[ var_name ]` section. All optional. + +| Attribute | Type | Default | Notes | +|------------------|------|---------|-------| +| `top_at_one` | bool | `False` | When host and scheme disagree, generator emits a vertical-flip transform with reverse-stride subscript on the host side. Meaningless on variables without a vertical dimension. | +| `constituent` | bool | `False` | Scheme metadata only. Marks the var as a constituent reference. | +| `advected` | bool | `False` | Scheme metadata only. | +| `molar_mass` | float | `0.0` | Scheme metadata only. | +| `diagnostic_name` | str | (defaults to `local_name`) | Host-tooling hint; mutually exclusive with `diagnostic_name_fixed`. | + +### 1.4 Sliced local names with long subscript indices + +Local names with array slices may carry CCPP standard names as subscript +tokens: + +``` +[ dqdt(:,:,index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array) ] + standard_name = ... +``` + +The 63-char Fortran-identifier limit is enforced only on the base +identifier (`dqdt`), not on subscript tokens (which are CCPP standard +names resolved at codegen time and routinely exceed 63 chars). + +### 1.5 Unit strings: bare vs explicit positive exponent + +`m2` and `m+2` (or any `` vs `+` +combo) are normalised internally and treated as equivalent. Pre-existing +unit-conversion entries don't need to be duplicated; either spelling +matches. + +### 1.6 Improved error messages + +- **Duplicate standard name**: error message now lists both colliding + access paths and hints at the "sibling DDT instance" pattern (when + applicable). +- **Subcycle bound unresolved**: error names the std_name and points + at the control/host metadata as the fix. +- **Instance-dim used without `instance_number`**: error explains the + paired-opt-in requirement (see §1.7). + +### 1.7 Optional `instance_number` / `number_of_instances` pair + +These two control variables are now **paired optional**: + +- Declare **both** (`instance_number` in `type=control`, + `number_of_instances` in `type=host`) → multi-instance API. +- Declare **neither** → single-instance API. Public entry points drop + `instance_number`; internal per-instance arrays size to length 1. +- Declare exactly one → hard error from the validator. + +Hosts that don't need multi-instance bookkeeping can drop both declarations. + +### 1.8 `horizontal_loop_extent` → `horizontal_dimension` + +ccpp-prebuild / original ccpp-capgen used `horizontal_loop_extent` as +the horizontal-axis std name in scheme metadata. capgen-ng uses +`horizontal_dimension` uniformly — the run-vs-non-run distinction +isn't expressed in scheme metadata anymore (host passes +`horizontal_loop_begin`/`horizontal_loop_end` as control vars and the +generated cap slices accordingly). + +Migration paths: + +1. **Edit the metadata** (recommended) — search-and-replace + `horizontal_loop_extent` → `horizontal_dimension` in every scheme + `.meta` you maintain. +2. **Use `--legacy-mode`** (transient) — pass `--legacy-mode` to both + `ccpp_capgen_ng.py` and `ccpp_validator.py` and the rename happens + at parse time. A loud warning banner prints at startup so the + rewrite is never invisible. This shim *will be removed*; treat + it as a runway, not a destination. + +--- + +## 2. Suite definition file (SDF) changes + +### 2.1 Schema v2.0 with nested-suite expansion + +Capgen-ng parses v2.0 SDFs and expands `` references +recursively at parse time. See `doc/redesign_prompt.md` §3 and the +`suite_v2_0.xsd` schema. + +### 2.2 `` with CCPP standard-name loop bound + +```xml + + effr_pre + +``` + +The `loop=` attribute accepts: + +- **Integer literal** (`loop="3"`) — emitted verbatim. +- **CCPP standard name** (`loop="num_subcycles_for_effr"`) — resolved + against host/control metadata; supports DDT-component access paths + (e.g. `phys_state%num_subcycles`). +- **Absent / empty** — treated as `loop="1"`. + +The loop-bound standard name is automatically included in the +introspection inputs list (`ccpp_physics_suite_variables` and +`_suite_host_data`). + +### 2.3 Nested `` elements + +```xml + + + effr_calc + + +``` + +Nested subcycles produce nested `do` loops in the generated cap. Loop +counter variables follow the convention: + +- Outermost / single-level: `ccpp_loop_counter`. +- Each deeper level: `ccpp_loop_counter_2`, `ccpp_loop_counter_3`, ... + +Effective iteration count = product of every level's `loop=` value. +`effr_calc` in the example runs 3·2 = 6 times. + +### 2.4 Suite-level `` and `` schemes + +```xml + + my_init_scheme + ... + my_final_scheme + +``` + +- Each element contains a **single** scheme name as text content. + Multiple `` children inside ``/`` is a schema + violation. (Group-shaped lists belong inside ``.) +- The named scheme's `init` / `final` phase metadata is resolved like + any other scheme phase; missing-phase metadata is a generator error. +- The scheme call is emitted inside `_init` / `_final` + with USE for the scheme module + per-arg host modules, and the + standard errflg check. +- Call ordering: + - `_init`: after all group `state_alloc` and + `suite_data_init_fields`, **before** the `CCPP_SUITE_FRAMEWORK_INITIALIZED` + state transition. + - `_final`: before the `CCPP_SUITE_UNREGISTERED` transition. + +**Accepted spellings**: `` and `` only. Legacy spellings +**``** (typo), **``** (correct long form), and +**``** are rejected with a clear error pointing at the +canonical short form. + +To exercise: + +1. Declare a scheme with `init` and/or `final` phases in its metadata + (minimal sig — just `errmsg` + `errflg` — is fine). +2. Reference it in the SDF as shown above. +3. Add the scheme's `.F90` to your build's source list. + +--- + +## 3. Host Fortran requirements + +### 3.1 Required control variables + +Every host's `type=control` table must declare: + +| Standard name | Fortran type | Purpose | +|-----------------------------------|--------------|-----------------------------------| +| `suite_name` | character | Drives suite dispatch | +| `horizontal_loop_begin` | integer | Lower chunk-bound | +| `horizontal_loop_end` | integer | Upper chunk-bound | +| `thread_number` | integer | Current thread | +| `number_of_threads` | integer | Total threads | +| `number_of_physics_threads` | integer | Physics-internal budget | +| `ccpp_error_code` | integer | Error flag | +| `ccpp_error_message` | character | Error message | + +Optional (paired — see §1.7): + +| Standard name | Fortran type | Table type | Purpose | +|-------------------------|--------------|------------|--------------------------------| +| `instance_number` | integer | control | Current instance index | +| `number_of_instances` | integer | host | Total instance count | + +### 3.2 Required entry-point call sequence + +``` +ccpp_register(suite_name, errflg, errmsg, [instance_number]) + └── per scheme that declares a register phase +ccpp_init(suite_name, errflg, errmsg, [instance_number]) + └── per scheme that declares an init phase +ccpp_physics_init(...) + └── physics phase routines per group: + ccpp_physics_init + ccpp_physics_timestep_init + ccpp_physics_run ← run-loop phase + ccpp_physics_timestep_final + ccpp_physics_final +ccpp_final(suite_name, errflg, errmsg, [instance_number]) +``` + +`instance_number` appears in every signature only when the host +declares the `instance_number` / `number_of_instances` pair (§1.7). + +### 3.3 Host module convention + +The Fortran module that exports a host metadata table's variables is +typically named after the table. When that's not the case, use the +`module_name` table-property override (§1.2): + +``` +[ccpp-table-properties] + name = test_host_data + type = host + module_name = mod_test_host_data +``` + +--- + +## 4. Generator CLI and build integration + +### 4.1 `ccpp_capgen_ng.py` invocation + +``` +python ccpp_capgen_ng.py \ + --host-files [,,...] \ + --scheme-files [,,...] \ + --suites [,,...] \ + --host-name \ + --output-root /ccpp \ + [--kind-type =[:]] \ + [--legacy-mode] \ + [--verbose] [--verbose] +``` + +`--kind-type` syntax: `=[:]`. When `:` is +omitted, `` must be an ISO_FORTRAN_ENV constant (REAL32/REAL64/...) +and the module defaults to `iso_fortran_env`. `kind_phys` is +auto-defaulted to `iso_fortran_env:REAL64` when not supplied. + +`--legacy-mode` (transient migration shim, will be removed): silently +rewrites legacy CCPP standard names that ccpp-prebuild / original +ccpp-capgen used to their capgen-ng equivalents at parse time. +Currently translates `horizontal_loop_extent` → `horizontal_dimension`. +Prints a loud warning banner at startup so the rewrite is never +invisible. Available on both `ccpp_capgen_ng.py` and `ccpp_validator.py` +(keep the flag consistent between the two when both are invoked from +CMake). All translation logic is isolated in +`metadata/legacy_compat.py` and tagged with `# legacy-compat:` comments +at every touchpoint, so the shim can be cleanly removed when migration +is complete. + +### 4.2 `ccpp_datafile.py` query CLI + +Generated `datatable.xml` carries: + +- `` — generated outputs (utilities/host_files/suite_files). +- `` — `.meta` and expanded SDF. +- `` — per-scheme call lists. +- `` — host/api/suite/group dictionaries. + +Query via `ccpp_datafile.py -- `. Flags include +`--dependencies`, `--capgen-files`, `--host-files`, `--utility-files`, +`--suite-list`, `--required-variables `, `--input-variables `, +`--output-variables `, `--host-variables`, `--show`. + +### 4.3 CMake helpers + +`cmake/ccpp_capgen.cmake` and `cmake/ccpp_validator.cmake` provide the +`ccpp_capgen(...)` and `ccpp_validator(...)` macros. `ccpp_datafile(...)` +queries datatable.xml at configure time. + +--- + +## 5. Generated cap layout — what's new and what changed + +### 5.1 Output files + +Always generated: + +- `ccpp_kinds.F90` — kind parameters. Listed under ``. +- `ccpp_static_api.F90` — public host-facing entry points + introspection routines. +- `ccpp__cap.F90` — per-suite dispatcher. +- `ccpp___cap.F90` — per-group phase implementations. +- `ccpp__data.F90` — suite-owned interstitial DDT + module-level array. +- `ccpp__types.F90` — pointer-wrapper types for optional args. +- `ccpp_.meta` — inspection artifact; matches the generated cap. +- `datatable.xml` — build-system + host-introspection metadata. + +When any scheme registers constituents: + +- `ccpp_host_constituents.F90` — owns `ccpp_model_constituents_obj(:)` + and the host-facing constituent API. + +### 5.2 Per-suite data: TARGET on the instance array + +`ccpp_suite_data(:)` carries the `TARGET` attribute: + +```fortran +type(ccpp__data_t), allocatable, target, public :: ccpp_suite_data(:) +``` + +This makes every `ccpp_suite_data(i)%component(...)` subobject a valid +pointer-assignment target — needed for transformation temps and +optional-arg pointer wrappers. + +### 5.3 Variable transformations + +The generator emits three kinds of transform on a per-arg basis: + +| Transform | Trigger | +|------------------|--------------------------------------------------| +| Unit conversion | `host.units != scheme.units` with a registered conversion entry. | +| Kind conversion | `host.kind != scheme.kind` (different strings). | +| Vertical flip | `host.top_at_one != scheme.top_at_one` on a var with a vertical dim. | + +These compose. A scheme arg that needs unit + flip emits a single +combined assignment through a transformation temp: + +```fortran +temp_l = 1.0E-3_kind_phys*host_var(lb:ub, nlev:1:-1) ! unit conversion: kind_phys to kind_phys; vertical flip (top_at_one mismatch) +call scheme_run(temp=temp_l, ...) +host_var(lb:ub, nlev:1:-1) = 1.0E+3_kind_phys*temp_l ! ... reverse ... +``` + +Identity unit conversions (registered for dimensionally-equivalent +spellings like `J kg-1 ↔ m2 s-2`, formula `'{var}'`) are not labelled +"unit conversion" in the comment. + +### 5.4 Subcycle emission + +```fortran +integer :: ccpp_loop_counter +integer :: ccpp_loop_counter_2 +... +do ccpp_loop_counter = 1, phys_state%num_subcycles ! outer + call scheme_pre(...) + do ccpp_loop_counter_2 = 1, 2 ! inner + call scheme_calc(...) + end do +end do +``` + +### 5.5 State machine + +Per-instance integer state arrays: + +- `ccpp_suite_state(:)` — suite-level (UNREGISTERED / REGISTERED / + FRAMEWORK_INITIALIZED). +- `ccpp_group_state(:)` — group-level (UNINITIALIZED / INITIALIZED / + IN_TIMESTEP). + +Single-instance hosts get length-1 arrays indexed with literal `1`. +See `doc/redesign_prompt.md` §7. + +--- + +## 6. Framework changes (constituents) + +### 6.1 `ccpp_constituent_prop_mod` ownership flag + +(Framework PR — needs upstream merge.) Adds: + +- `framework_owns_me` private flag on `ccpp_constituent_properties_t`, + default `.false.`. +- `set_framework_owned(value)` setter (call before + `obj%new_field(const_prop, ...)` when transferring ownership). +- `is_framework_owned()` getter. +- `ccpt_deallocate` only frees when the flag is set; otherwise just + nullifies. + +Backward-compatible. Original capgen's auto-clone path in +`scripts/constituents.py` has been updated to call the setter. + +### 6.2 capgen-ng constituent API + +(See `doc/constituents.md` for the full reference.) Highlights: + +- One `ccpp_model_constituents_obj(:)` array per generator invocation, + sized to `number_of_instances`. +- Host-facing API: + - `ccpp_register_constituents(host_constituents, instance_number, ...)` + - `ccpp_initialize_constituents(ncols, num_layers, instance_number, ...)` + - `ccpp_const_get_index(stdname, const_index, instance_number, ...)` + - `ccpp_constituents_array(instance_number) → pointer` + - `ccpp_advected_constituents_array(instance_number) → pointer` + - `ccpp_model_const_properties(instance_number) → pointer` + - `ccpp_number_constituents(num_flds, advected, instance_number, ...)` + - `ccpp_gather_constituents`, `ccpp_update_constituents` + - `ccpp_is_scheme_constituent(var_name, ...)` (not per-instance) +- Scheme-side registration: four rules — register-phase + `ccpp_constituent_properties_t(:)` arg, consume base via + `advected=true intent=in/inout`, produce tendency via + `constituent=true intent=out` + `tendency_of_` std name, mismatches + are codegen errors. + +### 6.3 Host metadata wins over auto-provisioning (2026-05-12) + +If the host declares a framework-named standard name +(`ccpp_constituents` / `ccpp_constituent_tendencies` / +`ccpp_constituent_properties` / `number_of_ccpp_constituents` / +`index_of_`) as a regular host variable, the resolver uses the +host's declaration and skips capgen-ng auto-provisioning. Matters +most for legacy hosts (GFS / SCM) that own their own tracer +indices — e.g. `[ntcw]` with `standard_name = +index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array` +resolves to the host's short local name `ntcw`, not a parallel +module-level integer named after the full standard name (which +would also blow Fortran's 63-char identifier limit). See +`doc/constituents.md` §3. + +Active design review for the next constituents iteration: +`doc/constituents_overhaul.md` (Class A vs Class B property +classification, three reform proposals). + +--- + +## 7. Validator + +`capgen-ng/ccpp_validator.py` — standalone Fortran-vs-metadata checker. +Today validates **scheme** metadata against scheme Fortran files +(subroutine signatures, optional args, paren-aware decl splitting). + +Continuation-line handling covers both free-form (`&` at trailing end +of prior line only) and fixed-form / dual-form (`&` at both ends, with +the leading marker at column 6). Comment-only and blank lines +interleaved between continuation lines are skipped as Fortran 90+ +permits. + +When the signature parser finds a subroutine but extracts zero args +while metadata declares many, the "Argument count mismatch" error +appends a HINT pointing at the parser rather than masquerading as a +real mismatch — common cause is an unsupported signature feature. + +**Known gap**: host-metadata validation is not yet implemented. When +invoked with non-scheme `.meta` files, the validator silently filters +to zero schemes and reports "Validation passed." Slated for revisit +after the e2e test suite settles (`unit_conv` + `variable_transform` +complete). See `project_validator_host_check_deferred.md` (memory). + +--- + +## 8. Known gaps and deferred items + +| Item | Status | +|--------------------------------------------|-----------------------------------------------| +| `ccpp_loop_counter` standard name inside nested subcycles | Maps to OUTERMOST loop var. None of cam-sima uses this; revisit if a scheme needs the innermost value. | +| Validator host-metadata check | Deferred; revisit after e2e tests stabilise. | +| Constituents overhaul (Class A/B + setters) | Discussion doc at `doc/constituents_overhaul.md`. | +| Framework setters: `set_advected`, `set_diagnostic_name`, `set_default_value` | Deferred; depends on constituents-overhaul decision. | +| Codegen-time scheme-registration cross-check | Deferred; would require new `registers_std_names` metadata attr. | +| `_FRAMEWORK_CONST_DIM_INPUTS` cleanup | **Done 2026-05-13**: hand-curated frozenset gone; framework-constituent dim refs ride on a dedicated `used_const_dim_std_names` field on `ResolvedArg`. | +| Suppress `ccpp_host_constituents.F90` when unused | Deferred; currently emitted for every build even when no scheme/host actually exercises the constituent system. Now *correct* (empty) for SCM-style hosts thanks to the host-wins rule, but still dead code. See `design_constituent_host_wins.md`. | +| Python linter / formatter pass | Deferred; pick `ruff` and apply across `capgen-ng/`. | +| Generated Fortran ↔ Codee formatter idempotency | Deferred; emitted `.F90` must round-trip cleanly through the project's Codee Fortran formatter. | +| `fortran_to_metadata` developer utility | Deferred; bootstraps a `.meta` skeleton from an existing `.F90` subroutine. | +| `--legacy-mode` shim removal | Transient; remove `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and every `# legacy-compat:` touchpoint when scheme metadata has migrated. | +| Original capgen auto-clone path | Intentionally dropped in favour of explicit registration; kept in memory as "Option B" fallback. | + +--- + +## Cross-references + +- `doc/redesign_prompt.md` — original design specification (sections + marked "historic" where the implementation has evolved). +- `doc/redesign_analysis.md` — analysis of the legacy ccpp-prebuild + + ccpp-capgen toolchains. +- `doc/constituents.md` — full constituents reference for capgen-ng. +- `doc/constituents_overhaul.md` — architecture review and reform + proposals for the next iteration. + diff --git a/doc/redesign_analysis.md - updated 20260513T0733.md b/doc/redesign_analysis_20260513T0733.md similarity index 100% rename from doc/redesign_analysis.md - updated 20260513T0733.md rename to doc/redesign_analysis_20260513T0733.md diff --git a/doc/redesign_analysis - original 202060505T2044.md b/doc/redesign_analysis_original_202060505T2044.md similarity index 100% rename from doc/redesign_analysis - original 202060505T2044.md rename to doc/redesign_analysis_original_202060505T2044.md diff --git a/doc/redesign_prompt.md - updated 20260513T0733.md b/doc/redesign_prompt_20260513T0733.md similarity index 100% rename from doc/redesign_prompt.md - updated 20260513T0733.md rename to doc/redesign_prompt_20260513T0733.md diff --git a/doc/redesign_prompt - original 20260505T2044.md b/doc/redesign_prompt_original_20260505T2044.md similarity index 100% rename from doc/redesign_prompt - original 20260505T2044.md rename to doc/redesign_prompt_original_20260505T2044.md From cfd4bf744b6d2b35fb2d16194142db77f2aa6363 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 11:54:35 -0600 Subject: [PATCH 09/74] Update doc --- doc/constituents.md | 2 +- doc/constituents_20260513T0733.md | 2 +- doc/constituents_overhaul.md | 29 ++-- doc/constituents_overhaul_20260513T0733.md | 29 ++-- doc/migration.md | 25 +++- doc/migration_20260513T0733.md | 2 +- doc/redesign_prompt.md | 150 ++++++++++++++------- doc/redesign_prompt_20260513T0733.md | 150 ++++++++++++++------- 8 files changed, 269 insertions(+), 120 deletions(-) diff --git a/doc/constituents.md b/doc/constituents.md index 8e5a7b77..c7581801 100644 --- a/doc/constituents.md +++ b/doc/constituents.md @@ -1,6 +1,6 @@ # CCPP capgen-ng — Constituents Reference -*Last revised: 2026-05-11.* +*Last revised: 2026-05-13.* This document is the authoritative reference for **constituent variables** in capgen-ng — what they are, how scheme authors declare them in metadata, what diff --git a/doc/constituents_20260513T0733.md b/doc/constituents_20260513T0733.md index 8e5a7b77..c7581801 100644 --- a/doc/constituents_20260513T0733.md +++ b/doc/constituents_20260513T0733.md @@ -1,6 +1,6 @@ # CCPP capgen-ng — Constituents Reference -*Last revised: 2026-05-11.* +*Last revised: 2026-05-13.* This document is the authoritative reference for **constituent variables** in capgen-ng — what they are, how scheme authors declare them in metadata, what diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index 7d177cae..2adb34dd 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -2,8 +2,14 @@ **Authors:** Dom Heinzeller (lead), Claude (assistant) **Date drafted:** 2026-05-12 +**Last revised:** 2026-05-13 **Intended audience:** CCPP framework team, CAM-SIMA team -**Status:** Discussion document — no decisions are final. +**Status:** Discussion document — no decisions are final. Proposals +A/B/C below remain pending the upcoming meeting; the bug fix from +Proposal A (the `ccpt_deallocate` ownership flag) and the capgen-ng +internal cleanup from Proposal B (§4.8) have landed; the missing +setters from Proposal A and the `is_match` relaxation from Proposal B +have not. --- @@ -361,8 +367,9 @@ intentionally left for this discussion. updated to call `set_framework_owned(.true.)` after `allocate`. Diffs in `src/ccpp_constituent_prop_mod.F90` (and capgen-ng's parallel copy) + `scripts/constituents.py`. -- **Status**: framework tests pass, capgen-ng tests pass (954). Needs - upstream PR to ccpp-framework + ccpp-capgen. +- **Status**: framework tests pass; capgen-ng unit-test suite (1127 passing + as of 2026-05-13) is green. Still needs upstream PR to ccpp-framework + + original ccpp-capgen. ### 4.2 Framework: missing setters (OPEN) @@ -446,12 +453,12 @@ The synthetic scope between suite and host serves correctness but adds a code path that most contributors don't read. If we drop it (capgen-ng has), the variable-matching algorithm shrinks. -### 4.8 Capgen-ng: `_FRAMEWORK_CONST_DIM_INPUTS` is hand-curated (OPEN) +### 4.8 Capgen-ng: `_FRAMEWORK_CONST_DIM_INPUTS` cleanup (LANDED 2026-05-13) -`generator/static_api.py` carries a frozenset of standard names -(currently just `number_of_ccpp_constituents`) that introspection -treats specially. Cleaner: a dedicated `used_const_dim_std_names` -field on `ResolvedArg`. Marked REVISIT in the code. +`generator/static_api.py` no longer carries the hand-curated frozenset of +standard names; framework-constituent dimension references now ride on a +dedicated `used_const_dim_std_names` field on `ResolvedArg`. Closes the +"hand-curated → structured field" REVISIT note that was in the code. ### 4.9 Capgen-ng: no codegen-time cross-check of scheme registration (OPEN) @@ -571,7 +578,7 @@ constituent property is conceptually owned by either the scheme - **Document the lifecycle** clearly. `doc/constituents.md` is ~960 lines; targeted additions for "register-then-override" workflow once the new setters land. -- **Capgen-ng-internal cleanup**: replace +- **Capgen-ng-internal cleanup** (LANDED 2026-05-13): replaced `_FRAMEWORK_CONST_DIM_INPUTS` with a `used_const_dim_std_names` field on `ResolvedArg`. @@ -696,8 +703,8 @@ scheme on `advected` still hit the "incompatible constituent" error. workflow. - (capgen-ng) Reject `diagnostic_name` on `is_constituent=True` scheme args at parse time, or downgrade it to a default-only hint. -- (capgen-ng) Replace `_FRAMEWORK_CONST_DIM_INPUTS` with a - `ResolvedArg.used_const_dim_std_names` field. +- (capgen-ng) **DONE 2026-05-13**: replaced `_FRAMEWORK_CONST_DIM_INPUTS` + with a `ResolvedArg.used_const_dim_std_names` field. **Cost**: ~150 lines framework + ~50 lines capgen-ng + tests. CAM-SIMA host code can stay as-is (the 4 scheme-side registrations diff --git a/doc/constituents_overhaul_20260513T0733.md b/doc/constituents_overhaul_20260513T0733.md index 7d177cae..2adb34dd 100644 --- a/doc/constituents_overhaul_20260513T0733.md +++ b/doc/constituents_overhaul_20260513T0733.md @@ -2,8 +2,14 @@ **Authors:** Dom Heinzeller (lead), Claude (assistant) **Date drafted:** 2026-05-12 +**Last revised:** 2026-05-13 **Intended audience:** CCPP framework team, CAM-SIMA team -**Status:** Discussion document — no decisions are final. +**Status:** Discussion document — no decisions are final. Proposals +A/B/C below remain pending the upcoming meeting; the bug fix from +Proposal A (the `ccpt_deallocate` ownership flag) and the capgen-ng +internal cleanup from Proposal B (§4.8) have landed; the missing +setters from Proposal A and the `is_match` relaxation from Proposal B +have not. --- @@ -361,8 +367,9 @@ intentionally left for this discussion. updated to call `set_framework_owned(.true.)` after `allocate`. Diffs in `src/ccpp_constituent_prop_mod.F90` (and capgen-ng's parallel copy) + `scripts/constituents.py`. -- **Status**: framework tests pass, capgen-ng tests pass (954). Needs - upstream PR to ccpp-framework + ccpp-capgen. +- **Status**: framework tests pass; capgen-ng unit-test suite (1127 passing + as of 2026-05-13) is green. Still needs upstream PR to ccpp-framework + + original ccpp-capgen. ### 4.2 Framework: missing setters (OPEN) @@ -446,12 +453,12 @@ The synthetic scope between suite and host serves correctness but adds a code path that most contributors don't read. If we drop it (capgen-ng has), the variable-matching algorithm shrinks. -### 4.8 Capgen-ng: `_FRAMEWORK_CONST_DIM_INPUTS` is hand-curated (OPEN) +### 4.8 Capgen-ng: `_FRAMEWORK_CONST_DIM_INPUTS` cleanup (LANDED 2026-05-13) -`generator/static_api.py` carries a frozenset of standard names -(currently just `number_of_ccpp_constituents`) that introspection -treats specially. Cleaner: a dedicated `used_const_dim_std_names` -field on `ResolvedArg`. Marked REVISIT in the code. +`generator/static_api.py` no longer carries the hand-curated frozenset of +standard names; framework-constituent dimension references now ride on a +dedicated `used_const_dim_std_names` field on `ResolvedArg`. Closes the +"hand-curated → structured field" REVISIT note that was in the code. ### 4.9 Capgen-ng: no codegen-time cross-check of scheme registration (OPEN) @@ -571,7 +578,7 @@ constituent property is conceptually owned by either the scheme - **Document the lifecycle** clearly. `doc/constituents.md` is ~960 lines; targeted additions for "register-then-override" workflow once the new setters land. -- **Capgen-ng-internal cleanup**: replace +- **Capgen-ng-internal cleanup** (LANDED 2026-05-13): replaced `_FRAMEWORK_CONST_DIM_INPUTS` with a `used_const_dim_std_names` field on `ResolvedArg`. @@ -696,8 +703,8 @@ scheme on `advected` still hit the "incompatible constituent" error. workflow. - (capgen-ng) Reject `diagnostic_name` on `is_constituent=True` scheme args at parse time, or downgrade it to a default-only hint. -- (capgen-ng) Replace `_FRAMEWORK_CONST_DIM_INPUTS` with a - `ResolvedArg.used_const_dim_std_names` field. +- (capgen-ng) **DONE 2026-05-13**: replaced `_FRAMEWORK_CONST_DIM_INPUTS` + with a `ResolvedArg.used_const_dim_std_names` field. **Cost**: ~150 lines framework + ~50 lines capgen-ng + tests. CAM-SIMA host code can stay as-is (the 4 scheme-side registrations diff --git a/doc/migration.md b/doc/migration.md index ce968aee..58c951f6 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -6,7 +6,7 @@ Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to **capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and `doc/redesign_analysis.md` (analysis of the old systems). -Generated as of 2026-05-13. Current unit-test suite: 1113 passing. +*Last revised: 2026-05-13.* Current unit-test suite: 1127 passing. **Repository layout** (post-2026-05-13 cleanup): tooling lives under `capgen-ng/` (top-level of this repo). Unit tests live at the top @@ -330,14 +330,32 @@ Generated `datatable.xml` carries: - `` — generated outputs (utilities/host_files/suite_files). - `` — `.meta` and expanded SDF. -- `` — per-scheme call lists. +- `` — per-scheme call lists, **scoped to schemes that are + actually referenced by the loaded suites** (group phase calls + the + suite-level ``/`` hooks). Scheme metadata files passed + on the CLI but never referenced are silently dropped. +- `` — `dependencies = …` from host/control/ddt tables + (always) plus the same per-scheme list as `` (filtered to + the used set). Build systems that compile against + `ccpp_datafile.py --dependencies` therefore only pull in scheme deps + for compiled schemes; missing transitive deps in scheme metadata + surface as link errors and should be fixed in the `.meta` file. - `` — host/api/suite/group dictionaries. Query via `ccpp_datafile.py -- `. Flags include `--dependencies`, `--capgen-files`, `--host-files`, `--utility-files`, -`--suite-list`, `--required-variables `, `--input-variables `, +`--suite-files`, `--scheme-files`, `--suite-list`, +`--required-variables `, `--input-variables `, `--output-variables `, `--host-variables`, `--show`. +`--suite-files` returns capgen-generated cap files (`ccpp__cap.F90`, +etc.). `--scheme-files` returns the **user-supplied scheme `.F90` sources** +that the loaded suites actually reference — the filtered compile manifest. +Each used scheme's source is resolved as `/.` +(extension preference order: `.F90`, `.f90`, `.F`, `.f`); missing files are +warned about and the canonical `.F90` guess is emitted so the build-system +query stays useful. + ### 4.3 CMake helpers `cmake/ccpp_capgen.cmake` and `cmake/ccpp_validator.cmake` provide the @@ -529,6 +547,7 @@ complete). See `project_validator_host_check_deferred.md` (memory). | Generated Fortran ↔ Codee formatter idempotency | Deferred; emitted `.F90` must round-trip cleanly through the project's Codee Fortran formatter. | | `fortran_to_metadata` developer utility | Deferred; bootstraps a `.meta` skeleton from an existing `.F90` subroutine. | | `--legacy-mode` shim removal | Transient; remove `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and every `# legacy-compat:` touchpoint when scheme metadata has migrated. | +| `ccpp_datafile.py` query CLI rework | Deferred (2026-05-13); collapse `--host-files` / `--suite-files` / `--utility-files` into `--capgen-files`, then repurpose `--host-files` as a filtered list of **input** host metadata files (parallel to `--scheme-files`). Most hosts pack all host data into a handful of shared files, so the filtering pay-off is small — the draw is API symmetry. | | Original capgen auto-clone path | Intentionally dropped in favour of explicit registration; kept in memory as "Option B" fallback. | --- diff --git a/doc/migration_20260513T0733.md b/doc/migration_20260513T0733.md index ce968aee..a2e631df 100644 --- a/doc/migration_20260513T0733.md +++ b/doc/migration_20260513T0733.md @@ -6,7 +6,7 @@ Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to **capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and `doc/redesign_analysis.md` (analysis of the old systems). -Generated as of 2026-05-13. Current unit-test suite: 1113 passing. +*Last revised: 2026-05-13.* Current unit-test suite: 1127 passing. **Repository layout** (post-2026-05-13 cleanup): tooling lives under `capgen-ng/` (top-level of this repo). Unit tests live at the top diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index c8e1ab3b..0ac76cde 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -1,5 +1,7 @@ # CCPP Framework Code Generator — Redesign Specification +*Last revised: 2026-05-13.* + ## Purpose This document is a complete implementation specification for a new CCPP Framework code @@ -7,6 +9,11 @@ generator (`ccpp-capgen-ng`). It supersedes both `ccpp-prebuild` and `ccpp-capge implementer should be able to build the new generator from scratch using this document alone, supplemented by the real-world examples in `redesign_analysis.md`. +The spec is essentially as-implemented as of the date above. User-facing +deltas relative to ccpp-prebuild and the original ccpp-capgen are +collected in `doc/migration.md`; section 18 of this document is a rolling +"outstanding work" tracker. + --- ## 1. Background and Motivation @@ -235,7 +242,12 @@ character arguments. | `number_of_physics_threads` | `integer` | Thread budget for physics-internal OpenMP; pass `1` if none | | `ccpp_error_message` | `character` | Error message string | | `ccpp_error_code` | `integer` | Integer error return code | -| `instance_number` | `integer` | Current model instance index; pass `1` if single-instance | + +`instance_number` is **paired-optional** with `number_of_instances` (host +table, §3.6): declare both for a multi-instance API, declare neither for a +single-instance API. Declaring exactly one is a hard error. When the pair +is absent, the static API signatures drop the `instance_number` argument +entirely and per-instance state arrays size to 1. `group_name` is **not** in the required set. It is included in the static API signature only if the host declares it in their `type=control` table. When absent: the static API @@ -290,25 +302,31 @@ Eight entry points are generated in the static API. Two tiers: ### 5.1 Framework lifecycle (no group_name dispatch) -These operate on the entire suite at once. They take `suite_name` plus -`ccpp_error_message` and `ccpp_error_code`. No scheme `_run/_init/_final` calls. +These operate on the entire suite at once. They take `suite_name`, +`ccpp_error_code`, and `ccpp_error_message` (plus `instance_number` when the +host opts into the multi-instance pair, §4.1). No scheme `_run/_init/_final` +calls. | Entry point | Purpose | |---|---| -| `ccpp_register(suite_name, constituents, errmsg, errcode)` | Calls each scheme's `_register` entrypoint; allocates and populates the `ccpp_model_constituents_t` object passed by the host | -| `ccpp_init(suite_name, [number_of_instances,] errmsg, errcode)` | Allocates integer state arrays in all group caps; allocates suite-owned interstitial data; no scheme calls | -| `ccpp_final(suite_name, errmsg, errcode)` | Deallocates integer state arrays and suite-owned data; no scheme calls | - -`constituents` in `ccpp_register` is `intent(inout)`: unallocated on entry, allocated -and populated on exit. The host declares and owns this object (imports -`ccpp_model_constituents_t` from the framework library). The argument is mandatory. - -`number_of_instances` in `ccpp_init` is **conditional**: included as an explicit -`intent(in)` integer argument when the host metadata declares a variable with standard -name `number_of_instances`; omitted entirely for single-instance models. The generator -detects this automatically at parse time (same conditionality rule as `instance_number`). -When present it is passed through the call chain: -`ccpp_init` → `_init` → each group's `state_alloc`. +| `ccpp_register(suite_name, errcode, errmsg, [instance_number])` | Calls each scheme's `_register` entrypoint; transitions suite state to `REGISTERED`. Auto-provisions `ccpp_model_constituents_obj(:)` and friends in `ccpp_host_constituents.F90` when any register-phase scheme declares `ccpp_constituent_properties_t(:)` (constituents are not a formal arg). | +| `ccpp_init(suite_name, errcode, errmsg, [instance_number])` | Allocates integer state arrays in all group caps; allocates suite-owned interstitial data; calls the suite-level `` scheme if declared (§5.5); no per-group scheme calls. | +| `ccpp_final(suite_name, errcode, errmsg, [instance_number])` | Calls the suite-level `` scheme if declared (§5.5); deallocates integer state arrays and suite-owned data; no per-group scheme calls. | + +Constituents are **opt-in**: a separate generated module +`ccpp_host_constituents.F90` declares `ccpp_model_constituents_obj(:)` and a +host-facing API (`ccpp_register_constituents`, `ccpp_initialize_constituents`, +`ccpp_const_get_index`, `ccpp_constituents_array(instance_number)`, +`ccpp_advected_constituents_array`, `ccpp_model_const_properties`, +`ccpp_number_constituents`, `ccpp_gather_constituents`, +`ccpp_update_constituents`, `ccpp_is_scheme_constituent`). The host calls +these directly — they are not formal arguments of `ccpp_register` / `ccpp_init`. +See `doc/constituents.md`. + +`instance_number` appears in every framework-lifecycle signature only when +the host declares the `instance_number` / `number_of_instances` pair (§4.1). +When present it propagates: `ccpp_init` → `_init` → each group's +`state_alloc(number_of_instances, ...)`. ### 5.2 Physics group invocation (dispatched by suite_name + group_name) @@ -373,26 +391,29 @@ Constraints: - The named scheme must have the matching phase in its metadata. Missing-phase metadata is a generator error. -### 5.4 Suite introspection routines (planned) +### 5.4 Suite introspection routines -In addition to the eight entry points above, the static API will expose four -**suite-introspection** subroutines that let a host query, at runtime, what is +In addition to the eight entry points above, the static API exposes **five** +suite-introspection subroutines that let a host query, at runtime, what is compiled into the API. These mirror the equivalent routines in the original -capgen (`scripts/ccpp_suite.py` — `write_inspection_routines`) and are required -for CMake integration and host-side build glue. They are **planned but not yet -implemented**; signatures below are the proposed shape and may be refined when -the original capgen sources are reviewed for adoption. +capgen (`scripts/ccpp_suite.py` — `write_inspection_routines`) and are +used by CMake integration and host-side build glue. | Entry point | Purpose | |---|---| | `ccpp_physics_suite_list(suites)` | Return all suite names compiled into the API | -| `ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errcode)` | Return the list of group ("part") names for a given suite | -| `ccpp_physics_suite_variables(suite_name, variable_list, errmsg, errcode, [input_vars], [output_vars], [struct_elements])` | Return the standard-name list a suite consumes/produces; optional flags filter by intent and whether DDT sub-fields are flattened | -| `ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errcode)` | Return the list of scheme module names that compose a suite | +| `ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errflg)` | Return the list of group ("part") names for a given suite | +| `ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errflg)` | Return the list of scheme module names that compose a suite | +| `ccpp_physics_suite_variables(suite_name, variable_list, errmsg, errflg, [input_vars], [output_vars], [struct_elements])` | Standard-name list a suite consumes/produces; optional flags filter by intent and whether DDT sub-fields are flattened | +| `ccpp_physics_suite_host_data(suite_name, variable_list, errmsg, errflg)` | Standard-name list of host data the suite reads — DDT-collapsed view, excludes generated control variables | These routines do not advance the state machine and do not call any scheme entrypoints. All inputs derive from generator-time data already held in `SuiteResolution` plus the host/scheme metadata; no new metadata is required. +The `_variables` vs `_host_data` split distinguishes the flat-leaf view +(every DDT field that is actually consumed) from the DDT-collapsed view +(parent DDT instances), and excludes capgen-ng-generated control +variables from `_host_data` since the host owns those. --- @@ -408,7 +429,8 @@ All three levels are fully auto-generated. No hand-written components in the cap USEd only by files that declare kind-typed variables: group caps, the suite types module, and the suite data module. - Dispatches all eight entry points by `suite_name` to the appropriate suite cap -- Passes `ccpp_model_constituents_t` through as an explicit argument (does not own it) +- Does not own constituent state; constituents are accessed via the separate + `ccpp_host_constituents.F90` module by both the host and group caps - Holds no physics state ### 6.2 Suite cap (`ccpp__cap.F90`) @@ -1105,10 +1127,9 @@ The following patterns from prebuild or capgen are explicitly **not** carried fo ## 18. Outstanding Work See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` -(deferred items) for the canonical list. Snapshot as of 2026-05-12 -(end of session): +(deferred items) for the canonical list. Snapshot as of 2026-05-13: -### Landed this session +### Landed in the 2026-05-12 session - **`instance_number` / `number_of_instances` paired opt-in** — hosts may omit both for a single-instance API. @@ -1128,8 +1149,9 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` - **TARGET on `ccpp_suite_data(:)`** module-level array. - **Group-state alloc idempotency** (matches suite-state alloc). - **Framework PR**: `ccpt_deallocate` ownership tracking via - `framework_owns_me` flag. Backward-compatible. Needs upstream - merge to ccpp-framework + ccpp-capgen. + `framework_owns_me` flag. Backward-compatible. Landed in + capgen-ng's vendored framework copy; still needs upstream merge + to ccpp-framework + original ccpp-capgen. - **Identity unit conversions** no longer emit misleading "unit conversion: kind_phys to kind_phys" comment. - **Improved duplicate-standard-name error** lists both colliding @@ -1138,12 +1160,38 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` scheme's init/final phase emitted inside `_init` / `_final`. Single scheme only; long-form spellings (``, ``, ``) rejected. -- **Test count**: 1070 passing. +- **Constituent resolver — host metadata wins**: hosts that declare + framework-named std_names (`ccpp_constituents`, `index_of_`, ...) + short-circuit capgen-ng's auto-provisioning so legacy hosts (GFS, + SCM) keep using their own short local names (e.g. `ntcw`) without + blowing Fortran's 63-char identifier limit. + +### Landed 2026-05-13 + +- **`--legacy-mode` shim** — transient parse-time rewrite of legacy + CCPP standard names (`horizontal_loop_extent` → + `horizontal_dimension`). Available on `ccpp_capgen_ng.py` and + `ccpp_validator.py`; loud banner at startup. Isolated in + `metadata/legacy_compat.py` and tagged `# legacy-compat:` for clean + removal once scheme metadata has been migrated. +- **`_FRAMEWORK_CONST_DIM_INPUTS` cleanup** — the hand-curated + frozenset in `generator/static_api.py` was removed; framework- + constituent dimension references now ride on a dedicated + `used_const_dim_std_names` field on `ResolvedArg`. +- **`active` expression case-folding** — mixed-case standard names + in `active = (...)` are now lowercased at parse time so they match + the canonical lowercase host_dict keys (Fortran is case-insensitive, + so embedded logical operators are unaffected). + +### Test status + +- **Unit tests**: 1127 passing (`python -m pytest unit-tests/`). +- **End-to-end tests**: `advection`, `unit_conv`, `nested_suite`, + `variable_transform` covered. Tree is off-limits for in-session + edits — user-driven. ### Still deferred -- **End-to-end integration tests** — user-driven; off-limits for in-session - edits. - **Constituents overhaul** — discussion doc at `doc/constituents_overhaul.md` (2026-05-12). Three proposals on the table (A bugfix-only / B class-A/B split + setters / C host-only @@ -1157,16 +1205,26 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` - **Codegen-time scheme-registration cross-check** — new metadata attr `registers_std_names = a, b, c` on register-phase tables; replaces current runtime `int_unassigned` check with codegen-time error. -- **Capgen-ng cleanup**: replace `_FRAMEWORK_CONST_DIM_INPUTS` frozenset - in `generator/static_api.py` with `used_const_dim_std_names: Set[str]` - on `ResolvedArg`. -- **Nested subcycle `ccpp_loop_counter` semantics**: currently a scheme - inside a nested subcycle requesting `ccpp_loop_counter` would get - the OUTERMOST counter, not the innermost. None of the cam-sima - schemes use this — revisit if a real scheme needs the innermost. +- **Suppress `ccpp_host_constituents.F90` when unused** — currently + emitted for every build; now *correct* (empty) for SCM-style hosts + thanks to the host-wins rule, but still dead code. +- **`--legacy-mode` shim removal** — transient; remove + `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and + every `# legacy-compat:` touchpoint when scheme metadata has + migrated. +- **Nested subcycle `ccpp_loop_counter` semantics**: a scheme inside a + nested subcycle requesting `ccpp_loop_counter` would get the + OUTERMOST counter, not the innermost. None of the cam-sima schemes + use this — revisit if a real scheme needs the innermost. +- **Python linter / formatter pass** — pick `ruff` and apply across + `capgen-ng/`. +- **Generated Fortran ↔ Codee formatter idempotency** — emitted `.F90` + must round-trip cleanly through the project's Codee formatter. +- **`fortran_to_metadata` developer utility** — bootstrap a `.meta` + skeleton from an existing `.F90` subroutine. ### Where to find the migration summary -`doc/migration.md` (created 2026-05-12) — user-facing single-page -summary of metadata + SDF + host-Fortran requirements after all the -above changes. Read it first when porting a host model. +`doc/migration.md` — user-facing single-page summary of metadata + SDF ++ host-Fortran requirements after all the above changes. Read it +first when porting a host model. diff --git a/doc/redesign_prompt_20260513T0733.md b/doc/redesign_prompt_20260513T0733.md index c8e1ab3b..0ac76cde 100644 --- a/doc/redesign_prompt_20260513T0733.md +++ b/doc/redesign_prompt_20260513T0733.md @@ -1,5 +1,7 @@ # CCPP Framework Code Generator — Redesign Specification +*Last revised: 2026-05-13.* + ## Purpose This document is a complete implementation specification for a new CCPP Framework code @@ -7,6 +9,11 @@ generator (`ccpp-capgen-ng`). It supersedes both `ccpp-prebuild` and `ccpp-capge implementer should be able to build the new generator from scratch using this document alone, supplemented by the real-world examples in `redesign_analysis.md`. +The spec is essentially as-implemented as of the date above. User-facing +deltas relative to ccpp-prebuild and the original ccpp-capgen are +collected in `doc/migration.md`; section 18 of this document is a rolling +"outstanding work" tracker. + --- ## 1. Background and Motivation @@ -235,7 +242,12 @@ character arguments. | `number_of_physics_threads` | `integer` | Thread budget for physics-internal OpenMP; pass `1` if none | | `ccpp_error_message` | `character` | Error message string | | `ccpp_error_code` | `integer` | Integer error return code | -| `instance_number` | `integer` | Current model instance index; pass `1` if single-instance | + +`instance_number` is **paired-optional** with `number_of_instances` (host +table, §3.6): declare both for a multi-instance API, declare neither for a +single-instance API. Declaring exactly one is a hard error. When the pair +is absent, the static API signatures drop the `instance_number` argument +entirely and per-instance state arrays size to 1. `group_name` is **not** in the required set. It is included in the static API signature only if the host declares it in their `type=control` table. When absent: the static API @@ -290,25 +302,31 @@ Eight entry points are generated in the static API. Two tiers: ### 5.1 Framework lifecycle (no group_name dispatch) -These operate on the entire suite at once. They take `suite_name` plus -`ccpp_error_message` and `ccpp_error_code`. No scheme `_run/_init/_final` calls. +These operate on the entire suite at once. They take `suite_name`, +`ccpp_error_code`, and `ccpp_error_message` (plus `instance_number` when the +host opts into the multi-instance pair, §4.1). No scheme `_run/_init/_final` +calls. | Entry point | Purpose | |---|---| -| `ccpp_register(suite_name, constituents, errmsg, errcode)` | Calls each scheme's `_register` entrypoint; allocates and populates the `ccpp_model_constituents_t` object passed by the host | -| `ccpp_init(suite_name, [number_of_instances,] errmsg, errcode)` | Allocates integer state arrays in all group caps; allocates suite-owned interstitial data; no scheme calls | -| `ccpp_final(suite_name, errmsg, errcode)` | Deallocates integer state arrays and suite-owned data; no scheme calls | - -`constituents` in `ccpp_register` is `intent(inout)`: unallocated on entry, allocated -and populated on exit. The host declares and owns this object (imports -`ccpp_model_constituents_t` from the framework library). The argument is mandatory. - -`number_of_instances` in `ccpp_init` is **conditional**: included as an explicit -`intent(in)` integer argument when the host metadata declares a variable with standard -name `number_of_instances`; omitted entirely for single-instance models. The generator -detects this automatically at parse time (same conditionality rule as `instance_number`). -When present it is passed through the call chain: -`ccpp_init` → `_init` → each group's `state_alloc`. +| `ccpp_register(suite_name, errcode, errmsg, [instance_number])` | Calls each scheme's `_register` entrypoint; transitions suite state to `REGISTERED`. Auto-provisions `ccpp_model_constituents_obj(:)` and friends in `ccpp_host_constituents.F90` when any register-phase scheme declares `ccpp_constituent_properties_t(:)` (constituents are not a formal arg). | +| `ccpp_init(suite_name, errcode, errmsg, [instance_number])` | Allocates integer state arrays in all group caps; allocates suite-owned interstitial data; calls the suite-level `` scheme if declared (§5.5); no per-group scheme calls. | +| `ccpp_final(suite_name, errcode, errmsg, [instance_number])` | Calls the suite-level `` scheme if declared (§5.5); deallocates integer state arrays and suite-owned data; no per-group scheme calls. | + +Constituents are **opt-in**: a separate generated module +`ccpp_host_constituents.F90` declares `ccpp_model_constituents_obj(:)` and a +host-facing API (`ccpp_register_constituents`, `ccpp_initialize_constituents`, +`ccpp_const_get_index`, `ccpp_constituents_array(instance_number)`, +`ccpp_advected_constituents_array`, `ccpp_model_const_properties`, +`ccpp_number_constituents`, `ccpp_gather_constituents`, +`ccpp_update_constituents`, `ccpp_is_scheme_constituent`). The host calls +these directly — they are not formal arguments of `ccpp_register` / `ccpp_init`. +See `doc/constituents.md`. + +`instance_number` appears in every framework-lifecycle signature only when +the host declares the `instance_number` / `number_of_instances` pair (§4.1). +When present it propagates: `ccpp_init` → `_init` → each group's +`state_alloc(number_of_instances, ...)`. ### 5.2 Physics group invocation (dispatched by suite_name + group_name) @@ -373,26 +391,29 @@ Constraints: - The named scheme must have the matching phase in its metadata. Missing-phase metadata is a generator error. -### 5.4 Suite introspection routines (planned) +### 5.4 Suite introspection routines -In addition to the eight entry points above, the static API will expose four -**suite-introspection** subroutines that let a host query, at runtime, what is +In addition to the eight entry points above, the static API exposes **five** +suite-introspection subroutines that let a host query, at runtime, what is compiled into the API. These mirror the equivalent routines in the original -capgen (`scripts/ccpp_suite.py` — `write_inspection_routines`) and are required -for CMake integration and host-side build glue. They are **planned but not yet -implemented**; signatures below are the proposed shape and may be refined when -the original capgen sources are reviewed for adoption. +capgen (`scripts/ccpp_suite.py` — `write_inspection_routines`) and are +used by CMake integration and host-side build glue. | Entry point | Purpose | |---|---| | `ccpp_physics_suite_list(suites)` | Return all suite names compiled into the API | -| `ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errcode)` | Return the list of group ("part") names for a given suite | -| `ccpp_physics_suite_variables(suite_name, variable_list, errmsg, errcode, [input_vars], [output_vars], [struct_elements])` | Return the standard-name list a suite consumes/produces; optional flags filter by intent and whether DDT sub-fields are flattened | -| `ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errcode)` | Return the list of scheme module names that compose a suite | +| `ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errflg)` | Return the list of group ("part") names for a given suite | +| `ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errflg)` | Return the list of scheme module names that compose a suite | +| `ccpp_physics_suite_variables(suite_name, variable_list, errmsg, errflg, [input_vars], [output_vars], [struct_elements])` | Standard-name list a suite consumes/produces; optional flags filter by intent and whether DDT sub-fields are flattened | +| `ccpp_physics_suite_host_data(suite_name, variable_list, errmsg, errflg)` | Standard-name list of host data the suite reads — DDT-collapsed view, excludes generated control variables | These routines do not advance the state machine and do not call any scheme entrypoints. All inputs derive from generator-time data already held in `SuiteResolution` plus the host/scheme metadata; no new metadata is required. +The `_variables` vs `_host_data` split distinguishes the flat-leaf view +(every DDT field that is actually consumed) from the DDT-collapsed view +(parent DDT instances), and excludes capgen-ng-generated control +variables from `_host_data` since the host owns those. --- @@ -408,7 +429,8 @@ All three levels are fully auto-generated. No hand-written components in the cap USEd only by files that declare kind-typed variables: group caps, the suite types module, and the suite data module. - Dispatches all eight entry points by `suite_name` to the appropriate suite cap -- Passes `ccpp_model_constituents_t` through as an explicit argument (does not own it) +- Does not own constituent state; constituents are accessed via the separate + `ccpp_host_constituents.F90` module by both the host and group caps - Holds no physics state ### 6.2 Suite cap (`ccpp__cap.F90`) @@ -1105,10 +1127,9 @@ The following patterns from prebuild or capgen are explicitly **not** carried fo ## 18. Outstanding Work See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` -(deferred items) for the canonical list. Snapshot as of 2026-05-12 -(end of session): +(deferred items) for the canonical list. Snapshot as of 2026-05-13: -### Landed this session +### Landed in the 2026-05-12 session - **`instance_number` / `number_of_instances` paired opt-in** — hosts may omit both for a single-instance API. @@ -1128,8 +1149,9 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` - **TARGET on `ccpp_suite_data(:)`** module-level array. - **Group-state alloc idempotency** (matches suite-state alloc). - **Framework PR**: `ccpt_deallocate` ownership tracking via - `framework_owns_me` flag. Backward-compatible. Needs upstream - merge to ccpp-framework + ccpp-capgen. + `framework_owns_me` flag. Backward-compatible. Landed in + capgen-ng's vendored framework copy; still needs upstream merge + to ccpp-framework + original ccpp-capgen. - **Identity unit conversions** no longer emit misleading "unit conversion: kind_phys to kind_phys" comment. - **Improved duplicate-standard-name error** lists both colliding @@ -1138,12 +1160,38 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` scheme's init/final phase emitted inside `_init` / `_final`. Single scheme only; long-form spellings (``, ``, ``) rejected. -- **Test count**: 1070 passing. +- **Constituent resolver — host metadata wins**: hosts that declare + framework-named std_names (`ccpp_constituents`, `index_of_`, ...) + short-circuit capgen-ng's auto-provisioning so legacy hosts (GFS, + SCM) keep using their own short local names (e.g. `ntcw`) without + blowing Fortran's 63-char identifier limit. + +### Landed 2026-05-13 + +- **`--legacy-mode` shim** — transient parse-time rewrite of legacy + CCPP standard names (`horizontal_loop_extent` → + `horizontal_dimension`). Available on `ccpp_capgen_ng.py` and + `ccpp_validator.py`; loud banner at startup. Isolated in + `metadata/legacy_compat.py` and tagged `# legacy-compat:` for clean + removal once scheme metadata has been migrated. +- **`_FRAMEWORK_CONST_DIM_INPUTS` cleanup** — the hand-curated + frozenset in `generator/static_api.py` was removed; framework- + constituent dimension references now ride on a dedicated + `used_const_dim_std_names` field on `ResolvedArg`. +- **`active` expression case-folding** — mixed-case standard names + in `active = (...)` are now lowercased at parse time so they match + the canonical lowercase host_dict keys (Fortran is case-insensitive, + so embedded logical operators are unaffected). + +### Test status + +- **Unit tests**: 1127 passing (`python -m pytest unit-tests/`). +- **End-to-end tests**: `advection`, `unit_conv`, `nested_suite`, + `variable_transform` covered. Tree is off-limits for in-session + edits — user-driven. ### Still deferred -- **End-to-end integration tests** — user-driven; off-limits for in-session - edits. - **Constituents overhaul** — discussion doc at `doc/constituents_overhaul.md` (2026-05-12). Three proposals on the table (A bugfix-only / B class-A/B split + setters / C host-only @@ -1157,16 +1205,26 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` - **Codegen-time scheme-registration cross-check** — new metadata attr `registers_std_names = a, b, c` on register-phase tables; replaces current runtime `int_unassigned` check with codegen-time error. -- **Capgen-ng cleanup**: replace `_FRAMEWORK_CONST_DIM_INPUTS` frozenset - in `generator/static_api.py` with `used_const_dim_std_names: Set[str]` - on `ResolvedArg`. -- **Nested subcycle `ccpp_loop_counter` semantics**: currently a scheme - inside a nested subcycle requesting `ccpp_loop_counter` would get - the OUTERMOST counter, not the innermost. None of the cam-sima - schemes use this — revisit if a real scheme needs the innermost. +- **Suppress `ccpp_host_constituents.F90` when unused** — currently + emitted for every build; now *correct* (empty) for SCM-style hosts + thanks to the host-wins rule, but still dead code. +- **`--legacy-mode` shim removal** — transient; remove + `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and + every `# legacy-compat:` touchpoint when scheme metadata has + migrated. +- **Nested subcycle `ccpp_loop_counter` semantics**: a scheme inside a + nested subcycle requesting `ccpp_loop_counter` would get the + OUTERMOST counter, not the innermost. None of the cam-sima schemes + use this — revisit if a real scheme needs the innermost. +- **Python linter / formatter pass** — pick `ruff` and apply across + `capgen-ng/`. +- **Generated Fortran ↔ Codee formatter idempotency** — emitted `.F90` + must round-trip cleanly through the project's Codee formatter. +- **`fortran_to_metadata` developer utility** — bootstrap a `.meta` + skeleton from an existing `.F90` subroutine. ### Where to find the migration summary -`doc/migration.md` (created 2026-05-12) — user-facing single-page -summary of metadata + SDF + host-Fortran requirements after all the -above changes. Read it first when porting a host model. +`doc/migration.md` — user-facing single-page summary of metadata + SDF ++ host-Fortran requirements after all the above changes. Read it +first when porting a host model. From b9a28d237df9c6ac36841893bb73d9ab85ff9938 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 11:54:54 -0600 Subject: [PATCH 10/74] Update capgen-ng: ccpp_datafile --scheme-files --- capgen-ng/ccpp_capgen_ng.py | 79 ++++++++- capgen-ng/ccpp_datafile.py | 60 +++++++ capgen-ng/generator/datatable.py | 41 +++++ end-to-end-tests/advection/CMakeLists.txt | 6 +- end-to-end-tests/capgen_ng/CMakeLists.txt | 6 +- end-to-end-tests/chunked_data/CMakeLists.txt | 6 +- end-to-end-tests/ddthost/CMakeLists.txt | 6 +- end-to-end-tests/instances/CMakeLists.txt | 6 +- end-to-end-tests/nested_suite/CMakeLists.txt | 6 +- end-to-end-tests/opt_arg/CMakeLists.txt | 6 +- end-to-end-tests/var_compat/CMakeLists.txt | 6 +- unit-tests/test_ccpp_datafile.py | 32 +++- unit-tests/test_integration.py | 173 +++++++++++++++++++ 13 files changed, 416 insertions(+), 17 deletions(-) diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index a108987e..e7a8d79c 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -89,7 +89,7 @@ ) from generator.kinds_writer import write_ccpp_kinds from generator.suite_xml import parse_suite_xml_files -from generator.suite_resolver import resolve_suite +from generator.suite_resolver import resolve_suite, iter_phase_calls from generator.group_cap import write_group_cap from generator.suite_data import write_suite_data, write_suite_meta from generator.suite_cap import write_suite_cap @@ -984,19 +984,82 @@ def capgen( # Expanded SDFs (one per parsed suite) are inspection artifacts; carry # the paths set by parse_suite_xml() forward into datatable.xml. expanded_sdf_paths = [s.expanded_file for s in suites if s.expanded_file] - # Collect dependency paths from EVERY parsed metadata table — - # host, control, ddt, and scheme alike. ``dependencies =`` is - # legal on any [ccpp-table-properties] block per - # ``MetadataTable.apply_table_props``, so all of them must - # contribute to datatable.xml's section. - # Duplicates are collapsed by ``write_datatable``. + # Collect dependency paths. Host/control/ddt tables always contribute + # (their Fortran is shared across suites). Scheme-type tables only + # contribute when the scheme is actually referenced by a resolved + # suite — group phase calls, the suite-level scheme, or the + # suite-level scheme. DDT tables co-located in a scheme + # ``.meta`` file (``type = ddt`` block alongside ``type = scheme`` + # blocks) always contribute since DDT modules are shared host-side + # data, not gated on which scheme uses them. Unreferenced scheme + # metadata may sit on the CLI line for build-system convenience; we + # don't want its dependencies to leak into datatable.xml. Duplicates + # are collapsed by ``write_datatable``. + used_scheme_names: set = set() + for sr in suite_resolutions: + for rg in sr.groups: + for items in rg.phase_calls.values(): + for rc in iter_phase_calls(items): + used_scheme_names.add(rc.scheme_name) + if sr.suite_init_call is not None: + used_scheme_names.add(sr.suite_init_call.scheme_name) + if sr.suite_final_call is not None: + used_scheme_names.add(sr.suite_final_call.scheme_name) + dependency_paths = [] - for tbl in host_tables + scheme_tables: + for tbl in host_tables: dependency_paths.extend(tbl.dependencies) + for tbl in scheme_tables: + if tbl.table_type != 'scheme': + # DDT (or other non-scheme) tables that live alongside scheme + # tables in scheme metadata files — always contribute. + dependency_paths.extend(tbl.dependencies) + elif tbl.table_name in used_scheme_names: + dependency_paths.extend(tbl.dependencies) + + # Used-scheme Fortran source paths. Convention (shared with the + # validator's ``_fortran_file_for_table``): the ``.F90`` (or ``.F`` / + # ``.f90`` / ``.f``) lives under ``table.source_path`` with the same + # base name as the ``.meta`` file. Multiple scheme tables in one + # ``.meta`` share that single source file, so dedupe per path. + scheme_file_paths: List[str] = [] + _seen_scheme_files: set = set() + for tbl in scheme_tables: + # DDT tables inside scheme .meta files do not correspond to a + # scheme .F90 — skip them outright. + if tbl.table_type != 'scheme': + continue + if tbl.table_name not in used_scheme_names: + continue + meta_base = os.path.splitext(os.path.basename(tbl.file_path))[0] + search_dir = tbl.source_path or os.path.dirname( + os.path.abspath(tbl.file_path) + ) + resolved = None + for ext in ('.F90', '.f90', '.F', '.f'): + candidate = os.path.join(search_dir, meta_base + ext) + if os.path.isfile(candidate): + resolved = candidate + break + if resolved is None: + # Fall back to the canonical .F90 guess so the build-system + # query returns a useful (if missing) path; surface the gap + # at the same time so the user can fix source_path. + resolved = os.path.join(search_dir, meta_base + '.F90') + log.warning( + "Scheme '%s': no Fortran source found under '%s' for " + "any of .F90/.f90/.F/.f; using '%s' as the datatable " + "entry. Check the scheme's source_path table-property.", + tbl.table_name, search_dir, resolved, + ) + if resolved not in _seen_scheme_files: + _seen_scheme_files.add(resolved) + scheme_file_paths.append(resolved) datatable_path = write_datatable( suite_resolutions, scheme_store, utility_paths, suite_file_paths, output_root, host_file_paths=host_file_paths, + scheme_file_paths=scheme_file_paths, dependency_paths=dependency_paths, suite_meta_paths=suite_meta_paths, expanded_sdf_paths=expanded_sdf_paths, diff --git a/capgen-ng/ccpp_datafile.py b/capgen-ng/ccpp_datafile.py index ae11cbab..777a3e8c 100755 --- a/capgen-ng/ccpp_datafile.py +++ b/capgen-ng/ccpp_datafile.py @@ -51,6 +51,12 @@ "help": "Return a list of host CAP Fortran files created by capgen"}, {"report": "suite_files", "type": bool, "help": "Return a list of suite CAP Fortran files created by capgen"}, + {"report": "scheme_files", "type": bool, + "help": ("Return a list of scheme Fortran source files actually " + "referenced by some loaded suite (group phases + " + "suite-level / hooks). These are the " + "user-supplied scheme .F90 files, NOT capgen-generated " + "caps")}, {"report": "utility_files", "type": bool, "help": ("Return a list of utility Fortran files created by " "capgen (e.g., ccpp_kinds.F90)")}, @@ -287,6 +293,58 @@ def _retrieve_capgen_files(table, file_type=None): return capgen_files +def _retrieve_scheme_files(table): + """Find and return the list of used-scheme Fortran source paths from
. + + The ```` section lists the user-supplied scheme ``.F90`` + (or ``.F`` / ``.f90`` / ``.f``) sources for schemes that the loaded + suites actually reference. Build systems use this to compile exactly + the scheme set the suites consume; unreferenced scheme metadata + files passed on the capgen-ng CLI for convenience are filtered out. + + # Test valid scheme files + >>> table = ET.fromstring(""\ + "/path/to/scheme1.F90"\ + "/path/to/scheme2.F90"\ + "") + >>> _retrieve_scheme_files(table) + ['/path/to/scheme1.F90', '/path/to/scheme2.F90'] + + # Test empty + >>> table = ET.fromstring(""\ + "") + >>> _retrieve_scheme_files(table) + [] + + # Test missing section + >>> table = ET.fromstring("") + >>> _retrieve_scheme_files(table) + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Element type, 'scheme_files', not found in table + + # Test invalid entry type + >>> table = ET.fromstring(""\ + "/path/to/scheme1.F90"\ + "") + >>> _retrieve_scheme_files(table) + Traceback (most recent call last): + ... + ccpp_datafile.CCPPDatatableError: Invalid scheme file entry type, 'banana' + """ + result = [] + section = _find_table_section(table, "scheme_files") + for entry in section: + if entry.tag == "file": + if entry.text is not None: + result.append(entry.text) + else: + raise CCPPDatatableError( + "Invalid scheme file entry type, '{}'".format(entry.tag) + ) + return result + + def _retrieve_inspection_files(table, file_type=None): """Find and retrieve a list of inspection filenames from
. @@ -686,6 +744,8 @@ def datatable_report(datatable, action, sep, exclude_protected=False): result = _retrieve_capgen_files(table, file_type="host_files") elif action.action_is("suite_files"): result = _retrieve_capgen_files(table, file_type="suite_files") + elif action.action_is("scheme_files"): + result = _retrieve_scheme_files(table) elif action.action_is("utility_files"): result = _retrieve_capgen_files(table, file_type="utilities") elif action.action_is("inspection_files"): diff --git a/capgen-ng/generator/datatable.py b/capgen-ng/generator/datatable.py index 6c26b0d4..fac63253 100644 --- a/capgen-ng/generator/datatable.py +++ b/capgen-ng/generator/datatable.py @@ -31,6 +31,10 @@ ... + + /abs/path/scheme_x.F90 + ... + /abs/path/ccpp_.meta @@ -149,6 +153,25 @@ def _build_capgen_files( f.text = path +def _build_scheme_files( + root: ET.Element, + scheme_file_paths: List[str], +) -> None: + """Append ```` to *root*. + + Lists the Fortran source files of schemes actually referenced by some + loaded suite (group phase calls + suite-level ```` / ````). + These files are *not* generated by capgen-ng — they are the + user-supplied scheme implementations the build system must compile. + The section is always written (possibly empty) so the schema stays + stable for ``ccpp_datafile.py`` consumers. + """ + scheme_files = ET.SubElement(root, 'scheme_files') + for path in scheme_file_paths: + f = ET.SubElement(scheme_files, 'file') + f.text = path + + def _build_inspection_files( root: ET.Element, suite_meta_paths: Optional[List[str]] = None, @@ -208,6 +231,17 @@ def _build_schemes( if sname not in seen_scheme_phases: seen_scheme_phases[sname] = set() seen_scheme_phases[sname].add(phase_name) + # Suite-level / hooks are not part of any group's + # phase_calls — fold them in explicitly so the schemes they + # name appear in (and downstream queries pick them + # up). Their phase is fixed by the SDF element. + for rc, phase_name in ( + (sr.suite_init_call, 'init'), + (sr.suite_final_call, 'final'), + ): + if rc is None: + continue + seen_scheme_phases.setdefault(rc.scheme_name, set()).add(phase_name) for sname in sorted(seen_scheme_phases): scheme_elem = ET.SubElement(schemes_elem, 'scheme') @@ -342,6 +376,7 @@ def write_datatable( suite_file_paths: List[str], output_root: str, host_file_paths: Optional[List[str]] = None, + scheme_file_paths: Optional[List[str]] = None, dependency_paths: Optional[List[str]] = None, suite_meta_paths: Optional[List[str]] = None, expanded_sdf_paths: Optional[List[str]] = None, @@ -364,6 +399,11 @@ def write_datatable( Absolute paths to host-facing API files (capgen-ng emits ``ccpp_static_api.F90`` here). The ```` section is always written (possibly empty). + scheme_file_paths : list of str, optional + Absolute paths to the Fortran source files (``.F90`` / ``.F`` / ...) + of schemes actually referenced by the loaded suites. Written as + ```` children of the ```` element so build + systems can compile exactly the scheme set the suites consume. dependency_paths : list of str, optional Absolute paths of scheme dependency files (collected from ``MetadataTable.dependencies``). Written as ```` @@ -404,6 +444,7 @@ def write_datatable( root, utility_paths, host_file_paths or [], suite_file_paths, ) + _build_scheme_files(root, scheme_file_paths or []) _build_inspection_files( root, suite_meta_paths=suite_meta_paths, diff --git a/end-to-end-tests/advection/CMakeLists.txt b/end-to-end-tests/advection/CMakeLists.txt index 11661ead..dbc86bae 100644 --- a/end-to-end-tests/advection/CMakeLists.txt +++ b/end-to-end-tests/advection/CMakeLists.txt @@ -40,11 +40,15 @@ ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--dependencies") set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--capgen-files") set(CAPGEN_FILES ${CCPP_FILES}) message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") # Add extra files needed for testing @@ -55,7 +59,7 @@ set(EXTRA_FILES add_executable(test_advection.x ${EXTRA_FILES} ${CAPGEN_DEPENDENCIES} - ${SCHEME_FORTRAN_FILES} + ${SCHEME_FORTRAN_FILES_FILTERED} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES} test_advection_host_integration.F90 diff --git a/end-to-end-tests/capgen_ng/CMakeLists.txt b/end-to-end-tests/capgen_ng/CMakeLists.txt index 6236c598..a1e8b8ae 100644 --- a/end-to-end-tests/capgen_ng/CMakeLists.txt +++ b/end-to-end-tests/capgen_ng/CMakeLists.txt @@ -52,11 +52,15 @@ ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--dependencies") set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--capgen-files") set(CAPGEN_FILES ${CCPP_FILES}) message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") set(EXTRA_FILES @@ -66,7 +70,7 @@ set(EXTRA_FILES add_executable(test_capgen_ng.x ${EXTRA_FILES} ${CAPGEN_DEPENDENCIES} - ${SCHEME_FORTRAN_FILES} + ${SCHEME_FORTRAN_FILES_FILTERED} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES} test_capgen_host_integration.F90 diff --git a/end-to-end-tests/chunked_data/CMakeLists.txt b/end-to-end-tests/chunked_data/CMakeLists.txt index 131b5239..69fb5b90 100644 --- a/end-to-end-tests/chunked_data/CMakeLists.txt +++ b/end-to-end-tests/chunked_data/CMakeLists.txt @@ -40,16 +40,20 @@ ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--dependencies") set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--capgen-files") set(CAPGEN_FILES ${CCPP_FILES}) message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") add_executable(test_chunked_data.x ${CAPGEN_DEPENDENCIES} - ${SCHEME_FORTRAN_FILES} + ${SCHEME_FORTRAN_FILES_FILTERED} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES} ) diff --git a/end-to-end-tests/ddthost/CMakeLists.txt b/end-to-end-tests/ddthost/CMakeLists.txt index a99274bc..58d9b184 100644 --- a/end-to-end-tests/ddthost/CMakeLists.txt +++ b/end-to-end-tests/ddthost/CMakeLists.txt @@ -41,11 +41,15 @@ ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--dependencies") set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--capgen-files") set(CAPGEN_FILES ${CCPP_FILES}) message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") set(EXTRA_FILES @@ -55,7 +59,7 @@ set(EXTRA_FILES add_executable(test_ddthost.x ${EXTRA_FILES} ${CAPGEN_DEPENDENCIES} - ${SCHEME_FORTRAN_FILES} + ${SCHEME_FORTRAN_FILES_FILTERED} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES} test_ddt_host_integration.F90 diff --git a/end-to-end-tests/instances/CMakeLists.txt b/end-to-end-tests/instances/CMakeLists.txt index 2b56aeab..02a1fd3b 100644 --- a/end-to-end-tests/instances/CMakeLists.txt +++ b/end-to-end-tests/instances/CMakeLists.txt @@ -40,14 +40,18 @@ ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--dependencies") set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--capgen-files") set(CAPGEN_FILES ${CCPP_FILES}) message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") -add_executable(test_instances.x ${SCHEME_FORTRAN_FILES} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES}) +add_executable(test_instances.x ${SCHEME_FORTRAN_FILES_FILTERED} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES}) target_link_libraries(test_instances.x PRIVATE MPI::MPI_Fortran) if(OPENMP) target_link_libraries(test_instances.x PRIVATE OpenMP::OpenMP_Fortran) diff --git a/end-to-end-tests/nested_suite/CMakeLists.txt b/end-to-end-tests/nested_suite/CMakeLists.txt index 43a2e1f4..c050f81a 100644 --- a/end-to-end-tests/nested_suite/CMakeLists.txt +++ b/end-to-end-tests/nested_suite/CMakeLists.txt @@ -41,11 +41,15 @@ ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--dependencies") set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--capgen-files") set(CAPGEN_FILES ${CCPP_FILES}) message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") set(EXTRA_FILES @@ -55,7 +59,7 @@ set(EXTRA_FILES add_executable(test_nested_suite.x ${EXTRA_FILES} ${CAPGEN_DEPENDENCIES} - ${SCHEME_FORTRAN_FILES} + ${SCHEME_FORTRAN_FILES_FILTERED} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES} test_nested_suite_integration.F90 diff --git a/end-to-end-tests/opt_arg/CMakeLists.txt b/end-to-end-tests/opt_arg/CMakeLists.txt index 95ede75d..83f68a9d 100644 --- a/end-to-end-tests/opt_arg/CMakeLists.txt +++ b/end-to-end-tests/opt_arg/CMakeLists.txt @@ -40,14 +40,18 @@ ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--dependencies") set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--capgen-files") set(CAPGEN_FILES ${CCPP_FILES}) message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") -add_executable(test_opt_arg.x ${SCHEME_FORTRAN_FILES} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES}) +add_executable(test_opt_arg.x ${SCHEME_FORTRAN_FILES_FILTERED} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES}) target_link_libraries(test_opt_arg.x PRIVATE MPI::MPI_Fortran) if(OPENMP) target_link_libraries(test_opt_arg.x PRIVATE OpenMP::OpenMP_Fortran) diff --git a/end-to-end-tests/var_compat/CMakeLists.txt b/end-to-end-tests/var_compat/CMakeLists.txt index 50d5c6d7..6c2bb8b7 100644 --- a/end-to-end-tests/var_compat/CMakeLists.txt +++ b/end-to-end-tests/var_compat/CMakeLists.txt @@ -40,11 +40,15 @@ ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--dependencies") set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--capgen-files") set(CAPGEN_FILES ${CCPP_FILES}) message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") set(EXTRA_FILES @@ -54,7 +58,7 @@ set(EXTRA_FILES add_executable(test_var_compat.x ${EXTRA_FILES} ${CAPGEN_DEPENDENCIES} - ${SCHEME_FORTRAN_FILES} + ${SCHEME_FORTRAN_FILES_FILTERED} ${HOST_FORTRAN_FILES} ${CAPGEN_FILES} test_var_compatibility_integration.F90 diff --git a/unit-tests/test_ccpp_datafile.py b/unit-tests/test_ccpp_datafile.py index 592cf08f..ee304436 100644 --- a/unit-tests/test_ccpp_datafile.py +++ b/unit-tests/test_ccpp_datafile.py @@ -37,7 +37,8 @@ def _build_datatable(tmpdir, host_name='test_host', host_file_paths=None, utility_paths=None, - suite_file_paths=None, dependency_paths=None, + suite_file_paths=None, scheme_file_paths=None, + dependency_paths=None, suite_meta_paths=None, expanded_sdf_paths=None, protect_first_host_var=False): """Build a real datatable.xml in *tmpdir* and return its path.""" @@ -56,6 +57,7 @@ def _build_datatable(tmpdir, host_name='test_host', '/out/ccpp_test_simple_physics_cap.F90'], tmpdir, host_file_paths=host_file_paths or ['/out/ccpp_static_api.F90'], + scheme_file_paths=scheme_file_paths, dependency_paths=dependency_paths or [], suite_meta_paths=suite_meta_paths, expanded_sdf_paths=expanded_sdf_paths, @@ -170,6 +172,34 @@ def test_dependencies_empty_when_none(self): self.assertEqual(out, '') +class TestDatatableReportSchemeFiles(unittest.TestCase): + """--scheme-files returns the used-scheme Fortran source paths from + ; the section is always present (possibly empty) so the + query never raises on a vanilla datatable.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self._tmpdir) + + def test_scheme_files_returns_listed_paths(self): + path = _build_datatable( + self._tmpdir, + scheme_file_paths=['/phys/scheme_b.F90', '/phys/scheme_a.F90'], + ) + out = datatable_report(path, DatatableReport('scheme_files'), ',') + items = out.split(',') + # Writer preserves caller order; do not assume sort. + self.assertIn('/phys/scheme_b.F90', items) + self.assertIn('/phys/scheme_a.F90', items) + + def test_scheme_files_empty_when_none_given(self): + path = _build_datatable(self._tmpdir) + out = datatable_report(path, DatatableReport('scheme_files'), ',') + self.assertEqual(out, '') + + class TestDatatableReportDependenciesPopulated(unittest.TestCase): """--dependencies returns the sorted, dedup'd dependency list.""" diff --git a/unit-tests/test_integration.py b/unit-tests/test_integration.py index c7817478..b705163f 100644 --- a/unit-tests/test_integration.py +++ b/unit-tests/test_integration.py @@ -1202,6 +1202,166 @@ def test_dependencies_path_applied_to_host_deps(self): self.assertIn('/tmp/fake_phys/chemistry/some_chem.F90', joined) +class TestUnusedSchemeDependenciesFiltered(unittest.TestCase): + """Scheme metadata files supplied on the CLI but not referenced by + any loaded suite must not contribute to datatable.xml's + . Host build systems often pass the full physics + metadata catalog and rely on capgen-ng to narrow the compile set. + """ + + _USED_DEP = '/tmp/used_phys/used_dep.F90' + _UNUSED_DEP = '/tmp/unused_phys/unused_dep.F90' + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + used = os.path.join(self._tmpdir, 'used_scheme.meta') + unused = os.path.join(self._tmpdir, 'unused_scheme.meta') + # The "used" scheme is the existing scheme_multipart fixture + # with a single ``dependencies =`` line spliced into its outer + # [ccpp-table-properties] block. + with open(_sf('scheme_multipart.meta')) as src: + body = src.read() + used_body = body.replace( + ' type = scheme\n', + ' type = scheme\n dependencies = {}\n'.format(self._USED_DEP), + 1, + ) + with open(used, 'w') as fh: + fh.write(used_body) + # The "unused" scheme has its own dependency. The SDF below + # references temp_calc_adjust only, so this file's deps must be + # filtered out of datatable.xml. + with open(unused, 'w') as fh: + fh.write( + "[ccpp-table-properties]\n" + " name = scheme_never_used\n" + " type = scheme\n" + " dependencies = {}\n" + "\n" + "[ccpp-arg-table]\n" + " name = scheme_never_used_run\n" + " type = scheme\n" + "[ errmsg ]\n" + " standard_name = ccpp_error_message\n" + " units = none\n" + " dimensions = ()\n" + " type = character\n" + " kind = len=512\n" + " intent = out\n" + "[ errflg ]\n" + " standard_name = ccpp_error_code\n" + " units = 1\n" + " dimensions = ()\n" + " type = integer\n" + " intent = out\n".format(self._UNUSED_DEP) + ) + # Drop a placeholder .F90 next to the used scheme's .meta so the + # source-path resolver picks it up rather than warning. The + # filename matches the .meta basename per the convention. + with open( + os.path.join(self._tmpdir, 'used_scheme.F90'), 'w' + ) as fh: + fh.write('! placeholder for source-path resolution\n') + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[used, unused], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + root = tree.getroot() + self._deps = [ + d.text for d in root.find('dependencies').findall('dependency') + ] + self._scheme_files = [ + f.text for f in root.find('scheme_files').findall('file') + ] + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_used_scheme_dependency_present(self): + self.assertIn(self._USED_DEP, self._deps) + + def test_unused_scheme_dependency_absent(self): + self.assertNotIn(self._UNUSED_DEP, self._deps) + + def test_used_scheme_source_listed(self): + """ contains the resolved .F90 path for the used + scheme (same-base-name convention against the .meta file).""" + names = [os.path.basename(p) for p in self._scheme_files] + self.assertIn('used_scheme.F90', names) + + def test_unused_scheme_source_absent(self): + """A scheme metadata file passed on the CLI but not called by any + suite must not contribute its .F90 to .""" + names = [os.path.basename(p) for p in self._scheme_files] + self.assertNotIn('unused_scheme.F90', names) + self.assertNotIn('unused_scheme.f90', names) + + +class TestDdtDependenciesInSchemeMetaPreserved(unittest.TestCase): + """A scheme metadata file may carry a ``type = ddt`` block alongside + its ``type = scheme`` blocks (real-world pattern: a scheme that + constructs a DDT instance declares the DDT type in the same .meta). + The DDT block's ``dependencies = …`` must reach datatable.xml even + though the table name ('vmr_type', not the scheme name) won't match + the used-schemes set. Regression for the bug that broke the + end-to-end-tests/capgen_ng test where ddt2.F90 went missing because + the DDT's deps were filtered out alongside actual scheme deps.""" + + _DDT_DEP = '/tmp/ddt_phys/inner_ddt.F90' + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + meta = os.path.join(self._tmpdir, 'mixed_scheme.meta') + # Splice a ``type = ddt`` table in front of the existing + # scheme_multipart body so the .meta carries both kinds. + with open(_sf('scheme_multipart.meta')) as src: + body = src.read() + ddt_block = ( + "[ccpp-table-properties]\n" + " name = inner_ddt_type\n" + " type = ddt\n" + " dependencies = {dep}\n" + "[ccpp-arg-table]\n" + " name = inner_ddt_type\n" + " type = ddt\n" + "[ pad ]\n" + " standard_name = inner_ddt_padding\n" + " units = count\n" + " dimensions = ()\n" + " type = integer\n" + "\n" + ).format(dep=self._DDT_DEP) + with open(meta, 'w') as fh: + fh.write(ddt_block) + fh.write(body) + capgen( + host_name='test_host', + host_files=[_sf('host_full.meta'), _sf('control_full.meta')], + scheme_files=[meta], + suite_files=[_suite_file('suite_test_simple.xml')], + output_root=self._tmpdir, + kind_types={}, + ) + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + self._deps = [ + d.text for d in tree.getroot() + .find('dependencies').findall('dependency') + ] + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_ddt_dependency_preserved(self): + self.assertIn(self._DDT_DEP, self._deps) + + class TestSuiteInitFinalEmission(unittest.TestCase): """End-to-end: an SDF with ```` and ```` at the suite level produces calls to the named scheme's init/final phases inside @@ -1286,6 +1446,19 @@ def test_final_call_precedes_unregister_transition(self): ) self.assertLess(call_pos, state_set) + def test_suite_init_final_scheme_listed_in_datatable(self): + """The suite-level / scheme is genuinely 'used', + so it must appear in datatable.xml's section alongside + any group-phase schemes. Without this, downstream consumers + (e.g. CMake glue iterating ) miss the file.""" + tree = ET.parse(os.path.join(self._tmpdir, 'datatable.xml')) + names = { + s.get('name') + for s in tree.getroot().find('schemes').findall('scheme') + } + self.assertIn('suite_init_final_scheme', names) + self.assertIn('temp_calc_adjust', names) + class TestNestedSubcycleEmission(unittest.TestCase): """End-to-end: a nested ```` in the SDF must produce From 0f8398979390786c9365a48a83fde06983616119 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 20:35:59 -0600 Subject: [PATCH 11/74] Update capgen-ng/** and unit-test/* with bug fixes from CCPP-SCM GFS v17p8 suite --- capgen-ng/ccpp_capgen_ng.py | 49 +-- capgen-ng/ccpp_validator.py | 61 +++- capgen-ng/generator/datatable.py | 20 +- capgen-ng/generator/group_cap.py | 6 +- capgen-ng/generator/host_constituents.py | 5 +- capgen-ng/generator/kinds_writer.py | 13 +- capgen-ng/generator/static_api.py | 18 +- capgen-ng/generator/suite_cap.py | 22 +- capgen-ng/generator/suite_data.py | 9 +- capgen-ng/generator/suite_resolver.py | 310 +++++++++++++++---- capgen-ng/generator/suite_types.py | 209 ++++++++++++- capgen-ng/generator/suite_xml.py | 4 +- capgen-ng/metadata/legacy_compat.py | 69 +++-- capgen-ng/metadata/metadata_table.py | 130 ++++---- capgen-ng/metadata/parse_tools/__init__.py | 4 + capgen-ng/metadata/parse_tools/io_helpers.py | 152 +++++++++ capgen-ng/metadata/parse_tools/xml_tools.py | 23 +- capgen-ng/metadata/registered_dimensions.py | 192 ++++++++++++ capgen-ng/metadata/variable_resolver.py | 236 ++++++++++++-- unit-tests/test_integration.py | 97 ++++++ unit-tests/test_io_helpers.py | 189 +++++++++++ unit-tests/test_legacy_compat.py | 22 +- unit-tests/test_metadata_table.py | 33 ++ unit-tests/test_registered_dimensions.py | 104 +++++++ unit-tests/test_static_api.py | 37 ++- unit-tests/test_suite_cap.py | 55 ++++ unit-tests/test_suite_resolver.py | 274 ++++++++++++++++ unit-tests/test_suite_types.py | 228 ++++++++++++++ unit-tests/test_validator.py | 27 ++ unit-tests/test_variable_resolver.py | 294 +++++++++++++++++- 30 files changed, 2656 insertions(+), 236 deletions(-) create mode 100644 capgen-ng/metadata/parse_tools/io_helpers.py create mode 100644 capgen-ng/metadata/registered_dimensions.py create mode 100644 unit-tests/test_io_helpers.py create mode 100644 unit-tests/test_registered_dimensions.py create mode 100644 unit-tests/test_suite_types.py diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index e7a8d79c..1856ff48 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -883,8 +883,10 @@ def capgen( len(scheme_store.scheme_names()), scheme_store.scheme_names()) # ---- write ccpp_kinds.F90 (always generated) --------------------------- - kinds_path = write_ccpp_kinds(kind_types, output_root) - log.info("Wrote %s", kinds_path) + # Every writer below logs its own "Wrote " / "Unchanged: " + # line via the write-if-changed helper when *logger* is threaded + # through. Don't duplicate that log here. + kinds_path = write_ccpp_kinds(kind_types, output_root, logger=log) # ---- parse suite XML files ---------------------------------------------- suites = parse_suite_xml_files(suite_files, output_root, log) @@ -902,46 +904,45 @@ def capgen( # Group caps for rg in suite_res.groups: - cap_path = write_group_cap( - suite.name, rg.group_name, rg, host_dict, output_root + write_group_cap( + suite.name, rg.group_name, rg, host_dict, output_root, + logger=log, ) - log.info("Wrote %s", cap_path) # Suite data module - data_path = write_suite_data( + write_suite_data( suite.name, suite_res.suite_vars, output_root, host_dict, - ddt_module_map=ddt_module_map, + ddt_module_map=ddt_module_map, logger=log, ) - log.info("Wrote %s", data_path) # Suite metadata (for inspection) - meta_path = write_suite_meta(suite.name, suite_res.suite_vars, output_root) - log.info("Wrote %s", meta_path) + write_suite_meta( + suite.name, suite_res.suite_vars, output_root, logger=log, + ) # Suite types module (only when optional args are present) - types_path = write_suite_types(suite.name, suite_res, output_root) - if types_path: - log.info("Wrote %s", types_path) + write_suite_types( + suite.name, suite_res, output_root, + ddt_module_map=ddt_module_map, logger=log, + ) # Suite cap - suite_cap_path = write_suite_cap( - suite.name, suite_res, scheme_store, output_root, host_dict + write_suite_cap( + suite.name, suite_res, scheme_store, output_root, host_dict, + logger=log, ) - log.info("Wrote %s", suite_cap_path) # ---- static API (one file for all suites) ------------------------------ - static_path = write_static_api( - suite_names, suite_resolutions, output_root, host_dict, scheme_store + write_static_api( + suite_names, suite_resolutions, output_root, host_dict, scheme_store, + logger=log, ) - log.info("Wrote %s", static_path) # ---- host-wide constituent module (only when any suite touches # constituent state) ------------------------------------------------ host_consts_path = write_host_constituents( - suite_resolutions, output_root, host_dict=host_dict, + suite_resolutions, output_root, host_dict=host_dict, logger=log, ) - if host_consts_path: - log.info("Wrote %s", host_consts_path) # ---- datatable.xml ------------------------------------------------------ abs_root = os.path.abspath(output_root) @@ -1056,7 +1057,7 @@ def capgen( _seen_scheme_files.add(resolved) scheme_file_paths.append(resolved) - datatable_path = write_datatable( + write_datatable( suite_resolutions, scheme_store, utility_paths, suite_file_paths, output_root, host_file_paths=host_file_paths, scheme_file_paths=scheme_file_paths, @@ -1064,8 +1065,8 @@ def capgen( suite_meta_paths=suite_meta_paths, expanded_sdf_paths=expanded_sdf_paths, host_dict=host_dict, host_name=host_name, + logger=log, ) - log.info("Wrote %s", datatable_path) log.info("Cap generation complete.") diff --git a/capgen-ng/ccpp_validator.py b/capgen-ng/ccpp_validator.py index 89161830..7e57e8cb 100755 --- a/capgen-ng/ccpp_validator.py +++ b/capgen-ng/ccpp_validator.py @@ -158,12 +158,20 @@ def _line_optional_names(line: str) -> List[str]: def _join_continuation(lines: List[str]) -> List[str]: """Join Fortran continuation lines (ending with ``&``) into single logical lines. - Handles both free-form (``&`` only at the trailing end of the prior - line) and fixed-form / dual-form continuation (``&`` at the trailing - end *and* at column 6 of the next line). In dual-form code the - leading ``&`` is part of the continuation marker, not the continued - expression — it must be stripped before the next line is appended - to the buffer. + Handles three continuation conventions seen in real CCPP physics code: + + * **Free-form**: ``&`` only at the trailing end of the prior line. + * **Dual-form**: ``&`` at the trailing end of the prior line *and* + at column 6 of the next line. In this case the leading ``&`` is + part of the continuation marker, not the continued expression, + and is stripped before the next line is appended to the buffer. + * **Fixed-form leading-only**: NO trailing ``&`` on the prior line, + but a ``&`` (or any non-blank) at column 6 of the next line. F77 + / fixed-form Fortran treats this as a continuation; CCPP physics + occasionally relies on it (e.g. ``sfc_sice.f``'s ``sfc_sice_run`` + signature, where the line before the closing ``)`` has no trailing + ``&``). Detected by look-ahead at the next non-blank, non-comment + line. Examples -------- @@ -171,29 +179,54 @@ def _join_continuation(lines: List[str]) -> List[str]: [' foo bar', ' baz'] >>> _join_continuation([' foo &\\n', ' & bar\\n', ' baz\\n']) [' foo bar', ' baz'] + >>> _join_continuation([' foo &\\n', ' & bar\\n', + ... ' & )\\n', ' baz\\n']) + [' foo bar )', ' baz'] """ - result = [] - buf = '' + # First pass: normalise each line — strip trailing newlines and any + # inline ``!`` comment. Keep blank/comment-only lines as ``''`` so + # we can skip them when buffering and still use their position for + # look-ahead. + norm: List[str] = [] for raw in lines: line = raw.rstrip('\n').rstrip('\r') - stripped = _COMMENT_RE.sub('', line) + norm.append(_COMMENT_RE.sub('', line)) + + def _next_starts_with_lead_cont(start_idx: int) -> bool: + """True iff the next non-blank, non-comment line begins with a + leading ``&`` (fixed-form column-6 continuation marker).""" + j = start_idx + 1 + while j < len(norm) and not norm[j].strip(): + j += 1 + return j < len(norm) and bool(_LEAD_CONT_RE.match(norm[j])) + + result: List[str] = [] + buf = '' + for i, stripped in enumerate(norm): if buf and not stripped.strip(): # Mid-continuation, and this line is blank or a pure # comment. Fortran 90+ permits such lines interleaved # between continuation lines without ending the logical - # line — skip it and keep accumulating. + # line — skip and keep accumulating. continue if buf: # Mid-continuation: drop a leading ``&`` (fixed-form # column-6 marker) so it doesn't end up glued into the # continued expression. No-op on free-form code. stripped = _LEAD_CONT_RE.sub('', stripped, count=1) - if _CONT_RE.search(stripped): + has_trailing = bool(_CONT_RE.search(stripped)) + if has_trailing: buf += _CONT_RE.sub('', stripped) - else: + continue + # No trailing ``&`` — but a fixed-form continuation may still + # be implied by the next line's column-6 ``&``. If so, keep + # buffering rather than flushing. + if _next_starts_with_lead_cont(i): buf += stripped - result.append(buf) - buf = '' + continue + buf += stripped + result.append(buf) + buf = '' if buf: result.append(buf) return result diff --git a/capgen-ng/generator/datatable.py b/capgen-ng/generator/datatable.py index fac63253..87819b63 100644 --- a/capgen-ng/generator/datatable.py +++ b/capgen-ng/generator/datatable.py @@ -109,10 +109,13 @@ forward-compatible. """ +import io +import logging import os import xml.etree.ElementTree as ET from typing import Dict, List, Optional, Set, Tuple +from metadata.parse_tools import write_if_changed from generator.suite_resolver import SuiteResolution, iter_phase_calls _API_DICT_NAME = 'ccpp_api' @@ -382,6 +385,7 @@ def write_datatable( expanded_sdf_paths: Optional[List[str]] = None, host_dict=None, host_name: str = 'host', + logger: Optional[logging.Logger] = None, ) -> str: """Write ``datatable.xml`` and return its absolute path. @@ -462,10 +466,18 @@ def write_datatable( tree = ET.ElementTree(root) ET.indent(tree, space=' ') out_path = os.path.join(os.path.abspath(output_root), 'datatable.xml') - tree.write(out_path, encoding='unicode', xml_declaration=True) - # Ensure file ends with a newline. - with open(out_path, 'a') as fh: - fh.write('\n') + # Serialise to a string buffer instead of writing directly, then route + # through write_if_changed so unchanged datatables don't get a fresh + # mtime (preserves CMake/Make's no-rebuild behaviour). + buf = io.StringIO() + tree.write(buf, encoding='unicode', xml_declaration=True) + content = buf.getvalue() + # ElementTree's text serialiser omits the trailing newline; add one + # for POSIX-text-file convention so the comparison is stable across + # editors that auto-append a newline. + if not content.endswith('\n'): + content += '\n' + write_if_changed(out_path, content, logger=logger) return out_path diff --git a/capgen-ng/generator/group_cap.py b/capgen-ng/generator/group_cap.py index 3d44de07..71aa5f72 100644 --- a/capgen-ng/generator/group_cap.py +++ b/capgen-ng/generator/group_cap.py @@ -21,10 +21,11 @@ and then sets the element to 1. """ +import logging import os from typing import Dict, List, Optional, Set, Tuple -from metadata.parse_tools import FORTRAN_CONDITIONAL_REGEX +from metadata.parse_tools import FORTRAN_CONDITIONAL_REGEX, open_if_changed from metadata.variable_resolver import HostVarEntry from generator.suite_types import _ptr_type_name, _ptr_type_for_arg from generator.suite_resolver import ( @@ -1157,6 +1158,7 @@ def write_group_cap( rg: ResolvedGroup, host_dict, output_root: str, + logger: Optional[logging.Logger] = None, ) -> str: """Write the group cap Fortran module to *output_root*. @@ -1181,6 +1183,6 @@ def write_group_cap( out_path = os.path.join(output_root, filename) lines = _generate_group_cap(suite_name, group_name, rg, host_dict) - with open(out_path, 'w', encoding='utf-8') as fh: + with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return out_path diff --git a/capgen-ng/generator/host_constituents.py b/capgen-ng/generator/host_constituents.py index 575bd2e6..114fac40 100644 --- a/capgen-ng/generator/host_constituents.py +++ b/capgen-ng/generator/host_constituents.py @@ -26,9 +26,11 @@ been removed. """ +import logging import os from typing import List, Optional, Set, Tuple +from metadata.parse_tools import open_if_changed from generator.suite_resolver import SuiteResolution _INDENT = ' ' @@ -629,6 +631,7 @@ def write_host_constituents( suite_results: List[SuiteResolution], outdir: str, host_dict=None, + logger: Optional[logging.Logger] = None, ) -> Optional[str]: """Write ``ccpp_host_constituents.F90`` if needed, return its path or ``None``.""" lines = _generate_host_constituents(suite_results, host_dict) @@ -637,6 +640,6 @@ def write_host_constituents( if not os.path.isdir(outdir): os.makedirs(outdir, exist_ok=True) path = os.path.join(outdir, 'ccpp_host_constituents.F90') - with open(path, 'w') as fh: + with open_if_changed(path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return os.path.abspath(path) diff --git a/capgen-ng/generator/kinds_writer.py b/capgen-ng/generator/kinds_writer.py index 1cdf28c3..99e868d7 100644 --- a/capgen-ng/generator/kinds_writer.py +++ b/capgen-ng/generator/kinds_writer.py @@ -42,10 +42,11 @@ generated Fortran files that reference any kind parameter. """ +import logging import os -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Tuple -from metadata.parse_tools import CCPPError +from metadata.parse_tools import CCPPError, open_if_changed _KINDS_FILENAME = 'ccpp_kinds.F90' _KINDS_MODULE = 'ccpp_kinds' @@ -59,7 +60,11 @@ # Public API ######################################################################## -def write_ccpp_kinds(kind_types: KindMap, output_root: str) -> str: +def write_ccpp_kinds( + kind_types: KindMap, + output_root: str, + logger: Optional[logging.Logger] = None, +) -> str: """Write ``ccpp_kinds.F90`` to *output_root*. Parameters @@ -86,7 +91,7 @@ def write_ccpp_kinds(kind_types: KindMap, output_root: str) -> str: os.makedirs(output_root, exist_ok=True) lines = _generate_ccpp_kinds(kind_types) out_path = os.path.join(output_root, _KINDS_FILENAME) - with open(out_path, 'w', encoding='utf-8') as fh: + with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return out_path diff --git a/capgen-ng/generator/static_api.py b/capgen-ng/generator/static_api.py index b5f1dc8c..ebe83a9c 100644 --- a/capgen-ng/generator/static_api.py +++ b/capgen-ng/generator/static_api.py @@ -40,10 +40,11 @@ host has to read/write to interface with the suite. """ +import logging import os from typing import Dict, List, Optional, Set, Tuple -from metadata.parse_tools import CCPPError +from metadata.parse_tools import CCPPError, open_if_changed from metadata.variable_resolver import SchemeStore from generator.suite_resolver import ( ResolvedArg, @@ -565,6 +566,18 @@ def _physics_subroutine( lines.append('{} {}{}'.format(i3, carg, sep)) else: lines.append('{}call {}()'.format(i3, cap_sub)) + # case default: unknown suite name is a runtime error (not silent + # fall-through). Skip emission only when the host doesn't carry the + # standard error-reporting control vars — without somewhere to write + # the message, there is nothing meaningful to do here. + if errflg_local and errmsg_local: + lines.append('{}case default'.format(i2)) + lines.append('{}{} = 1'.format(i3, errflg_local)) + lines.append( + "{}{} = '{}: unknown suite: ' // trim({})".format( + i3, errmsg_local, sub_name, suite_name_local, + ) + ) lines.append('{}end select'.format(i2)) lines.append('') @@ -949,6 +962,7 @@ def write_static_api( output_root: str, host_dict=None, scheme_store: Optional[SchemeStore] = None, + logger: Optional[logging.Logger] = None, ) -> str: """Write ``ccpp_static_api.F90`` to *output_root*. @@ -977,6 +991,6 @@ def write_static_api( out_path = os.path.join(output_root, filename) lines = _generate_static_api(suite_names, suite_resolutions, host_dict, scheme_store) - with open(out_path, 'w', encoding='utf-8') as fh: + with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return out_path diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index f8c178f0..1a4b949c 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -19,9 +19,11 @@ these subroutines. """ +import logging import os -from typing import Dict, List, Set +from typing import Dict, List, Optional, Set +from metadata.parse_tools import open_if_changed from metadata.variable_resolver import HostVarEntry, SchemeStore from generator.suite_resolver import ( ResolvedArg, @@ -828,6 +830,21 @@ def _emit_group_call(rg, indent): for rg in suite_res.groups: lines.append("{}case('{}')".format(i2, rg.group_name)) _emit_group_call(rg, i3) + # case default: anything other than '', 'all', or a known group + # is a runtime error — caller asked for a group this suite + # doesn't define. Without ccpp_error_code/_message in the host + # control table there's nowhere to write the message, so skip + # emission rather than silently swallow. + if errflg_local and errmsg_local: + sub_label = '{}_physics_{}'.format(suite_name, phase) + lines.append('{}case default'.format(i2)) + lines.append('{}{} = 1'.format(i3, errflg_local)) + lines.append( + "{}{} = '{}: unknown group: ' // trim({})".format( + i3, errmsg_local, sub_label, grp_local, + ) + ) + lines.append('{}return'.format(i3)) lines.append('{}end select'.format(i2)) else: # No group_name control var: call all groups unconditionally. @@ -1028,6 +1045,7 @@ def write_suite_cap( scheme_store: SchemeStore, output_root: str, host_dict=None, + logger: Optional[logging.Logger] = None, ) -> str: """Write ``ccpp__cap.F90`` to *output_root*. @@ -1051,6 +1069,6 @@ def write_suite_cap( out_path = os.path.join(output_root, filename) lines = _generate_suite_cap(suite_name, suite_res, scheme_store, host_dict) - with open(out_path, 'w', encoding='utf-8') as fh: + with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return out_path diff --git a/capgen-ng/generator/suite_data.py b/capgen-ng/generator/suite_data.py index 4b7601c9..faf198f6 100644 --- a/capgen-ng/generator/suite_data.py +++ b/capgen-ng/generator/suite_data.py @@ -2,10 +2,11 @@ """Generate the suite data module ``ccpp__data.F90``.""" +import logging import os from typing import Dict, List, Optional -from metadata.parse_tools import CCPPError +from metadata.parse_tools import CCPPError, open_if_changed from generator.suite_resolver import SuiteVar _INDENT = ' ' @@ -377,6 +378,7 @@ def write_suite_data( output_root: str, host_dict=None, ddt_module_map: Optional[Dict[str, str]] = None, + logger: Optional[logging.Logger] = None, ) -> str: """Write ``ccpp__data.F90`` to *output_root*.""" os.makedirs(output_root, exist_ok=True) @@ -384,7 +386,7 @@ def write_suite_data( out_path = os.path.join(output_root, filename) lines = _generate_suite_data(suite_name, suite_vars, host_dict, ddt_module_map=ddt_module_map) - with open(out_path, 'w', encoding='utf-8') as fh: + with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return out_path @@ -435,12 +437,13 @@ def write_suite_meta( suite_name: str, suite_vars: Dict[str, SuiteVar], output_root: str, + logger: Optional[logging.Logger] = None, ) -> str: """Write ``ccpp_.meta`` to *output_root* and return its path.""" os.makedirs(output_root, exist_ok=True) filename = 'ccpp_{}.meta'.format(suite_name) out_path = os.path.join(output_root, filename) lines = _generate_suite_meta(suite_name, suite_vars) - with open(out_path, 'w', encoding='utf-8') as fh: + with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return out_path diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index dcad5a35..f7d2e3b6 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -30,10 +30,14 @@ After normalisation the upper-bound standard name drives dispatch: -- ``instance_dimension`` / ``number_of_instances`` → ```` - (scalar extraction; the instance subscript is already in the access path for - DDT fields, but needed here for the DDT instance variable itself when it is - passed directly). +- Registered scalar-index dim (see + ``metadata.registered_dimensions.SCALAR_INDEX_DIMS``; currently + ``number_of_instances`` → ``instance_number``, + ``number_of_threads`` → ``thread_number``) → scalar extraction using + the paired index variable's local name. The scalar subscript is + already in the access path for DDT-component fields, but needed here + for a DDT instance variable itself when passed directly, and for any + flat-array dim that hits the same registered name. - ``horizontal_dimension`` / ``horizontal_loop_extent`` → ``:`` (all phases). The lower bound must resolve to ``1`` (i.e. be ``ccpp_constant_one`` or the integer literal ``1``). @@ -59,7 +63,12 @@ from typing import Dict, List, Optional, Set, Tuple, Union from metadata.parse_tools import CCPPError, FORTRAN_CONDITIONAL_REGEX -from metadata.variable_resolver import HostVarEntry, _INSTANCE_DIMS +from metadata.registered_dimensions import ( + SCALAR_INDEX_DIMS, + scalar_index_for, + is_scalar_index_dim, +) +from metadata.variable_resolver import HostVarEntry # Dimension standard names that map to horizontal loop bounds. _HORIZ_LOOP_DIMS: frozenset = frozenset({ @@ -335,7 +344,14 @@ def _resolve_single_bound( # so the emitted subscript references the actual storage and # the USE statement (which walks back to the root via # ``_root_symbol``) imports the right top-level symbol. - return entry.access_path + # The DDT-instance walk bakes registered scalar-index std + # names (e.g. ``(thread_number)``, ``(instance_number)``) into + # the access path as placeholders; resolve them to the host's + # local Fortran names here so a bound that turns into a + # subscript on a per-thread/per-instance DDT field doesn't + # leak the std-name placeholder through to the emitted cap + # code (Fortran rejects it as "no IMPLICIT type"). + return _substitute_scalar_idx(entry.access_path, host_dict) if suite_vars: sv = suite_vars.get(bound) if sv is not None: @@ -409,7 +425,10 @@ def _one_dim_part( Rules applied after normalisation: - * Upper bound in :data:`_INSTANCE_DIMS` → scalar ``instance_number``. + * Upper bound is a registered scalar-index dim (see + :data:`metadata.registered_dimensions.SCALAR_INDEX_DIMS`) → scalar + subscript using the paired index variable's local name (e.g. + ``instance_number``, ``thread_number``). * Upper bound in :data:`_HORIZ_LOOP_DIMS` → ``lb:ub`` (loop bounds). Lower bound **must** resolve to ``'1'`` (i.e. be ``ccpp_constant_one`` or the integer literal ``1``); any other value is an error. @@ -438,20 +457,27 @@ def _one_dim_part( lower_str = lower_str.strip() upper_str = upper_str.strip() - # Instance dimension: scalar subscript regardless of lower bound - if upper_str in _INSTANCE_DIMS: - inst_entry = host_dict.get(_INSTANCE_NUM_STD) - if inst_entry is None: + # Registered scalar-index dimension: collapse to the paired index + # variable's local Fortran name regardless of lower bound. See + # capgen-ng/metadata/registered_dimensions.py for the contract. + idx_std = scalar_index_for(upper_str) + if idx_std is not None: + idx_entry = host_dict.get(idx_std) + if idx_entry is None: raise CCPPError( - "Host metadata references instance dimension '{}' but the " - "host's type=control table does not declare " - "'instance_number'. Declare 'instance_number' and " - "'number_of_instances' (paired) for a multi-instance API, " - "or remove the instance dimension from the affected " - "metadata for a single-instance host.".format(upper_str) + "Metadata references registered scalar-index dimension " + "'{dim}', which is paired with index variable '{idx}', " + "but the host has not declared '{idx}' in any type=control " + "or type=host table. Either declare '{idx}' as a scalar " + "integer in the host control/host metadata, or remove " + "the '{dim}' dimension from the affected metadata. See " + "capgen-ng/metadata/registered_dimensions.py for the full " + "table of registered scalar-index pairings.".format( + dim=upper_str, idx=idx_std, + ) ) - used.add(_INSTANCE_NUM_STD) - return inst_entry.local_name, used + used.add(idx_std) + return idx_entry.local_name, used # Horizontal dimension: validate lower, return loop bounds if upper_str in _HORIZ_LOOP_DIMS: @@ -599,29 +625,54 @@ def _dim_has_vertical(dim: str) -> bool: return upper in _VDIM_STDS -def _substitute_instance_idx( +def _substitute_scalar_idx( expr: str, host_dict: Dict[str, HostVarEntry], ) -> str: - """Resolve the DDT-instance template ``(instance_number)`` in an - access expression. - - :func:`metadata.variable_resolver._instance_subscript` bakes the - literal string ``(instance_number)`` into the access path of every - HostVarEntry derived from a DDT-instance array. That string is a - *standard-name placeholder*; at codegen time it must be substituted - with the host's actual Fortran local name for ``instance_number``. - When the host has not declared the instance pair (single-instance - API), substitute ``(1)`` so the access path is still well-formed - against length-1 internal arrays. + """Resolve registered scalar-index placeholders in a DDT access expr. + + :func:`metadata.variable_resolver._instance_subscript` bakes one + placeholder per registered scalar-index dim into the access path of + every HostVarEntry derived from a DDT-instance container. The + placeholders are *standard names* (e.g. ``instance_number``, + ``thread_number``) drawn from + :data:`metadata.registered_dimensions.SCALAR_INDEX_DIMS`; this + function resolves each to the host's actual Fortran local name at + codegen time. + + Resolution rules per placeholder: + + * Found in *host_dict* → substitute the host's ``local_name``. + * Absent from *host_dict* → substitute the literal ``1`` (consistent + with the ``instance_number`` paired-opt-in single-instance fallback; + length-1 internal arrays still address correctly). + + Multi-pair access paths like ``foo(instance_number, thread_number)`` + are handled in a single pass — every registered placeholder in the + expression is rewritten. """ - if '(instance_number)' not in expr: + # Optimization: skip the work when no placeholder could possibly be + # present. ``(`` is the cheapest distinguishing token. + if '(' not in expr: return expr - inst_entry = host_dict.get(_INSTANCE_NUM_STD) - if inst_entry is None: - return expr.replace('(instance_number)', '(1)') - return expr.replace( - '(instance_number)', '({})'.format(inst_entry.local_name) - ) + out = expr + for idx_std in SCALAR_INDEX_DIMS.values(): + # Multiple placeholders may appear: as a sole subscript + # ``(idx_std)`` or as one of several ``(a, idx_std)``. Replace + # the bare std name token-wise, but only when it's clearly an + # index placeholder (preceded by ``(`` or ``, `` and followed by + # ``)`` or ``,``). In practice _instance_subscript only emits + # these inside a fresh subscript, so word-boundary replace is + # safe; we use re.sub to enforce the word boundary. + pattern = r'\b' + re.escape(idx_std) + r'\b' + entry = host_dict.get(idx_std) + replacement = entry.local_name if entry is not None else '1' + out = re.sub(pattern, replacement, out) + return out + + +# Backwards-compatibility shim — older code (and one external test) may +# still import the old name. Forward to the generalized impl. +_substitute_instance_idx = _substitute_scalar_idx def _translate_active_expr(active: str, host_dict: Dict[str, HostVarEntry]) -> str: @@ -1071,6 +1122,14 @@ def _local_name_conflict( n += 1 +#: Standard names of the two loop-context control variables (see +#: doc/redesign_prompt.md §4.2). Scheme args declaring these resolve +#: to the generated do-loop locals emitted by the group cap — they are +#: in scope only inside a ```` block. +_LOOP_COUNTER_STD = 'ccpp_loop_counter' +_LOOP_EXTENT_STD = 'ccpp_loop_extent' + + def _resolve_one_arg( scheme_var, # MetaVar from scheme metadata phase: str, @@ -1079,6 +1138,7 @@ def _resolve_one_arg( scheme_name: str, used_local_names: Set[str], suite_name: str = '', + loop_context: Optional[List[Tuple[str, Optional[str]]]] = None, ) -> ResolvedArg: """Resolve one scheme argument against host/control/suite dictionaries. @@ -1099,6 +1159,12 @@ def _resolve_one_arg( used_local_names : set of str Already-used local variable names in this group cap function (for conflict resolution of temp/pointer names). + loop_context : list of (str, str or None), optional + Stack of ``(loop_count_expr, loop_std_name)`` pairs for the + enclosing ```` blocks, outermost first. Empty (or + omitted) when the call is not inside any subcycle. Used to + resolve ``ccpp_loop_counter`` / ``ccpp_loop_extent`` scheme + args against the generated do-loop locals. Returns ------- @@ -1114,6 +1180,75 @@ def _resolve_one_arg( local = scheme_var.local_name optional = scheme_var.optional + # ---- loop-context std names (ccpp_loop_counter / ccpp_loop_extent) - + # Per design (doc/redesign_prompt.md §4.2): these are scoped to the + # body of a ````. Resolve them against the generated do- + # loop locals; outside a subcycle, raise a clear error pointing at + # the SDF contract rather than the host metadata. + if std_name in (_LOOP_COUNTER_STD, _LOOP_EXTENT_STD): + if not loop_context: + raise CCPPError( + "Scheme '{scheme}' (phase '{phase}') requests standard " + "name '{std}' for argument '{local}', but the scheme is " + "not placed inside a ```` block in the suite " + "definition file.\n" + "\n" + "'{std}' is a loop-context control variable scoped to a " + "subcycle do-loop body (see doc/redesign_prompt.md " + "§4.2). Either wrap the scheme in ``…`` in the SDF, or remove the " + "'{std}' argument from the scheme metadata.".format( + scheme=scheme_name, + phase=phase, + std=std_name, + local=local, + ) + ) + # Resolve to the OUTERMOST enclosing subcycle. Per the deferred + # item in doc/migration.md §8 ("Nested subcycle + # ccpp_loop_counter semantics"), nested-loop schemes that need + # the innermost counter aren't supported yet — every cam-sima + # / SCM use we've audited reads the OUTERMOST counter only. + outer_count_expr, _outer_std = loop_context[0] + if std_name == _LOOP_COUNTER_STD: + # The group cap emits the outermost do-loop with local + # variable ``ccpp_loop_counter`` (group_cap._loop_counter_name + # depth 1). Match that name verbatim — it's in scope wherever + # this scheme call site is emitted. + call_expr = 'ccpp_loop_counter' + else: # _LOOP_EXTENT_STD + # ``ccpp_loop_extent`` is the OUTERMOST subcycle's loop + # count — either an integer literal (e.g. ``'3'``) or a + # host-resolved local name (e.g. ``'n_sub'``) depending on + # how the SDF declared ``loop=``. + call_expr = outer_count_expr + return ResolvedArg( + standard_name=std_name, + scheme_local_name=local, + intent=intent, + is_optional=optional, + active='', + active_local='', + source='control', + host_entry=None, + suite_var=None, + base_expr=call_expr, + subscript='', + call_expr=call_expr, + used_dim_std_names=set(), + needs_unit_transform=False, + needs_kind_transform=False, + unit_forward='', + unit_backward='', + kind_scheme=scheme_var.kind, + kind_host='', + temp_name='', + ptr_name='', + transform_case=1, + scheme_dimensions=list(scheme_var.dimensions), + used_const_dim_std_names=set(), + ) + # ---- detect constituent register args (special-cased) --------------- # Schemes that register dynamic constituents declare an intent=out # ``ccpp_constituent_properties_t`` allocatable array. Those arguments @@ -1709,9 +1844,54 @@ def resolve_suite( phases = ['register', 'init', 'timestep_init', 'run', 'timestep_final', 'final'] - # Detect whether any host variable uses an instance dimension. + # Validate up-front: every scheme name referenced by this suite — + # in any group (including nested subcycles/subcols) AND the + # suite-level / hooks — MUST be present in the scheme + # store. When a scheme is missing the resolver silently emits + # empty phase entries, which the cap generator then writes as a + # syntactically valid but semantically empty group cap (the user + # gets a successful build with the wrong runtime behaviour). + # Surface the configuration error here with the full list of + # missing schemes and a remediation pointer. + referenced_schemes: List[str] = list(suite.all_scheme_names()) + if suite.init_scheme: + referenced_schemes.append(suite.init_scheme) + if suite.final_scheme: + referenced_schemes.append(suite.final_scheme) + missing: List[str] = [] + seen_missing: Set[str] = set() + for sname in referenced_schemes: + if not scheme_store.has_scheme(sname) and sname not in seen_missing: + missing.append(sname) + seen_missing.add(sname) + if missing: + raise CCPPError( + "Suite '{suite}' references {n} scheme(s) whose metadata is " + "not loaded:\n\n" + " {names}\n\n" + "These schemes are listed in the SDF but no matching " + "``[ccpp-table-properties] type = scheme`` table is available " + "in the metadata files passed via ``--scheme-files``. Add " + "the missing scheme ``.meta`` files to the generator's " + "--scheme-files argument (CMake users: add them to the " + "scheme metadata list in the relevant ``CMakeLists.txt``).\n" + "\n" + "Without this check capgen-ng would silently emit an empty " + "group cap and the build would succeed with the wrong " + "runtime behaviour (schemes never run).".format( + suite=suite.name, + n=len(missing), + names='\n '.join(missing), + ) + ) + + # Detect whether any host variable uses the instance dimension + # specifically (multi-instance API marker on SuiteResolution). This + # is narrower than the general "registered scalar-index dim" check — + # we want to know only about the multi-instance pair here, not + # number_of_threads or future additions. uses_instance = any( - any(d in _INSTANCE_DIMS for d in entry.dimensions) + 'number_of_instances' in entry.dimensions for entry in host_dict.values() ) @@ -1875,8 +2055,16 @@ def _resolve_one_call( suite_vars: Dict[str, 'SuiteVar'], used_local_names: Set[str], suite_name: str = '', + loop_context: Optional[List[Tuple[str, Optional[str]]]] = None, ) -> Optional[ResolvedCall]: - """Build a ResolvedCall for one scheme/phase, or return None if not defined.""" + """Build a ResolvedCall for one scheme/phase, or return None if not defined. + + *loop_context* is a list of ``(loop_count_expr, loop_std_name)`` tuples + describing the enclosing ```` blocks, outermost first. Empty + when the call is not inside any subcycle. Forwarded to + :func:`_resolve_one_arg` so scheme args declaring ``ccpp_loop_counter`` + or ``ccpp_loop_extent`` can resolve against the generated loop locals. + """ vars_list = scheme_store.variables_for(scheme_name, phase) if vars_list is None: return None @@ -1887,7 +2075,7 @@ def _resolve_one_call( for sv in vars_list: arg = _resolve_one_arg( sv, phase, host_dict, suite_vars, scheme_name, used_local_names, - suite_name=suite_name, + suite_name=suite_name, loop_context=loop_context, ) rc.args.append(arg) return rc @@ -1933,10 +2121,18 @@ def _resolve_run_phase( """ from generator.suite_xml import SuiteScheme, SuiteSubcycle, SuiteSubcol - def _resolve_items(suite_items) -> List[PhaseItem]: + def _resolve_items( + suite_items, + loop_context: List[Tuple[str, Optional[str]]], + ) -> List[PhaseItem]: """Recursively turn a list of SuiteScheme/SuiteSubcycle/SuiteSubcol children into a list of :data:`PhaseItem`. Used at the top level of a group AND for the body of every (possibly nested) subcycle. + + *loop_context* is the stack of enclosing ```` blocks + (outermost first), each as ``(loop_count_expr, loop_std_name)``. + Empty at the top level of a group; one entry per nested + subcycle depth. """ out: List[PhaseItem] = [] for sub in suite_items: @@ -1945,15 +2141,18 @@ def _resolve_items(suite_items) -> List[PhaseItem]: sub.name, phase, scheme_store, host_dict, suite_vars, used_local_names, suite_name=suite_name, + loop_context=loop_context, ) if rc is not None: out.append(rc) elif isinstance(sub, SuiteSubcycle): - inner = _resolve_items(sub.items) + loop_count, loop_std = _resolve_subcycle_loop_bound( + sub.loop, host_dict, suite_vars=suite_vars, + ) + inner = _resolve_items( + sub.items, loop_context + [(loop_count, loop_std)], + ) if inner: - loop_count, loop_std = _resolve_subcycle_loop_bound( - sub.loop, host_dict, suite_vars=suite_vars, - ) out.append(ResolvedSubcycle( loop=loop_count, calls=inner, loop_std_name=loop_std, @@ -1966,6 +2165,7 @@ def _resolve_items(suite_items) -> List[PhaseItem]: sn, phase, scheme_store, host_dict, suite_vars, used_local_names, suite_name=suite_name, + loop_context=loop_context, ) if rc is not None: out.append(rc) @@ -1977,15 +2177,18 @@ def _resolve_items(suite_items) -> List[PhaseItem]: if isinstance(item, SuiteScheme): rc = _resolve_one_call(item.name, phase, scheme_store, host_dict, suite_vars, used_local_names, - suite_name=suite_name) + suite_name=suite_name, + loop_context=[]) if rc is not None: result.append(rc) elif isinstance(item, SuiteSubcycle): - inner = _resolve_items(item.items) + loop_count, loop_std = _resolve_subcycle_loop_bound( + item.loop, host_dict, suite_vars=suite_vars, + ) + inner = _resolve_items( + item.items, [(loop_count, loop_std)], + ) if inner: - loop_count, loop_std = _resolve_subcycle_loop_bound( - item.loop, host_dict, suite_vars=suite_vars, - ) result.append(ResolvedSubcycle( loop=loop_count, calls=inner, loop_std_name=loop_std, @@ -1994,7 +2197,8 @@ def _resolve_items(suite_items) -> List[PhaseItem]: for sn in item.scheme_names(): rc = _resolve_one_call(sn, phase, scheme_store, host_dict, suite_vars, used_local_names, - suite_name=suite_name) + suite_name=suite_name, + loop_context=[]) if rc is not None: result.append(rc) diff --git a/capgen-ng/generator/suite_types.py b/capgen-ng/generator/suite_types.py index 672e64f7..20c0dea7 100644 --- a/capgen-ng/generator/suite_types.py +++ b/capgen-ng/generator/suite_types.py @@ -16,13 +16,41 @@ Only generated when at least one optional argument is present. """ +import logging import os -from typing import List, Optional, Set, Tuple +import re +from typing import Dict, List, Optional, Set, Tuple +from metadata.parse_tools import CCPPError, open_if_changed from generator.suite_resolver import SuiteResolution, iter_phase_calls _INDENT = ' ' +# Fortran intrinsic types; anything else is treated as a DDT (or an +# ``external::`` reference, handled separately). +_INTRINSICS = frozenset({ + 'real', 'integer', 'character', 'logical', 'complex', 'double precision' +}) + + +def _is_intrinsic(type_: str) -> bool: + return type_.strip().lower() in _INTRINSICS + + +def _is_external(type_: str) -> bool: + return type_.strip().lower().startswith('external:') + + +def _split_external(type_: str) -> Tuple[str, str]: + """Parse ``external::`` into ``(module, typename)``.""" + parts = type_.strip().split(':', 2) + if len(parts) != 3: + raise CCPPError( + "Malformed external type spec '{}'; expected " + "'external::'".format(type_) + ) + return parts[1], parts[2] + ######################################################################## # Type-name helpers @@ -34,7 +62,10 @@ def _ptr_type_name(type_: str, kind: str, rank: int) -> str: Parameters ---------- type_ : str - Fortran intrinsic type (e.g. ``'real'``, ``'integer'``). + Fortran intrinsic type (e.g. ``'real'``, ``'integer'``) or DDT + type name (e.g. ``'cmpfsw_type'``). External types + (``'external::'``) are reduced to just + ```` for the wrapper name. kind : str Kind parameter (e.g. ``'kind_phys'``), or ``''`` if none. rank : int @@ -54,15 +85,88 @@ def _ptr_type_name(type_: str, kind: str, rank: int) -> str: 'real_rank0_ptr_type' >>> _ptr_type_name('real', 'kind_phys', 2) 'real_kind_phys_rank2_ptr_type' + >>> _ptr_type_name('cmpfsw_type', '', 1) + 'cmpfsw_type_rank1_ptr_type' + >>> _ptr_type_name('external:mpi_f08:mpi_comm', '', 0) + 'mpi_comm_rank0_ptr_type' + >>> _ptr_type_name('character', 'len=10', 1) + 'character_len10_rank1_ptr_type' + >>> _ptr_type_name('character', 'len=3', 1) + 'character_len3_rank1_ptr_type' """ - parts = [type_] - if kind and not kind.startswith('len='): - parts.append(kind) + if _is_external(type_): + _, typename = _split_external(type_) + name = typename + else: + name = type_ + parts = [name] + if kind: + if kind.startswith('len='): + # Different character lengths require *different* wrapper + # types — a DDT component can't be ``character(len=*)``, so + # the wrapper name must encode the length spec. Sanitise + # the suffix because raw lengths like ``:`` (deferred) or + # ``*`` (assumed) aren't valid Fortran identifier chars, + # and would otherwise produce illegal type names like + # ``character_len:_rank1_ptr_type`` that the compiler + # rejects. + len_spec = kind[len('len='):].strip() + parts.append('len' + _sanitize_len_suffix(len_spec)) + else: + parts.append(kind) parts.append('rank{}'.format(rank)) parts.append('ptr_type') return '_'.join(parts) +def _sanitize_len_suffix(len_spec: str) -> str: + """Return a Fortran-identifier-safe suffix for a ``character(len=…)`` spec. + + Pointer-wrapper type names embed the length specifier, e.g. + ``character_len10_rank1_ptr_type``. Raw length specs include + forms that aren't valid Fortran identifier characters: + + * ``len=N`` (positive integer literal) — already safe. + * ``len=:`` (deferred length, paired with ``pointer`` / ``allocatable``) — + replaced with ``_deferred``. + * ``len=*`` (assumed length) — REJECTED: assumed-length is not + legal as a DDT component spec. Raises :class:`CCPPError`. + * ``len=NAME`` (Fortran parameter or constant symbol) — kept verbatim + when it is already a valid identifier. + + Anything that doesn't fit those forms raises ``CCPPError`` rather + than silently producing an illegal Fortran identifier. + """ + spec = len_spec.strip() + if not spec: + raise CCPPError( + "Empty character length specifier 'len=' in pointer-wrapper " + "type name construction; expected an integer literal, " + "a parameter name, or ':' for deferred length." + ) + if spec == ':': + return '_deferred' + if spec == '*': + raise CCPPError( + "character(len=*) cannot appear as a DDT component, so " + "capgen-ng cannot generate a pointer-wrapper type for it. " + "Use a concrete length, a parameter constant, or 'len=:' " + "(deferred length, paired with allocatable / pointer) " + "in the metadata instead." + ) + # Plain integer literal (digits) or Fortran identifier — accept verbatim. + if spec.isdigit() or _IDENT_RE.match(spec): + return spec + raise CCPPError( + "Cannot derive a Fortran-identifier-safe pointer-wrapper type " + "name from 'len={}'. Expected an integer literal, a parameter " + "identifier, or ':' (deferred length).".format(spec) + ) + + +_IDENT_RE = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$') + + def _ptr_rank(arg) -> int: """Return the effective rank of the value pointed to by *arg*'s pointer. @@ -126,8 +230,39 @@ def _collect_ptr_type_combos( ######################################################################## def _fortran_type_str_simple(type_: str, kind: str) -> str: - """Minimal Fortran type-clause builder (intrinsics only).""" + """Build a Fortran type-clause for a pointer-wrapper declaration. + + Handles three categories: + + * **Intrinsics** (``real``, ``integer``, ``character``, ...) — emit + with optional ``(kind=...)`` (or ``(len=...)`` for character). + * **DDT types** (anything not an intrinsic and not ``external:`` — + e.g. ``cmpfsw_type``) — wrap in ``type(...)`` so the declaration + is a syntactically valid Fortran derived-type reference. Kind is + not meaningful for DDTs. + * **External types** (``external::``) — emit + ``type()``. The defining module is USE'd separately + (see :func:`_collect_ddt_uses`). + + >>> _fortran_type_str_simple('real', 'kind_phys') + 'real(kind=kind_phys)' + >>> _fortran_type_str_simple('integer', '') + 'integer' + >>> _fortran_type_str_simple('character', 'len=512') + 'character(len=512)' + >>> _fortran_type_str_simple('cmpfsw_type', '') + 'type(cmpfsw_type)' + >>> _fortran_type_str_simple('external:mpi_f08:mpi_comm', '') + 'type(mpi_comm)' + """ t = type_.strip() + if _is_external(t): + _, typename = _split_external(t) + return 'type({})'.format(typename) + if not _is_intrinsic(t): + # DDT — Fortran requires the type(...) wrapper. Kind is not + # meaningful here; drop it. + return 'type({})'.format(t) if kind: if t.lower().startswith('character'): return 'character({})'.format(kind) @@ -135,6 +270,49 @@ def _fortran_type_str_simple(type_: str, kind: str) -> str: return t +def _collect_ddt_uses( + combos: Set[Tuple[str, str, int]], + ddt_module_map: Optional[Dict[str, str]], +) -> Dict[str, Set[str]]: + """Group DDT and external types referenced in *combos* by USE module. + + Intrinsics are skipped (they need no USE). DDT types are looked up + in *ddt_module_map* (built by + :func:`metadata.variable_resolver.build_ddt_module_map`); external + types parse their module out of the ``external::`` + prefix. + + Returns + ------- + dict mapping ``module_name -> {typename, ...}``. + + Raises + ------ + CCPPError + If a DDT referenced by a pointer wrapper is absent from + *ddt_module_map* — the generator can't emit a valid USE for it. + """ + uses: Dict[str, Set[str]] = {} + for type_, _kind, _rank in combos: + t = type_.strip() + if _is_intrinsic(t): + continue + if _is_external(t): + mod, typename = _split_external(t) + uses.setdefault(mod, set()).add(typename) + continue + # DDT — look up the defining Fortran module. + if ddt_module_map is None or t not in ddt_module_map: + raise CCPPError( + "Pointer wrapper needs DDT '{}' but no defining module " + "is known. Declare the DDT via a 'type = ddt' metadata " + "table co-located with its scheme/host/control metadata " + "so build_ddt_module_map can pick it up.".format(t) + ) + uses.setdefault(ddt_module_map[t], set()).add(t) + return uses + + def _dim_spec(rank: int) -> str: """Return the deferred-shape dimension specifier for a *rank*-dimensional pointer. @@ -153,6 +331,7 @@ def _dim_spec(rank: int) -> str: def _generate_suite_types( suite_name: str, combos: Set[Tuple[str, str, int]], + ddt_module_map: Optional[Dict[str, str]] = None, ) -> List[str]: """Generate the Fortran source lines for the suite types module. @@ -160,6 +339,10 @@ def _generate_suite_types( ---------- suite_name : str combos : set of (type_, kind, rank) + ddt_module_map : dict, optional + DDT type name → defining Fortran module. Required when any + ``combos`` entry's *type_* is a DDT — the module is USE'd so + the ``type()`` reference resolves. Returns ------- @@ -184,6 +367,14 @@ def _generate_suite_types( lines.append( '{}use ccpp_kinds, only: {}'.format(_INDENT, ', '.join(kind_names)) ) + + # USE the defining module for every DDT (or external) type the + # pointer wrappers reference, so ``type()`` resolves. + ddt_uses = _collect_ddt_uses(combos, ddt_module_map) + for mod in sorted(ddt_uses): + symbols = ', '.join(sorted(ddt_uses[mod])) + lines.append('{}use {}, only: {}'.format(_INDENT, mod, symbols)) + if kind_names or ddt_uses: lines.append('') lines.append('{}implicit none'.format(_INDENT)) @@ -221,6 +412,8 @@ def write_suite_types( suite_name: str, suite_res: SuiteResolution, output_root: str, + ddt_module_map: Optional[Dict[str, str]] = None, + logger: Optional[logging.Logger] = None, ) -> Optional[str]: """Write the suite types module to *output_root*. @@ -246,7 +439,7 @@ def write_suite_types( filename = 'ccpp_{}_types.F90'.format(suite_name) out_path = os.path.join(output_root, filename) - lines = _generate_suite_types(suite_name, combos) - with open(out_path, 'w', encoding='utf-8') as fh: + lines = _generate_suite_types(suite_name, combos, ddt_module_map) + with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return out_path diff --git a/capgen-ng/generator/suite_xml.py b/capgen-ng/generator/suite_xml.py index 863937d2..0dfee524 100644 --- a/capgen-ng/generator/suite_xml.py +++ b/capgen-ng/generator/suite_xml.py @@ -592,9 +592,11 @@ def parse_suite_xml( os.makedirs(output_root, exist_ok=True) expanded_name = f"ccpp_{suite.name}_expanded.xml" expanded_path = os.path.join(output_root, expanded_name) + # write_xml_file logs "Wrote " or "Unchanged: " via + # write_if_changed when the logger is provided — no need to duplicate + # here. write_xml_file(root, expanded_path, log) suite.expanded_file = expanded_path - log.info("Wrote expanded suite XML: %s", expanded_path) # ---- re-validate the expanded XML (catches duplicate xs:ID errors) ---- if not skip_validation: diff --git a/capgen-ng/metadata/legacy_compat.py b/capgen-ng/metadata/legacy_compat.py index 366928e1..cb380914 100644 --- a/capgen-ng/metadata/legacy_compat.py +++ b/capgen-ng/metadata/legacy_compat.py @@ -62,6 +62,14 @@ # ccpp-prebuild / original ccpp-capgen used ``horizontal_loop_extent`` # in scheme metadata where capgen-ng uses ``horizontal_dimension``. 'horizontal_loop_extent': 'horizontal_dimension', + + # Legacy CCPP-physics hosts (and SCM 17p8 in particular) sized + # per-thread DDT containers by ``number_of_openmp_threads``; the + # capgen-ng convention is ``number_of_threads`` (matching the + # ``thread_number`` control variable name). Aliasing here lets the + # host metadata flow through unchanged; once hosts have migrated, + # drop this entry. + 'number_of_openmp_threads': 'number_of_threads', } @@ -96,21 +104,41 @@ def enable(logger=None, _stream: Optional[TextIO] = None) -> None: _ENABLED = True stream = _stream if _stream is not None else sys.stderr - banner_lines = [ - '', - '*' * 70, - '*** WARNING: LEGACY-MODE ENABLED ***', - '*** ***', - '*** Scheme metadata using the deprecated standard name ***', - "*** 'horizontal_loop_extent' ***", - '*** will be silently rewritten to ***', - "*** 'horizontal_dimension' ***", - '*** at parse time. ***', - '*** ***', - '*** This is a TRANSIENT migration shim. Update your scheme ***', - '*** metadata to use the canonical name; legacy mode WILL BE ***', - '*** REMOVED in a future capgen-ng release. ***', - '*' * 70, + border_width = 70 + border = '*' * border_width + # Content width between the leading ``*** `` and trailing ` ***``. + _content_width = border_width - len('*** ') - len(' ***') + + def _pad(s: str) -> str: + """Format *s* as a banner row, left-padded to the border width.""" + return '*** {:<{w}} ***'.format(s[:_content_width], w=_content_width) + + banner_lines = ['', border, _pad('WARNING: LEGACY-MODE ENABLED'), + _pad('')] + if len(_LEGACY_NAME_MAP) == 1: + # Singular phrasing reads better when there's only one pair. + old, new = next(iter(_LEGACY_NAME_MAP.items())) + banner_lines += [ + _pad('Metadata using the deprecated standard name'), + _pad(" '{}'".format(old)), + _pad('will be silently rewritten to'), + _pad(" '{}'".format(new)), + _pad('at parse time.'), + ] + else: + banner_lines += [ + _pad('Metadata using any of these deprecated standard names'), + _pad('will be silently rewritten at parse time:'), + _pad(''), + ] + for old, new in sorted(_LEGACY_NAME_MAP.items()): + banner_lines.append(_pad(" '{}' -> '{}'".format(old, new))) + banner_lines += [ + _pad(''), + _pad('This is a TRANSIENT migration shim. Update your'), + _pad('metadata to use the canonical names; legacy mode'), + _pad('WILL BE REMOVED in a future capgen-ng release.'), + border, '', ] stream.write('\n'.join(banner_lines) + '\n') @@ -120,10 +148,15 @@ def enable(logger=None, _stream: Optional[TextIO] = None) -> None: pass if logger is not None: + pair_str = ', '.join( + "'{}' -> '{}'".format(old, new) + for old, new in sorted(_LEGACY_NAME_MAP.items()) + ) logger.warning( - "Legacy mode enabled: 'horizontal_loop_extent' will be " - "rewritten to 'horizontal_dimension' in scheme metadata. " - "This shim is transient and will be removed." + "Legacy mode enabled: the following deprecated standard " + "names will be rewritten in metadata at parse time: %s. " + "This shim is transient and will be removed.", + pair_str, ) diff --git a/capgen-ng/metadata/metadata_table.py b/capgen-ng/metadata/metadata_table.py index 631e636a..3f62d89f 100644 --- a/capgen-ng/metadata/metadata_table.py +++ b/capgen-ng/metadata/metadata_table.py @@ -557,61 +557,85 @@ def set_attr(self, key: str, value: str, context: ParseContext) -> None: ) self._set_attrs.add(key) - if key == 'standard_name': - # legacy-compat: rewrite deprecated names (e.g. - # horizontal_loop_extent → horizontal_dimension) when - # legacy mode is enabled. Applied *after* - # check_cf_standard_name (which lowercases) so mixed-case - # legacy spellings are captured. No-op otherwise. - self.standard_name = legacy_compat.translate( - check_cf_standard_name(value, None, error=True)) - elif key == 'long_name': - self.long_name = value - elif key == 'units': - self.units = check_units(value, None, error=True) - elif key == 'dimensions': - self.dimensions = _parse_dimensions(value, context) - elif key == 'type': - self.type = _check_var_type(value, context) - elif key == 'kind': - self.kind = value.strip() - elif key == 'intent': - iv = value.strip().lower() - if iv not in VALID_INTENTS: - raise CCPPError( - "Invalid intent '{}' for '{}'; must be one of {}, at {}".format( - value, self.local_name, sorted(VALID_INTENTS), context + # Wrap the per-attribute validation so any CCPPError raised by a + # check_X helper (which only sees the raw value) gets enriched + # with the variable name, the attribute name, and the source + # location. Without this, a parse failure like an empty + # ``units =`` line surfaces as a bare "'' is not a valid unit" + # with no clue which file/line/variable is at fault. + try: + if key == 'standard_name': + # legacy-compat: rewrite deprecated names (e.g. + # horizontal_loop_extent → horizontal_dimension) when + # legacy mode is enabled. Applied *after* + # check_cf_standard_name (which lowercases) so mixed-case + # legacy spellings are captured. No-op otherwise. + self.standard_name = legacy_compat.translate( + check_cf_standard_name(value, None, error=True)) + elif key == 'long_name': + self.long_name = value + elif key == 'units': + self.units = check_units(value, None, error=True) + elif key == 'dimensions': + self.dimensions = _parse_dimensions(value, context) + elif key == 'type': + self.type = _check_var_type(value, context) + elif key == 'kind': + self.kind = value.strip() + elif key == 'intent': + iv = value.strip().lower() + if iv not in VALID_INTENTS: + raise CCPPError( + "Invalid intent '{}'; must be one of {}".format( + value, sorted(VALID_INTENTS), + ) ) + self.intent = iv + elif key == 'optional': + self.optional = _parse_bool(value, context) + elif key == 'active': + # Standard names elsewhere are canonicalised to lowercase by + # check_cf_standard_name; an active expression references those + # same names, so normalise here too. Fortran is case-insensitive, + # so embedded logical operators/literals are unaffected. + self.active = value.strip().lower() + elif key == 'protected': + self.protected = _parse_bool(value, context) + elif key == 'allocatable': + self.allocatable = _parse_bool(value, context) + elif key == 'diagnostic_name': + self._diagnostic_name = check_diagnostic_id( + value.strip(), self._prop_snapshot(), error=True ) - self.intent = iv - elif key == 'optional': - self.optional = _parse_bool(value, context) - elif key == 'active': - # Standard names elsewhere are canonicalised to lowercase by - # check_cf_standard_name; an active expression references those - # same names, so normalise here too. Fortran is case-insensitive, - # so embedded logical operators/literals are unaffected. - self.active = value.strip().lower() - elif key == 'protected': - self.protected = _parse_bool(value, context) - elif key == 'allocatable': - self.allocatable = _parse_bool(value, context) - elif key == 'diagnostic_name': - self._diagnostic_name = check_diagnostic_id( - value.strip(), self._prop_snapshot(), error=True - ) - elif key == 'diagnostic_name_fixed': - self.diagnostic_name_fixed = check_diagnostic_fixed( - value.strip(), self._prop_snapshot(), error=True - ) - elif key == 'constituent': - self.constituent = _parse_bool(value, context) - elif key == 'advected': - self.advected = _parse_bool(value, context) - elif key == 'molar_mass': - self.molar_mass = check_molar_mass(value.strip(), None, error=True) - elif key == 'top_at_one': - self.top_at_one = _parse_bool(value, context) + elif key == 'diagnostic_name_fixed': + self.diagnostic_name_fixed = check_diagnostic_fixed( + value.strip(), self._prop_snapshot(), error=True + ) + elif key == 'constituent': + self.constituent = _parse_bool(value, context) + elif key == 'advected': + self.advected = _parse_bool(value, context) + elif key == 'molar_mass': + self.molar_mass = check_molar_mass(value.strip(), None, error=True) + elif key == 'top_at_one': + self.top_at_one = _parse_bool(value, context) + except CCPPError as exc: + # Avoid double-wrapping if the inner check already carried + # the location (some helpers do; most don't). + inner = str(exc) + location = str(context) + if location and location in inner: + raise + raise CCPPError( + "Invalid metadata for variable '{name}', attribute " + "'{key}' = '{value}', at {ctx}:\n {inner}".format( + name=self.local_name or '', + key=key, + value=value, + ctx=context, + inner=inner, + ) + ) from exc # ------------------------------------------------------------------ @property diff --git a/capgen-ng/metadata/parse_tools/__init__.py b/capgen-ng/metadata/parse_tools/__init__.py index 424d8f14..24b357f8 100644 --- a/capgen-ng/metadata/parse_tools/__init__.py +++ b/capgen-ng/metadata/parse_tools/__init__.py @@ -37,3 +37,7 @@ expand_nested_suites, write_xml_file, ) +from .io_helpers import ( + write_if_changed, + open_if_changed, +) diff --git a/capgen-ng/metadata/parse_tools/io_helpers.py b/capgen-ng/metadata/parse_tools/io_helpers.py new file mode 100644 index 00000000..be74d464 --- /dev/null +++ b/capgen-ng/metadata/parse_tools/io_helpers.py @@ -0,0 +1,152 @@ +"""File-write helpers with no-op-if-unchanged semantics. + +The original ``ccpp-prebuild`` and ``ccpp-capgen`` both avoided rewriting +generated cap files when their content was unchanged — preserving each +file's mtime so downstream build systems (CMake, Make, Ninja) don't +trigger unnecessary recompilation cascades. This module reproduces that +behaviour for ``capgen-ng``. + +Staging strategy +---------------- +A naive ``open(path, 'w')`` always touches the mtime, even when the +content is identical. Instead each writer builds the file's content in +memory and calls :func:`write_if_changed`, which: + +1. Reads the existing file at *file_path* (if any). +2. If the existing content matches the new content byte-for-byte, returns + ``False`` without touching the filesystem. +3. Otherwise writes the new content to a sibling temp file (in the same + directory, which sits **under the generator's output root** — never + ``/tmp``, so this works on systems that disallow ``/tmp`` writes) and + then ``os.replace``s it over the target. Same-directory replace is + atomic on POSIX and Windows; no partial writes. + +For writers that already produce content via a ``with open(...) as fh`` +pattern, use :func:`open_if_changed` as a drop-in replacement — it yields +a string buffer that gets staged on context exit. +""" + +import io +import logging +import os +import tempfile +from contextlib import contextmanager +from typing import Iterator, Optional + + +def write_if_changed( + file_path: str, + content: str, + encoding: str = 'utf-8', + logger: Optional[logging.Logger] = None, +) -> bool: + """Write *content* to *file_path* iff it differs from existing content. + + Parameters + ---------- + file_path : str + Absolute or relative path of the target file. + content : str + Full file content to write. + encoding : str + Encoding passed to :func:`open` for both the read-back comparison + and the staged write. Defaults to ``'utf-8'`` to match every + capgen-ng writer. + logger : logging.Logger, optional + When supplied, the helper logs an ``info``-level message after + each call: ``"Wrote "`` if the file was newly written or + rewritten, or ``"Unchanged: "`` if the existing content + matched and the filesystem was left untouched. Callers want + this so end users can tell at a glance which generated files + actually changed on a rerun (the original ccpp-prebuild / + ccpp-capgen output distinguished the two cases too). + + Returns + ------- + bool + ``True`` if the file was written or replaced; ``False`` if the + existing content already matched. + + Notes + ----- + The temp file is created in the target's parent directory via + :func:`tempfile.mkstemp` (which generates a unique name and opens it + with ``O_EXCL`` semantics). On any exception, the temp file is + removed so we never leak ``.capgen_tmp_*`` artifacts. Crucially the + staging directory is the target's parent — which is under the + generator's output root — so no ``/tmp`` access is required. + """ + parent = os.path.dirname(os.path.abspath(file_path)) or '.' + os.makedirs(parent, exist_ok=True) + + if os.path.isfile(file_path): + try: + with open(file_path, 'r', encoding=encoding) as fh: + existing = fh.read() + if existing == content: + if logger is not None: + logger.info("Unchanged: %s", file_path) + return False + except (OSError, UnicodeDecodeError): + # Fall through to overwrite if the existing file is + # unreadable or has a different encoding. + pass + + tmp_fd, tmp_path = tempfile.mkstemp( + dir=parent, + prefix='.capgen_tmp_', + suffix='_' + os.path.basename(file_path), + ) + try: + with os.fdopen(tmp_fd, 'w', encoding=encoding) as fh: + fh.write(content) + os.replace(tmp_path, file_path) + except BaseException: + try: + os.unlink(tmp_path) + except OSError: + pass + raise + if logger is not None: + logger.info("Wrote %s", file_path) + return True + + +@contextmanager +def open_if_changed( + file_path: str, + mode: str = 'w', + encoding: str = 'utf-8', + logger: Optional[logging.Logger] = None, +) -> Iterator[io.StringIO]: + """Drop-in replacement for ``open(file_path, 'w', encoding=...)``. + + Yields an in-memory :class:`io.StringIO` buffer. When the context + exits without an exception, the buffer's contents are handed to + :func:`write_if_changed` — so the on-disk file is only touched when + the content actually changes. + + Only text-mode writing is supported; *mode* must be ``'w'`` (the + parameter exists for call-site parity with the stdlib ``open``). + + Example + ------- + Refactor:: + + with open(out_path, 'w', encoding='utf-8') as fh: + fh.write('\\n'.join(lines) + '\\n') + + into:: + + with open_if_changed(out_path) as fh: + fh.write('\\n'.join(lines) + '\\n') + """ + if mode != 'w': + raise ValueError( + "open_if_changed only supports text-write mode 'w'; " + "got mode={!r}".format(mode) + ) + buf = io.StringIO() + yield buf + write_if_changed(file_path, buf.getvalue(), encoding=encoding, + logger=logger) diff --git a/capgen-ng/metadata/parse_tools/xml_tools.py b/capgen-ng/metadata/parse_tools/xml_tools.py index 337b9954..35105141 100644 --- a/capgen-ng/metadata/parse_tools/xml_tools.py +++ b/capgen-ng/metadata/parse_tools/xml_tools.py @@ -537,7 +537,14 @@ def expand_nested_suites(suite, default_path, logger=None): ############################################################################### def write_xml_file(root, file_path, logger=None): ############################################################################### - """Pretty-prints element root to an ASCII file using xml.dom.minidom""" + """Pretty-prints element root to an ASCII file using xml.dom.minidom. + + Routes the serialised XML through :func:`io_helpers.write_if_changed` + so an unchanged regeneration leaves the on-disk mtime alone. When + *logger* is supplied, the helper logs ``Wrote `` or + ``Unchanged: `` so the user can see at a glance whether the + file actually changed. + """ def remove_whitespace_nodes(node): """Helper function to recursively remove all text nodes that contain @@ -560,12 +567,12 @@ def remove_whitespace_nodes(node): # Generate pretty-printed XML string pretty_xml = reparsed.toprettyxml(indent=" ") - # Write to file - with open(file_path, 'w', errors='xmlcharrefreplace') as f: - f.write(pretty_xml) - - # Tell everyone! - if logger: - logger.debug(f"Writing XML file {file_path}") + # Route through write_if_changed so identical regenerated XML doesn't + # touch the mtime (downstream build tools rely on this to skip + # recompilation). Passing *logger* lets the helper emit the + # "Wrote / Unchanged" line directly, replacing the old debug-only + # write notice. + from .io_helpers import write_if_changed + write_if_changed(file_path, pretty_xml, logger=logger) ############################################################################## diff --git a/capgen-ng/metadata/registered_dimensions.py b/capgen-ng/metadata/registered_dimensions.py new file mode 100644 index 00000000..cba31f29 --- /dev/null +++ b/capgen-ng/metadata/registered_dimensions.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 + +"""Registered scalar-index dimensions for capgen-ng. + +This module is the **single source of truth** for the small set of CCPP +standard-name dimensions that capgen-ng treats specially: each one is a +*count* (e.g. ``number_of_instances``, ``number_of_threads``) whose access +pattern collapses to a paired scalar *index* variable (e.g. +``instance_number``, ``thread_number``). + +Where this is used +------------------ + +A container DDT-instance variable in host metadata may carry one of these +dimensions, e.g.:: + + [Interstitial] + standard_name = GFS_interstitial_type_instance + type = GFS_interstitial_type + dimensions = (number_of_threads) # <-- registered scalar dim + +When a scheme reaches into a field of that container, capgen-ng emits the +scalar index automatically:: + + physics%Interstitial(thread_number)%alpha(lb:ub, 1:nlev) + ^^^^^^^^^^^^^^^ + substituted from the registered scalar-index pair + +The same machinery applies anywhere capgen-ng needs to subscript a +container by an index variable that the host carries as a separate +control/host variable. + +The two rules +------------- + +**Rule 1 (generalized, NOT enforced as a hard gate)**: +A container DDT-instance variable may carry registered scalar-index +dimensions in its ``dimensions`` clause. When it does, capgen-ng emits +the paired index variable's local Fortran name at every call site that +reaches into the container. Anything *not* in :data:`SCALAR_INDEX_DIMS` +flows through the normal slice/bounds machinery +(``horizontal_loop_begin:horizontal_loop_end``, ``1:vertical_*``, …) +exactly like flat-array dims. + +**Rule 2 (ENFORCED at host-dict-build time)**: +A **leaf** variable (intrinsic-typed or ``external:`` Fortran type — the +kind a physics scheme actually binds to) MUST NOT declare a registered +scalar-index dim. Leaves see only spatial / tracer / count dims; scalar +indexing is a container-DDT concept. Violations raise a ``CCPPError`` +at parse time that names the offending variable, the offending dim, the +expected index variable, and points the user back to this file. + +Why both rules matter +--------------------- + +* Rule 1 generalization lets capgen-ng support multi-instance + (``number_of_instances``) **and** per-thread (``number_of_threads``) + container DDTs from one mechanism — no per-dimension code path. +* Rule 2 keeps the substitution mechanism contained: the rule "leaves + never carry registered scalar dims" means the scheme-call-site dim + handler (``_one_dim_part`` in ``generator/suite_resolver.py``) never + has to special-case scalar-index dims. One code path for leaves, one + code path for containers. + +How to extend the table +----------------------- + +Adding a new pairing (e.g. a new ``number_of_blocks`` ↔ ``block_number`` +convention) is a four-step process: + +1. **Add an entry below** mapping the *count* standard name to the + *index* standard name. Both are CCPP standard names the host must + declare. +2. **Verify the host metadata declares both**: a ``[ccpp-table-properties]`` + block with ``type = control`` (or ``type = host``) carrying a scalar + integer with the index standard name, plus a similar declaration for + the count. +3. **Add a unit test** to ``unit-tests/test_registered_dimensions.py`` + exercising the new pairing. +4. **Update** ``doc/migration.md`` §3 "Registered scalar-index + dimensions" and ``doc/redesign_prompt.md`` §4.3. + +The contract for users (host model authors) +------------------------------------------- + +If you see the error message:: + + Variable '' (standard_name='') declares dimension '' + on a leaf-data variable, but '' is a registered scalar-index + dimension reserved for DDT-instance containers (see + capgen-ng/metadata/registered_dimensions.py). + [...] + +it means you wrote something like:: + + [my_array] + standard_name = some_leaf_quantity + type = real | kind = kind_phys + dimensions = (number_of_threads, horizontal_dimension) + +The fix is to **wrap** the leaf in a per-thread container DDT, e.g.:: + + [Interstitial] + type = my_interstitial_type + dimensions = (number_of_threads) + +with ``my_interstitial_type`` declaring the leaf with only its spatial +dims:: + + [some_leaf_quantity] + type = real | kind = kind_phys + dimensions = (horizontal_dimension) + +This mirrors how every CCPP host in production today (UFS, NEPTUNE, +CAM-SIMA, ccpp-scm) structures per-thread / per-instance state. +""" + +from typing import Dict, FrozenSet, Optional + + +######################################################################## +# The registered scalar-index dimension table +######################################################################## + +#: Map from *count* dimension standard name → paired *index* variable +#: standard name. The host metadata MUST declare the index variable as +#: a scalar integer (in a ``type = control`` table when it is a +#: framework-lifecycle variable, or in a ``type = host`` table +#: otherwise). +#: +#: Every entry here is treated as a hard convention across the entire +#: CCPP ecosystem. Adding an entry binds capgen-ng to a specific +#: standard-name pairing; once an entry lands and hosts adopt it, +#: removing or renaming it is a breaking change. +SCALAR_INDEX_DIMS: Dict[str, str] = { + # Multi-instance API: the framework's instance_number paired opt-in. + # Hosts that declare instance_number + number_of_instances opt into + # the multi-instance API; capgen-ng auto-substitutes (instance_number) + # wherever a container DDT carries this dimension. + 'number_of_instances': 'instance_number', + + # Per-thread DDT containers (e.g. ``physics%Interstitial(thread_number)``) + # — the host's openmp-thread index. ``thread_number`` is a required + # control variable (see doc/migration.md §3.1), so any host using + # this dim already has the paired index in scope. + 'number_of_threads': 'thread_number', +} + + +######################################################################## +# Public helpers +######################################################################## + +def scalar_index_for(dim_std_name: str) -> Optional[str]: + """Return the paired scalar-index std name for *dim_std_name*, or None. + + >>> scalar_index_for('number_of_instances') + 'instance_number' + >>> scalar_index_for('number_of_threads') + 'thread_number' + >>> scalar_index_for('horizontal_dimension') is None + True + """ + return SCALAR_INDEX_DIMS.get(dim_std_name) + + +def is_scalar_index_dim(dim_std_name: str) -> bool: + """Return True iff *dim_std_name* is a registered scalar-index dimension. + + >>> is_scalar_index_dim('number_of_instances') + True + >>> is_scalar_index_dim('horizontal_dimension') + False + """ + return dim_std_name in SCALAR_INDEX_DIMS + + +def registered_count_dims() -> FrozenSet[str]: + """Return the set of all registered count-side dimension std names. + + Equivalent to ``frozenset(SCALAR_INDEX_DIMS)``; provided as a helper + so consumers don't have to reach into the dict. + """ + return frozenset(SCALAR_INDEX_DIMS) + + +def registered_index_vars() -> FrozenSet[str]: + """Return the set of all registered scalar-index variable std names. + + Equivalent to ``frozenset(SCALAR_INDEX_DIMS.values())``. + """ + return frozenset(SCALAR_INDEX_DIMS.values()) diff --git a/capgen-ng/metadata/variable_resolver.py b/capgen-ng/metadata/variable_resolver.py index 5bea7f89..37f0272f 100644 --- a/capgen-ng/metadata/variable_resolver.py +++ b/capgen-ng/metadata/variable_resolver.py @@ -56,14 +56,18 @@ # Registered dimension constants ######################################################################## -#: Dimension standard names that indicate a DDT is indexed by instance. -#: The host access path gains a ``(instance_number)`` subscript. -#: See Section 4.3 ("instance_dimension") and the Section 3.5 example -#: ("number_of_instances"). -_INSTANCE_DIMS: frozenset = frozenset({ - 'instance_dimension', - 'number_of_instances', -}) +# The set of registered scalar-index dimensions (e.g. +# ``number_of_instances`` → ``instance_number``, +# ``number_of_threads`` → ``thread_number``) lives in a single +# documented module so the contract is easy for users and developers to +# find and extend. See ``capgen-ng/metadata/registered_dimensions.py`` +# for the full table and the two rules that govern it. +from .registered_dimensions import ( + SCALAR_INDEX_DIMS, + scalar_index_for, + is_scalar_index_dim, + registered_count_dims, +) #: Regex that normalises ``type(typename)`` → ``typename``. _TYPE_PAREN_RE = re.compile( @@ -150,16 +154,87 @@ def _resolve_subscript(subscript: str, host_dict: Dict[str, 'HostVarEntry']) -> return ', '.join(resolved) +def _validate_leaf_dims(var: 'MetaVar', source_label: str) -> None: + """Reject leaf variables that declare a registered scalar-index dim. + + Rule 2 of the registered-scalar-index-dimension contract (see + :mod:`metadata.registered_dimensions`): a *leaf* variable — one that + a physics scheme actually binds to (intrinsic-typed or ``external:`` + Fortran type) — MUST NOT declare a dim like ``number_of_instances`` + or ``number_of_threads``. Those dims belong on container + DDT-instance variables in the access path, never on the leaf data + itself. + + Raises + ------ + CCPPError + With a message that names the offending variable, the offending + dim, the paired index variable, the source label (host file / + DDT table where the leaf was declared), and a pointer back to + :mod:`metadata.registered_dimensions` for the full table and + the remediation pattern. + """ + offenders = [d for d in var.dimensions if is_scalar_index_dim(d)] + if not offenders: + return + dim = offenders[0] + idx_std = scalar_index_for(dim) + raise CCPPError( + "Variable '{name}' (standard_name='{std}', declared in {src}) " + "is a leaf data variable but its dimensions list includes " + "'{dim}', a registered scalar-index dimension reserved for " + "DDT-instance container variables (paired with index " + "'{idx}').\n" + "\n" + "Leaf variables (intrinsic- or external-typed, the kind a " + "physics scheme binds to) MUST NOT carry registered scalar-" + "index dimensions. Wrap '{name}' in a container DDT whose " + "dimensions = ({dim}), and declare '{name}' inside that DDT " + "with only its spatial / tracer / count dims. The generator " + "will emit '({idx})%{name}(...)' at every scheme " + "call site automatically.\n" + "\n" + "See capgen-ng/metadata/registered_dimensions.py for the full " + "table of registered scalar-index pairings and how to extend " + "it.".format( + name=var.local_name, + std=var.standard_name, + src=source_label, + dim=dim, + idx=idx_std, + ) + ) + + def _instance_subscript(var: MetaVar) -> str: - """Return ``'(instance_number)'`` if *var* is a DDT instance array, else ``''``. + """Return the scalar-index subscript for a container DDT-instance variable. - A variable is treated as a DDT instance array when any of its declared - dimension standard names matches one of the :data:`_INSTANCE_DIMS` names. + Walks *var*'s declared dimensions in order; for each dim that is a + registered scalar-index dim (see + :mod:`metadata.registered_dimensions`), emits the paired index + variable's standard name as a placeholder. The placeholder is + resolved to the host's local Fortran name at codegen time by + :func:`generator.suite_resolver._substitute_scalar_idx`. + + Returns + ------- + str + Subscript string such as ``'(instance_number)'``, + ``'(thread_number)'``, or for multi-pair containers + ``'(instance_number, thread_number)'`` — one component per + registered scalar-index dim found in *var.dimensions* in + declared order. Returns ``''`` when no registered scalar dim + is present (the caller is left to handle non-registered dims + through the normal slice machinery). """ + parts = [] for dim in var.dimensions: - if dim in _INSTANCE_DIMS: - return '(instance_number)' - return '' + idx = scalar_index_for(dim) + if idx is not None: + parts.append(idx) + if not parts: + return '' + return '({})'.format(', '.join(parts)) ######################################################################## @@ -277,21 +352,61 @@ def _build_ddt_index(ddt_tables: List[MetadataTable]) -> Dict[str, MetadataTable return {tbl.table_name: tbl for tbl in ddt_tables} +def _resolve_module_name(tbl: MetadataTable) -> str: + """Return the Fortran module that exports *tbl*'s symbols. + + Honors the per-table ``module_name = …`` override from + ``[ccpp-table-properties]`` when present (the + ``design_module_name_override`` rule); otherwise falls back to the + table name (the implicit "module name = table name" convention). + """ + return (tbl.module_name or '').strip() or tbl.table_name + + def build_ddt_module_map( all_tables: List[MetadataTable], ) -> Dict[str, str]: """Build a map from DDT type name → Fortran module that defines it. - A DDT table inherits its defining Fortran module from a co-located - ``host``, ``control``, or ``scheme`` table in the same ``.meta`` file. - The convention is that a CCPP scheme/host/control table's name is the - name of the Fortran module that contains it; a DDT type defined alongside - such a table is assumed to be defined in the same Fortran module. - - DDT tables in a file with no co-located scheme/host/control table are - skipped (no entry written). DDTs that are only referenced as types of - host instance variables (declared in the host's own Fortran code) do not - need an entry — the host's Fortran code already imports the type. + Resolution order, per DDT table: + + 1. **DDT's own override.** If the DDT's own ``[ccpp-table-properties]`` + carries ``module_name = …``, that wins. Most specific source — a + DDT may genuinely live in a different Fortran module than the + scheme/host its ``.meta`` is paired with. Required when the DDT + lives in a file with no co-located scheme/host/control table at + all (real-world example: CCPP-physics + ``Radiation/RRTMG/radsw_param.meta`` declares ``cmpfsw_type`` in + Fortran ``module module_radsw_parameters``, with no co-located + scheme metadata). + 2. **Co-located table's resolved module.** Failing the DDT's own + override, inherit from a co-located ``host``, ``control``, or + ``scheme`` table in the same ``.meta`` file. Its module is + resolved by the same rule used elsewhere in capgen-ng + (:func:`_resolve_module_name`): the co-located table's own + ``module_name = …`` if declared, else its table name. + + DDT tables that pass neither rule are skipped (no entry written). + Those DDTs are only safe to leave out when no generator output + references them directly — e.g. a DDT referenced only as the type + of a host instance variable, where the host's own Fortran already + imports the type. + + What happens when both are present + ---------------------------------- + + +-------------------------+-------------------------+-----------------+ + | DDT ``module_name=`` | Co-located ``module_ | Result | + | | name=`` (or table name) | | + +=========================+=========================+=================+ + | X (set) | Y (set or default) | X — DDT wins | + +-------------------------+-------------------------+-----------------+ + | unset | Y (set or default) | Y | + +-------------------------+-------------------------+-----------------+ + | X (set) | (no co-located table) | X | + +-------------------------+-------------------------+-----------------+ + | unset | (no co-located table) | (skipped) | + +-------------------------+-------------------------+-----------------+ Parameters ---------- @@ -309,16 +424,25 @@ def build_ddt_module_map( result: Dict[str, str] = {} for fpath, tables in by_file.items(): - module_name: Optional[str] = None + # Co-located non-DDT table provides the fallback module name. + # Apply the same module_name-override-then-table-name resolution + # that ``build_flat_host_dict`` uses so a host/scheme that + # carries ``module_name = X`` is honored consistently. + colocated_module: Optional[str] = None for tbl in tables: if tbl.table_type in ('scheme', 'host', 'control'): - module_name = tbl.table_name + colocated_module = _resolve_module_name(tbl) break - if module_name is None: - continue + for tbl in tables: - if tbl.table_type == 'ddt': - result[tbl.table_name] = module_name + if tbl.table_type != 'ddt': + continue + # Per-table explicit override on the DDT itself wins. + if tbl.module_name: + result[tbl.table_name] = tbl.module_name + continue + if colocated_module is not None: + result[tbl.table_name] = colocated_module return result @@ -383,6 +507,28 @@ def _flatten_ddt_instance( ddt_table = ddt_index[ddt_name] subscript = _instance_subscript(var) + # If the DDT instance has dimensions but NONE of them are a + # registered scalar-index dim, capgen-ng can't bake a meaningful + # scalar subscript into field access paths. Two outcomes are both + # legitimate, depending on how schemes use this DDT: + # + # (a) Schemes take the whole sliced DDT array as a single arg + # (e.g. ``call rad_lw_run(fluxLW=phys_state%fluxLW(lb:ub), …)``) + # and dereference inner fields inside the scheme. Flattening + # this DDT's components into host_dict is wasted and emits + # Fortran the compiler rejects. + # (b) Schemes request individual inner fields by standard name, + # which would require ``parent%var()%field(…)`` access + # with a meaningful ```` capgen-ng can't synthesize. + # + # Skip the recursion either way: the DDT-instance's own entry is + # still recorded (case (a) just works), and case (b) trips the + # resolver's existing "standard_name not found" error when a scheme + # tries to use a would-have-been-flattened inner field. Use + # ``--legacy-mode`` (or fix the host metadata) when the underlying + # cause is a deprecated dimension name like + # ``number_of_openmp_threads``. + skip_recurse = bool(var.dimensions) and not subscript # Fortran access path to this DDT instance (without field component). instance_access = access_prefix + var.local_name + subscript @@ -406,6 +552,14 @@ def _flatten_ddt_instance( top_at_one=var.top_at_one, )) + # When the DDT-instance carries non-registered dims (skip_recurse), + # leave its fields un-flattened — only the DDT-instance entry above + # is recorded. Schemes taking the whole sliced DDT work via that + # entry; schemes asking for inner fields by std_name trip the + # resolver's standard "not found" error. + if skip_recurse: + return entries + # Expand each field of the DDT. for sec in ddt_table.sections(): for field in sec.variables: @@ -420,6 +574,16 @@ def _flatten_ddt_instance( max_depth=max_depth, )) else: + # Rule 2: a leaf DDT field cannot carry a registered + # scalar-index dim. Surface the violation at parse time + # with a clear remediation pointer. + _validate_leaf_dims( + field, + "DDT '{}' (file: {})".format( + ddt_name, + ddt_table.file_path, + ), + ) base_field, sub_str = _split_local_name(field.local_name) sub_tokens = [t.strip() for t in sub_str.split(',') if t.strip()] if sub_str else [] field_path = instance_access + '%' + base_field @@ -524,6 +688,12 @@ def _add(entry: HostVarEntry, source_label: str) -> None: ): _add(entry, tbl.table_name) elif _is_intrinsic(var.type) or _is_external(var.type): + _validate_leaf_dims( + var, + "host table '{}' (file: {})".format( + tbl.table_name, tbl.file_path, + ), + ) base_name, sub_str = _split_local_name(var.local_name) sub_tokens = [t.strip() for t in sub_str.split(',') if t.strip()] if sub_str else [] _add(HostVarEntry( @@ -559,6 +729,12 @@ def _add(entry: HostVarEntry, source_label: str) -> None: for tbl in control_tables: for sec in tbl.sections(): for var in sec.variables: + _validate_leaf_dims( + var, + "control table '{}' (file: {})".format( + tbl.table_name, tbl.file_path, + ), + ) base_name, sub_str = _split_local_name(var.local_name) sub_tokens = [t.strip() for t in sub_str.split(',') if t.strip()] if sub_str else [] _add(HostVarEntry( diff --git a/unit-tests/test_integration.py b/unit-tests/test_integration.py index b705163f..1eb2a656 100644 --- a/unit-tests/test_integration.py +++ b/unit-tests/test_integration.py @@ -8,6 +8,7 @@ import os import tempfile +import time import unittest import xml.etree.ElementTree as ET @@ -1303,6 +1304,102 @@ def test_unused_scheme_source_absent(self): self.assertNotIn('unused_scheme.f90', names) +class TestRegenerationIsNoopWhenContentUnchanged(unittest.TestCase): + """Running capgen twice with the same inputs must leave every + generated file's mtime untouched on the second invocation. This is + what ccpp-prebuild and original ccpp-capgen did so CMake / Make / + Ninja do NOT rebuild dependents on a no-op regeneration. The + generator stages each file via a sibling temp under the output root + and only replaces the target when content actually differs. + """ + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + self._first_run_files = sorted( + f for f in os.listdir(self._tmpdir) if not f.startswith('.') + ) + # Capture mtimes after the first run, then back-date everything + # by an hour so any rewrite is detectable as a bumped mtime. + self._old_mtimes = {} + old = time.time() - 3600 + for name in self._first_run_files: + path = os.path.join(self._tmpdir, name) + os.utime(path, (old, old)) + self._old_mtimes[name] = os.path.getmtime(path) + # Second run with identical inputs. + _run_simple(self._tmpdir) + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_no_generated_file_was_rewritten(self): + unchanged = [] + rewritten = [] + for name, old in self._old_mtimes.items(): + path = os.path.join(self._tmpdir, name) + new = os.path.getmtime(path) + if new == old: + unchanged.append(name) + else: + rewritten.append((name, old, new)) + self.assertEqual( + rewritten, [], + "files rewritten on no-op regeneration: {}".format(rewritten), + ) + # Sanity check: we did see at least one file on disk to compare. + self.assertTrue(unchanged) + + def test_no_temp_artifacts_remain(self): + """The staging temp files (``.capgen_tmp_*``) must be cleaned up + after each run; none may survive into the next build step.""" + leftovers = [ + f for f in os.listdir(self._tmpdir) + if f.startswith('.capgen_tmp_') + ] + self.assertEqual(leftovers, []) + + +class TestRegenerationRewritesWhenContentChanges(unittest.TestCase): + """Negative of the above: when the inputs change between runs, the + affected generated files MUST be rewritten (bumped mtime). + """ + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + _run_simple(self._tmpdir) + old = time.time() - 3600 + for name in os.listdir(self._tmpdir): + if name.startswith('.'): + continue + path = os.path.join(self._tmpdir, name) + os.utime(path, (old, old)) + # Second run picks a different SDF — exercises every cap file. + _run_simple(self._tmpdir, suite_xml='suite_test_subcycle.xml') + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_at_least_one_cap_was_rewritten(self): + """The subcycle suite differs from the simple suite — some cap + files MUST have a fresh mtime.""" + now = time.time() + any_recent = False + for name in os.listdir(self._tmpdir): + if name.startswith('.'): + continue + path = os.path.join(self._tmpdir, name) + if os.path.getmtime(path) > now - 60: + any_recent = True + break + self.assertTrue( + any_recent, + "no cap file was rewritten when the suite changed", + ) + + class TestDdtDependenciesInSchemeMetaPreserved(unittest.TestCase): """A scheme metadata file may carry a ``type = ddt`` block alongside its ``type = scheme`` blocks (real-world pattern: a scheme that diff --git a/unit-tests/test_io_helpers.py b/unit-tests/test_io_helpers.py new file mode 100644 index 00000000..ecb4c53c --- /dev/null +++ b/unit-tests/test_io_helpers.py @@ -0,0 +1,189 @@ +"""Unit tests for :mod:`metadata.parse_tools.io_helpers`. + +These cover the write-if-changed contract: + * unchanged content leaves the on-disk mtime untouched, so downstream + build tools (CMake, Make, Ninja) do not trigger unnecessary + recompiles; + * changed content (or missing file) results in an atomic + same-directory replace; + * the temp file lives under the target's parent directory (which sits + under the generator's output root), never ``/tmp``. +""" + +import logging +import os +import tempfile +import time +import unittest + +from metadata.parse_tools.io_helpers import open_if_changed, write_if_changed + + +class _Capture(logging.Handler): + """Tiny in-memory log handler for assertions.""" + + def __init__(self): + super().__init__(level=logging.DEBUG) + self.records = [] + + def emit(self, record): + self.records.append(self.format(record)) + + +class TestWriteIfChanged(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._path = os.path.join(self._tmpdir, 'out.txt') + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_creates_file_when_missing(self): + self.assertFalse(os.path.exists(self._path)) + wrote = write_if_changed(self._path, 'hello\n') + self.assertTrue(wrote) + with open(self._path) as fh: + self.assertEqual(fh.read(), 'hello\n') + + def test_skips_write_when_content_identical(self): + write_if_changed(self._path, 'hello\n') + # Set an old mtime so we can detect any rewrite. + old_mtime = time.time() - 3600 + os.utime(self._path, (old_mtime, old_mtime)) + observed = os.path.getmtime(self._path) + + wrote = write_if_changed(self._path, 'hello\n') + + self.assertFalse(wrote) + self.assertEqual(os.path.getmtime(self._path), observed) + + def test_rewrites_when_content_differs(self): + write_if_changed(self._path, 'hello\n') + old_mtime = time.time() - 3600 + os.utime(self._path, (old_mtime, old_mtime)) + + wrote = write_if_changed(self._path, 'goodbye\n') + + self.assertTrue(wrote) + self.assertGreater(os.path.getmtime(self._path), old_mtime) + with open(self._path) as fh: + self.assertEqual(fh.read(), 'goodbye\n') + + def test_temp_file_lives_under_target_directory(self): + """The staging temp file must be under the target's parent dir + (which is under output_root), not /tmp. Verified by listing + siblings of the target during a write.""" + observed_siblings = [] + + # Patch tempfile.mkstemp via a wrapper: we can't intercept inside + # the helper, but we can verify the contract by writing many + # files in the same dir and asserting no .capgen_tmp_* survives. + for i in range(5): + write_if_changed(self._path, 'iteration_{}\n'.format(i)) + observed_siblings.append(set(os.listdir(self._tmpdir))) + + for snapshot in observed_siblings: + for name in snapshot: + self.assertFalse( + name.startswith('.capgen_tmp_'), + "leaked temp file: {}".format(name), + ) + + def test_creates_parent_directory_if_missing(self): + nested = os.path.join(self._tmpdir, 'a', 'b', 'c', 'out.txt') + write_if_changed(nested, 'hi\n') + self.assertTrue(os.path.isfile(nested)) + + +class TestOpenIfChanged(unittest.TestCase): + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._path = os.path.join(self._tmpdir, 'out.txt') + + def tearDown(self): + import shutil + shutil.rmtree(self._tmpdir) + + def test_drop_in_for_open_write(self): + with open_if_changed(self._path) as fh: + fh.write('line1\n') + fh.write('line2\n') + with open(self._path) as fh: + self.assertEqual(fh.read(), 'line1\nline2\n') + + def test_idempotent_no_mtime_bump(self): + with open_if_changed(self._path) as fh: + fh.write('stable\n') + old_mtime = time.time() - 3600 + os.utime(self._path, (old_mtime, old_mtime)) + + with open_if_changed(self._path) as fh: + fh.write('stable\n') + + self.assertEqual(os.path.getmtime(self._path), old_mtime) + + def test_rejects_non_write_mode(self): + with self.assertRaises(ValueError): + with open_if_changed(self._path, mode='r'): + pass + + +class TestLogging(unittest.TestCase): + """The helper emits one info-level log line per call when *logger* is + supplied — ``Wrote `` on write, ``Unchanged: `` on no-op. + Build-tool users rely on the wording to tell at a glance which + generated files actually changed on a rerun.""" + + def setUp(self): + self._tmpdir = tempfile.mkdtemp() + self._path = os.path.join(self._tmpdir, 'out.txt') + self._logger = logging.getLogger('test_io_helpers') + self._logger.setLevel(logging.DEBUG) + # Strip any preexisting handlers so we only see ours. + for h in list(self._logger.handlers): + self._logger.removeHandler(h) + self._handler = _Capture() + self._handler.setFormatter(logging.Formatter('%(levelname)s %(message)s')) + self._logger.addHandler(self._handler) + self._logger.propagate = False + + def tearDown(self): + self._logger.removeHandler(self._handler) + import shutil + shutil.rmtree(self._tmpdir) + + def test_logs_wrote_on_new_file(self): + write_if_changed(self._path, 'hi\n', logger=self._logger) + joined = '\n'.join(self._handler.records) + self.assertIn('INFO Wrote', joined) + self.assertIn(self._path, joined) + self.assertNotIn('Unchanged', joined) + + def test_logs_unchanged_on_identical_rewrite(self): + write_if_changed(self._path, 'hi\n', logger=self._logger) + self._handler.records.clear() + write_if_changed(self._path, 'hi\n', logger=self._logger) + joined = '\n'.join(self._handler.records) + self.assertIn('INFO Unchanged:', joined) + self.assertIn(self._path, joined) + self.assertNotIn('Wrote', joined) + + def test_logs_wrote_on_content_change(self): + write_if_changed(self._path, 'hi\n', logger=self._logger) + self._handler.records.clear() + write_if_changed(self._path, 'bye\n', logger=self._logger) + joined = '\n'.join(self._handler.records) + self.assertIn('INFO Wrote', joined) + self.assertNotIn('Unchanged', joined) + + def test_open_if_changed_forwards_logger(self): + with open_if_changed(self._path, logger=self._logger) as fh: + fh.write('hi\n') + self.assertIn('INFO Wrote', '\n'.join(self._handler.records)) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_legacy_compat.py b/unit-tests/test_legacy_compat.py index 3b668fd2..6253d74b 100644 --- a/unit-tests/test_legacy_compat.py +++ b/unit-tests/test_legacy_compat.py @@ -67,11 +67,14 @@ def test_enable_flips_flag_and_writes_banner(self): legacy_compat.enable(_stream=sink) self.assertTrue(legacy_compat.is_enabled()) out = sink.getvalue() - # Bold banner: starred border, the deprecated name, and the - # canonical replacement all appear. + # Bold banner: starred border, the deprecated names, and the + # canonical replacements all appear. self.assertIn('LEGACY-MODE ENABLED', out) self.assertIn('horizontal_loop_extent', out) self.assertIn('horizontal_dimension', out) + # Banner also enumerates the number_of_openmp_threads pair. + self.assertIn('number_of_openmp_threads', out) + self.assertIn('number_of_threads', out) self.assertIn('TRANSIENT', out) self.assertIn('REMOVED', out) self.assertGreaterEqual(out.count('*' * 10), 2) @@ -105,7 +108,9 @@ def emit(self, record): legacy_compat.enable(logger=logger, _stream=io.StringIO()) self.assertEqual(len(records), 1) self.assertEqual(records[0].levelno, logging.WARNING) - self.assertIn('horizontal_loop_extent', records[0].getMessage()) + msg = records[0].getMessage() + self.assertIn('horizontal_loop_extent', msg) + self.assertIn('number_of_openmp_threads', msg) finally: logger.removeHandler(handler) @@ -123,6 +128,17 @@ def test_horizontal_loop_extent_rewritten(self): 'horizontal_dimension', ) + def test_number_of_openmp_threads_rewritten(self): + """Legacy CCPP-physics hosts (and SCM 17p8) size per-thread DDT + containers by ``number_of_openmp_threads``. The capgen-ng + convention is ``number_of_threads`` (matching the + ``thread_number`` control variable name). Legacy mode rewrites + both as a standard_name attribute AND as a dimension token.""" + self.assertEqual( + legacy_compat.translate('number_of_openmp_threads'), + 'number_of_threads', + ) + def test_unknown_name_passes_through(self): self.assertEqual( legacy_compat.translate('air_temperature'), 'air_temperature', diff --git a/unit-tests/test_metadata_table.py b/unit-tests/test_metadata_table.py index 622e46c7..2cd3aff9 100644 --- a/unit-tests/test_metadata_table.py +++ b/unit-tests/test_metadata_table.py @@ -278,6 +278,39 @@ def test_intent_invalid(self): with self.assertRaises(CCPPError): var.set_attr('intent', 'banana', ctx) + def test_invalid_attribute_error_carries_full_context(self): + """When a check_X helper rejects a value, the resulting CCPPError + MUST name the offending variable, the attribute, the raw value, + AND the source location. Without this enrichment the user sees + a bare ``'' is not a valid unit`` with no clue which file/line/var + is at fault — every check_X helper is unaware of context. + Regression for the SCM ccpp-physics 61-file parse where the user + couldn't locate the offending metadata. + """ + ctx = ParseContext(linenum=42, filename='broken_scheme.meta') + var = MetaVar('my_bad_var', ctx) + with self.assertRaises(CCPPError) as raised: + var.set_attr('units', '', ctx) + msg = str(raised.exception) + self.assertIn("'my_bad_var'", msg) # variable name + self.assertIn("'units'", msg) # attribute name + self.assertIn("broken_scheme.meta", msg) # file + self.assertIn(":43", msg) # line (1-based) + self.assertIn("not a valid unit", msg) # inner reason + + def test_invalid_attribute_error_does_not_double_wrap(self): + """Helpers that already include the location (``_parse_dimensions``, + ``_check_var_type``) should not have their location duplicated in + the wrapper message. Test confirms the wrapper detects the + already-present location and re-raises unchanged.""" + ctx = ParseContext(linenum=5, filename='dim_broken.meta') + var = MetaVar('v', ctx) + with self.assertRaises(CCPPError) as raised: + var.set_attr('dimensions', '(this is malformed', ctx) + msg = str(raised.exception) + # Location appears exactly once (no nested duplication). + self.assertEqual(msg.count('dim_broken.meta'), 1) + def test_protected_bool(self): var = self._make_var(protected='True') self.assertTrue(var.protected) diff --git a/unit-tests/test_registered_dimensions.py b/unit-tests/test_registered_dimensions.py new file mode 100644 index 00000000..bff8923f --- /dev/null +++ b/unit-tests/test_registered_dimensions.py @@ -0,0 +1,104 @@ +"""Tests for :mod:`metadata.registered_dimensions`. + +This module is the single source of truth for capgen-ng's registered +scalar-index dimension table. Tests here cover: + + * Every entry in :data:`SCALAR_INDEX_DIMS` is present (regression + against accidental removal during refactors). + * Helper functions return the expected values for entries inside and + outside the table. + * Adding a new pairing follows a clear, documented recipe — the + test_table_shape test exists so anyone extending the dict sees what + they need to update. + +When you add a new dim → index pair, add a corresponding test below +following the existing pattern. See the module's top docstring for +the four-step extension recipe. +""" + +import unittest + +from metadata.registered_dimensions import ( + SCALAR_INDEX_DIMS, + is_scalar_index_dim, + registered_count_dims, + registered_index_vars, + scalar_index_for, +) + + +class TestSCalarIndexDimsContents(unittest.TestCase): + """The registered table must contain at least the two pairings that + capgen-ng has committed to. Removing or renaming either is a + breaking change for hosts in production.""" + + def test_number_of_instances_pair(self): + self.assertEqual( + SCALAR_INDEX_DIMS['number_of_instances'], + 'instance_number', + ) + + def test_number_of_threads_pair(self): + self.assertEqual( + SCALAR_INDEX_DIMS['number_of_threads'], + 'thread_number', + ) + + def test_no_dead_instance_dimension_entry(self): + """The pre-2026-05-13 'instance_dimension' name was never used + in any real metadata; it shipped only in the original design + prompt. It MUST stay out of the registered table.""" + self.assertNotIn('instance_dimension', SCALAR_INDEX_DIMS) + + +class TestHelpers(unittest.TestCase): + + def test_scalar_index_for_registered(self): + self.assertEqual(scalar_index_for('number_of_instances'), + 'instance_number') + self.assertEqual(scalar_index_for('number_of_threads'), + 'thread_number') + + def test_scalar_index_for_unregistered_returns_none(self): + self.assertIsNone(scalar_index_for('horizontal_dimension')) + self.assertIsNone(scalar_index_for('vertical_layer_dimension')) + self.assertIsNone(scalar_index_for('number_of_ccpp_constituents')) + + def test_is_scalar_index_dim(self): + self.assertTrue(is_scalar_index_dim('number_of_instances')) + self.assertTrue(is_scalar_index_dim('number_of_threads')) + self.assertFalse(is_scalar_index_dim('horizontal_dimension')) + self.assertFalse(is_scalar_index_dim('instance_dimension')) + + def test_registered_count_dims_returns_keys(self): + self.assertEqual(registered_count_dims(), + frozenset(SCALAR_INDEX_DIMS)) + + def test_registered_index_vars_returns_values(self): + self.assertEqual(registered_index_vars(), + frozenset(SCALAR_INDEX_DIMS.values())) + + +class TestExtensionRecipe(unittest.TestCase): + """Anyone extending SCALAR_INDEX_DIMS should: + 1. Add an entry. + 2. Add a regression test in TestSCalarIndexDimsContents above. + 3. Update doc/migration.md §3 and doc/redesign_prompt.md §4.3. + 4. Add a unit test exercising the new pairing in the resolver. + This single test exists so the table's shape is asserted in one + place — if you add a key, this test fails and points you at the + above checklist. + """ + + def test_table_size(self): + # Update this assertion when you add a new pairing. See the + # docstring on this class for the full checklist. + self.assertEqual( + len(SCALAR_INDEX_DIMS), 2, + "Registered scalar-index table grew without test update — " + "see TestExtensionRecipe checklist", + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_static_api.py b/unit-tests/test_static_api.py index 8ccabe72..b8463a23 100644 --- a/unit-tests/test_static_api.py +++ b/unit-tests/test_static_api.py @@ -230,8 +230,11 @@ def test_init_dispatches_to_suite_cap(self): def test_final_dispatches_to_suite_cap(self): self.assertIn('call test_simple_physics_final()', self.text) - def test_physics_no_default_error_case(self): - # Physics dispatch has no error case for unknown suite — just skips. + def test_physics_no_default_error_case_without_host_dict(self): + # When no host_dict is available the standard error-reporting + # control vars (ccpp_error_code / ccpp_error_message) aren't in + # scope, so the physics dispatch has nowhere to write a "unknown + # suite" message — case default is intentionally omitted. run_block_start = self.text.index('subroutine ccpp_physics_run') run_block_end = self.text.index('end subroutine ccpp_physics_run') run_block = self.text[run_block_start:run_block_end] @@ -241,6 +244,36 @@ def test_select_case_on_suite_name(self): self.assertIn('select case(trim(suite_name))', self.text) +class TestCcppPhysicsUnknownSuiteErrors(unittest.TestCase): + """When the host provides ccpp_error_code / ccpp_error_message in + its control table, the physics dispatch ``select case`` MUST end + with a ``case default`` that sets errflg=1 and writes a message + naming the unknown suite — never silently fall through. + """ + + def setUp(self): + hd = _load_full_host_dict() + sr = _resolve() + self.text = '\n'.join(_generate_static_api(['test_simple'], [sr], hd)) + + def test_physics_run_has_default_case_with_errflg(self): + run_block_start = self.text.index('subroutine ccpp_physics_run') + run_block_end = self.text.index('end subroutine ccpp_physics_run') + run_block = self.text[run_block_start:run_block_end] + self.assertIn('case default', run_block) + # errflg must be set non-zero in the default branch. + self.assertRegex(run_block, r'case default[^!]*?errflg = 1') + + def test_physics_run_default_message_names_suite(self): + run_block_start = self.text.index('subroutine ccpp_physics_run') + run_block_end = self.text.index('end subroutine ccpp_physics_run') + run_block = self.text[run_block_start:run_block_end] + self.assertIn( + "ccpp_physics_run: unknown suite: ' // trim(suite_name)", + run_block, + ) + + class TestMultipleSuites(unittest.TestCase): """Static API with two suites uses select case for both.""" diff --git a/unit-tests/test_suite_cap.py b/unit-tests/test_suite_cap.py index 972d7b75..5178fa99 100644 --- a/unit-tests/test_suite_cap.py +++ b/unit-tests/test_suite_cap.py @@ -230,6 +230,61 @@ def test_no_select_case_without_group_name_ctrl(self): self.assertNotIn('select case(trim(group_name))', self.text) +class TestGroupDispatchUnknownGroupError(unittest.TestCase): + """When the host carries ``group_name`` in its control table the + suite cap dispatches via ``select case(trim(group_name))``. Every + such dispatch MUST end with a ``case default`` that sets errflg=1 + and writes a message naming the unknown group — caller asking for a + group this suite doesn't define is a runtime error, not a silent + fall-through. + """ + + def setUp(self): + sr, store = _resolve() + self.text = '\n'.join( + _generate_suite_cap('test_simple', sr, store, _load_full_host_dict()) + ) + + def _phase_block(self, phase): + sub = 'subroutine test_simple_physics_{}'.format(phase) + start = self.text.index(sub) + end = self.text.index('end subroutine test_simple_physics_{}'.format(phase), start) + return self.text[start:end] + + def test_run_dispatch_has_case_default(self): + block = self._phase_block('run') + # control_full.meta names the group_name local as grp_name; assert + # against the local name rather than the standard name. + self.assertIn('select case(trim(grp_name))', block) + self.assertIn('case default', block) + # errflg must be set non-zero in the default branch. + self.assertRegex(block, r'case default[^!]*?errflg = 1') + + def test_default_message_names_unknown_group(self): + block = self._phase_block('run') + self.assertIn( + "test_simple_physics_run: unknown group: ' // trim(grp_name)", + block, + ) + + def test_default_branch_returns(self): + """The default branch must ``return`` after setting errflg — + otherwise execution falls out of the select and into any code + that follows the dispatch (state transitions, etc.).""" + block = self._phase_block('run') + # Find the case-default section. + case_idx = block.index('case default') + end_idx = block.index('end select', case_idx) + default_block = block[case_idx:end_idx] + self.assertIn('return', default_block) + + def test_all_phases_have_default_case(self): + for phase in ('init', 'timestep_init', 'run', 'timestep_final', 'final'): + block = self._phase_block(phase) + self.assertIn('case default', block, + "phase '{}' missing case default".format(phase)) + + class TestWriteSuiteCap(unittest.TestCase): def test_writes_file(self): diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index d8753d97..9304ef6f 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -24,6 +24,7 @@ _apply_transform_formula, _build_call_subscript, _build_merged_subscript, + _resolve_single_bound, _substitute_instance_idx, _translate_active_expr, _root_symbol, @@ -456,6 +457,73 @@ def test_ccpp_constant_one_as_lower_vertical(self): ''' +class TestResolveSingleBoundSubstitutesScalarIdx(unittest.TestCase): + """``_resolve_single_bound`` returns the host entry's access path + for a DDT-component dim bound. The access path may carry baked-in + registered scalar-index placeholders (``(instance_number)``, + ``(thread_number)``) — those MUST be resolved to the host's local + Fortran names before the bound is spliced into a generated cap + subscript. Otherwise the cap leaks the std-name placeholder + verbatim and the Fortran compiler rejects it as "no IMPLICIT + type". Regression for the SCM phys_ps cap bug where + ``physics%Interstitial(thread_number)%nvdiff`` appeared inside the + ``vdftra`` slice expression.""" + + def _build_dict(self): + from metadata.metadata_table import _parse_lines + from metadata.variable_resolver import build_flat_host_dict + ddt_src = ( + "[ccpp-table-properties]\n name = GFS_interstitial_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_interstitial_type\n type = ddt\n" + "[ nvdiff ]\n standard_name = number_of_vertical_diffusion_tracers\n" + " units = count\n dimensions = ()\n type = integer\n" + "\n" + "[ccpp-table-properties]\n name = scm_phys_type\n type = ddt\n" + "[ccpp-arg-table]\n name = scm_phys_type\n type = ddt\n" + "[ Interstitial ]\n standard_name = GFS_interstitial_type_instance\n" + " units = DDT\n dimensions = (number_of_threads)\n" + " type = GFS_interstitial_type\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ physics ]\n standard_name = scm_physics_type_instance\n" + " units = DDT\n dimensions = ()\n type = scm_phys_type\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ mythread ]\n standard_name = thread_number\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + ) + return build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + + def test_thread_number_placeholder_substituted_to_host_local(self): + hd = self._build_dict() + # Pre-condition: the baked access path carries the std-name + # placeholder (``thread_number``), not yet ``mythread``. + self.assertEqual( + hd['number_of_vertical_diffusion_tracers'].access_path, + 'physics%Interstitial(thread_number)%nvdiff', + ) + used = set() + resolved = _resolve_single_bound( + 'number_of_vertical_diffusion_tracers', hd, used, + ) + # The substitution must fire: ``thread_number`` → host local + # ``mythread``. Without it the cap leaks the std-name placeholder. + self.assertEqual( + resolved, 'physics%Interstitial(mythread)%nvdiff', + ) + self.assertNotIn('thread_number', resolved) + # The bound std name is recorded in *used* for USE-list tracking. + self.assertIn('number_of_vertical_diffusion_tracers', used) + + class TestBuildMergedSubscript(unittest.TestCase): """Cover the slicing-with-standard-name-index code path. @@ -1198,6 +1266,212 @@ def test_control_args_no_module(self): self.assertIsNone(c.module_name) +class TestResolveSuiteLoopContextVariables(unittest.TestCase): + """``ccpp_loop_counter`` and ``ccpp_loop_extent`` are loop-context + control variables scoped to the body of a ```` block. + Scheme args declaring them MUST resolve against the generated + do-loop locals when inside a subcycle, and raise a clear error + when outside. Regression for the SCM GFS_surface_loop_control + failure where the resolver bailed with the generic 'not provided + by host' message.""" + + def _build_suite_from_xml(self, xml_src: str): + import tempfile, os + from generator.suite_xml import parse_suite_xml + from metadata.parse_tools import init_log + log = init_log('test_loop_ctx') + with tempfile.TemporaryDirectory() as tdir: + path = os.path.join(tdir, 's.xml') + with open(path, 'w') as fh: + fh.write(xml_src) + return parse_suite_xml(path, output_root=tdir, logger=log) + + _LOOP_SCHEME_SRC = ''' +[ccpp-table-properties] + name = loop_scheme + type = scheme +[ccpp-arg-table] + name = loop_scheme_run + type = scheme +[ iter ] + standard_name = ccpp_loop_counter + units = index + dimensions = () + type = integer + intent = in +[ niter ] + standard_name = ccpp_loop_extent + units = index + dimensions = () + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + + def _store(self): + from metadata.variable_resolver import SchemeStore + return SchemeStore.build_from( + _parse(self._LOOP_SCHEME_SRC, 'loop_scheme.meta') + ) + + def _args_by_name(self, sr): + calls = list(iter_phase_calls(sr.groups[0].phase_calls['run'])) + return {a.scheme_local_name: a for a in calls[0].args} + + def test_counter_inside_subcycle_resolves_to_loop_local(self): + xml = ( + "\n" + "\n" + " \n" + " \n" + " loop_scheme\n" + " \n" + " \n" + "\n" + ) + sr = resolve_suite(self._build_suite_from_xml(xml), + self._store(), _load_full_host_dict()) + args = self._args_by_name(sr) + self.assertEqual(args['iter'].standard_name, 'ccpp_loop_counter') + self.assertEqual(args['iter'].call_expr, 'ccpp_loop_counter') + self.assertEqual(args['iter'].source, 'control') + + def test_extent_integer_literal(self): + xml = ( + "\n" + "\n" + " \n" + " \n" + " loop_scheme\n" + " \n" + " \n" + "\n" + ) + sr = resolve_suite(self._build_suite_from_xml(xml), + self._store(), _load_full_host_dict()) + args = self._args_by_name(sr) + # ``loop=3`` is a literal — extent resolves to the same literal. + self.assertEqual(args['niter'].call_expr, '3') + + def test_extent_std_name_resolves_to_host_local(self): + """``loop=`` (e.g. host control var ``num_subcycles_for_test`` + with local name ``n_sub``) must resolve ccpp_loop_extent to the + host's local Fortran name.""" + from metadata.metadata_table import parse_metadata_file + from metadata.variable_resolver import build_flat_host_dict + host_tbls = parse_metadata_file(_sf('host_full.meta')) + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + extra_tbls = parse_metadata_file(_sf('host_subcycle_stdname.meta')) + hd = build_flat_host_dict(host_tbls + extra_tbls, ctrl_tbls, []) + xml = ( + "\n" + "\n" + " \n" + " \n" + " loop_scheme\n" + " \n" + " \n" + "\n" + ) + sr = resolve_suite(self._build_suite_from_xml(xml), + self._store(), hd) + args = self._args_by_name(sr) + self.assertEqual(args['niter'].call_expr, 'n_sub') + + def test_outside_subcycle_raises_clear_error(self): + xml = ( + "\n" + "\n" + " \n" + " loop_scheme\n" + " \n" + "\n" + ) + with self.assertRaises(CCPPError) as ctx: + resolve_suite(self._build_suite_from_xml(xml), + self._store(), _load_full_host_dict()) + msg = str(ctx.exception) + # Names the scheme + the offending std_name + the SDF remediation. + self.assertIn('loop_scheme', msg) + self.assertIn('ccpp_loop_counter', msg) + self.assertIn('\n" + "\n" + " \n" + " temp_calc_adjust\n" + " not_a_real_scheme\n" + " also_missing\n" + " \n" + "\n" + ) + suite = self._build_suite_from_xml(xml_src) + with self.assertRaises(CCPPError) as ctx: + resolve_suite(suite, store, hd) + msg = str(ctx.exception) + # Names every missing scheme. + self.assertIn('not_a_real_scheme', msg) + self.assertIn('also_missing', msg) + # Names the suite for context. + self.assertIn('bad_suite', msg) + # Points the user at --scheme-files (or the CMake equivalent). + self.assertIn('--scheme-files', msg) + + def test_unknown_scheme_in_suite_init_raises(self): + hd = _load_full_host_dict() + store = _load_scheme_store() + xml_src = ( + "\n" + "\n" + " nowhere_to_find_me\n" + " \n" + " temp_calc_adjust\n" + " \n" + "\n" + ) + suite = self._build_suite_from_xml(xml_src) + with self.assertRaises(CCPPError) as ctx: + resolve_suite(suite, store, hd) + self.assertIn('nowhere_to_find_me', str(ctx.exception)) + + class TestDedupSchemeNames(unittest.TestCase): """Unit tests for the non-run-phase dedup helper.""" diff --git a/unit-tests/test_suite_types.py b/unit-tests/test_suite_types.py new file mode 100644 index 00000000..0b9be141 --- /dev/null +++ b/unit-tests/test_suite_types.py @@ -0,0 +1,228 @@ +"""Tests for generator.suite_types — pointer-wrapper types module emitter. + +Focuses on the type-clause builder and DDT USE emission. Doctests on +the helpers themselves cover the small cases; this file exercises the +end-to-end ``_generate_suite_types`` output structure. +""" + +import unittest + +from generator.suite_types import ( + _collect_ddt_uses, + _fortran_type_str_simple, + _generate_suite_types, + _ptr_type_name, +) +from metadata.parse_tools import CCPPError + + +class TestFortranTypeStrSimple(unittest.TestCase): + + def test_intrinsic_with_kind(self): + self.assertEqual( + _fortran_type_str_simple('real', 'kind_phys'), + 'real(kind=kind_phys)', + ) + + def test_intrinsic_no_kind(self): + self.assertEqual(_fortran_type_str_simple('integer', ''), 'integer') + + def test_character_len(self): + self.assertEqual( + _fortran_type_str_simple('character', 'len=512'), + 'character(len=512)', + ) + + def test_ddt_gets_type_wrapper(self): + """The bug: a bare ``cmpfsw_type, pointer :: ptr(:)`` is invalid + Fortran. Must emit ``type(cmpfsw_type)``.""" + self.assertEqual( + _fortran_type_str_simple('cmpfsw_type', ''), + 'type(cmpfsw_type)', + ) + + def test_external_gets_type_wrapper(self): + self.assertEqual( + _fortran_type_str_simple('external:mpi_f08:mpi_comm', ''), + 'type(mpi_comm)', + ) + + def test_ddt_ignores_kind(self): + """Kind is meaningless on DDTs (no kind parameters) — silently + dropped rather than emitted as ``type(foo)(kind=...)``.""" + self.assertEqual( + _fortran_type_str_simple('my_ddt', 'kind_phys'), + 'type(my_ddt)', + ) + + +class TestPtrTypeName(unittest.TestCase): + + def test_intrinsic(self): + self.assertEqual( + _ptr_type_name('real', 'kind_phys', 1), + 'real_kind_phys_rank1_ptr_type', + ) + + def test_ddt_unchanged(self): + self.assertEqual( + _ptr_type_name('cmpfsw_type', '', 1), + 'cmpfsw_type_rank1_ptr_type', + ) + + def test_external_drops_module_prefix(self): + """External types should produce a wrapper name keyed on the + typename only — otherwise it would carry colons (illegal in a + Fortran identifier).""" + self.assertEqual( + _ptr_type_name('external:mpi_f08:mpi_comm', '', 0), + 'mpi_comm_rank0_ptr_type', + ) + + def test_character_len_is_part_of_wrapper_name(self): + """Two ``character`` arguments of different lengths must produce + DIFFERENT wrapper types — Fortran disallows ``character(len=*)`` + as a DDT component, so the wrapper must bake in the literal + length. Regression for the SCM build failure where + ``character(len=10)`` and ``character(len=3)`` both got the + same ``character_rank1_ptr_type`` name and the compiler rejected + the second declaration as a duplicate.""" + n10 = _ptr_type_name('character', 'len=10', 1) + n3 = _ptr_type_name('character', 'len=3', 1) + self.assertEqual(n10, 'character_len10_rank1_ptr_type') + self.assertEqual(n3, 'character_len3_rank1_ptr_type') + self.assertNotEqual(n10, n3) + + def test_character_len_deferred(self): + """``character(len=:), allocatable`` / ``pointer`` (deferred- + length string) is legal as a DDT component when paired with + ``pointer``. The wrapper name's len suffix must NOT contain + ``:`` (illegal Fortran identifier char); emit ``_deferred``.""" + self.assertEqual( + _ptr_type_name('character', 'len=:', 1), + 'character_len_deferred_rank1_ptr_type', + ) + + def test_character_len_parameter_symbol(self): + """``len=MY_LEN`` (a Fortran parameter constant) is a valid + identifier — keep it verbatim in the wrapper name.""" + self.assertEqual( + _ptr_type_name('character', 'len=MY_LEN', 1), + 'character_lenMY_LEN_rank1_ptr_type', + ) + + def test_character_len_assumed_rejected(self): + """``character(len=*)`` cannot appear as a DDT component, so + capgen-ng cannot synthesise a wrapper. Error must explain that + and suggest using a concrete length or deferred-length.""" + with self.assertRaisesRegex(CCPPError, 'cannot appear as a DDT component'): + _ptr_type_name('character', 'len=*', 1) + + def test_character_len_expression_rejected(self): + """Length specs that aren't identifiers (e.g. ``N+1``) can't be + encoded in a Fortran type name — error rather than emit an + illegal identifier.""" + with self.assertRaisesRegex(CCPPError, 'Fortran-identifier-safe'): + _ptr_type_name('character', 'len=N+1', 1) + + +class TestCollectDdtUses(unittest.TestCase): + + def test_intrinsics_skipped(self): + combos = {('real', 'kind_phys', 1), ('integer', '', 0)} + self.assertEqual(_collect_ddt_uses(combos, {}), {}) + + def test_ddt_grouped_by_module(self): + combos = {('cmpfsw_type', '', 1), ('cmpfsw_type', '', 2)} + ddt_map = {'cmpfsw_type': 'module_radsw_parameters'} + uses = _collect_ddt_uses(combos, ddt_map) + self.assertEqual(uses, {'module_radsw_parameters': {'cmpfsw_type'}}) + + def test_external_module_parsed_from_type(self): + combos = {('external:mpi_f08:mpi_comm', '', 0)} + uses = _collect_ddt_uses(combos, None) + self.assertEqual(uses, {'mpi_f08': {'mpi_comm'}}) + + def test_missing_ddt_in_map_raises(self): + combos = {('some_ddt', '', 1)} + with self.assertRaisesRegex(CCPPError, "no defining module"): + _collect_ddt_uses(combos, {}) + + +class TestGenerateSuiteTypesIncludesDdtUse(unittest.TestCase): + """End-to-end at the source-line level: an optional DDT-typed + argument's pointer wrapper must come with the matching ``use`` so + ``type()`` resolves at compile time, AND the declaration must + be wrapped (``type(), pointer`` — not bare ``, pointer``). + Regression for the SCM build failure on + ``ccpp_SCM_GFS_v17_p8_types.F90`` where ``cmpfsw_type, pointer :: + ptr(:)`` was emitted bare. + """ + + def test_ddt_pointer_wrapper_well_formed(self): + combos = {('cmpfsw_type', '', 1)} + ddt_map = {'cmpfsw_type': 'module_radsw_parameters'} + lines = _generate_suite_types('demo', combos, ddt_map) + text = '\n'.join(lines) + self.assertIn( + 'use module_radsw_parameters, only: cmpfsw_type', text, + "DDT USE line missing — declarations will not compile", + ) + self.assertIn( + 'type(cmpfsw_type), pointer :: ptr(:) => null()', text, + "DDT pointer declaration missing type(...) wrapper", + ) + self.assertNotIn( + '\n cmpfsw_type, pointer ::', text, + "Bare DDT name in pointer declaration — invalid Fortran", + ) + + def test_intrinsic_only_emits_no_ddt_use(self): + combos = {('real', 'kind_phys', 1)} + lines = _generate_suite_types('demo', combos, {}) + text = '\n'.join(lines) + self.assertIn('use ccpp_kinds, only: kind_phys', text) + # Sanity: only one USE line total. + self.assertEqual(text.count(' use '), 1) + + def test_distinct_character_lengths_get_distinct_wrappers(self): + """End-to-end: two ``character`` ptr-wrappers of different + lengths must emit two distinct ``type`` declarations. The + ``public ::`` list also carries both names (no duplicates). + Regression for the SCM types-module duplicate-symbol bug.""" + combos = { + ('character', 'len=10', 1), + ('character', 'len=3', 1), + } + lines = _generate_suite_types('demo', combos, {}) + text = '\n'.join(lines) + # Each length appears in exactly one ``type :: ... ptr_type`` block. + self.assertIn( + 'type :: character_len10_rank1_ptr_type', text, + ) + self.assertIn( + 'type :: character_len3_rank1_ptr_type', text, + ) + # And the declarations themselves carry the right Fortran len=. + self.assertIn( + 'character(len=10), pointer :: ptr(:) => null()', text, + ) + self.assertIn( + 'character(len=3), pointer :: ptr(:) => null()', text, + ) + # No duplicate symbol — a name appears exactly twice (``public ::`` + # line and the ``type :: NAME`` opener; ``end type NAME`` does + # not count since ``end`` precedes it). + for name in ('character_len10_rank1_ptr_type', + 'character_len3_rank1_ptr_type'): + self.assertEqual( + text.count('public :: {}'.format(name)), 1, + 'expected one public:: declaration for {}, got: {!r}'.format( + name, + [l for l in lines if name in l], + ), + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/unit-tests/test_validator.py b/unit-tests/test_validator.py index 587861a2..ce1e0cb5 100644 --- a/unit-tests/test_validator.py +++ b/unit-tests/test_validator.py @@ -109,6 +109,33 @@ def test_blank_line_between_continuation_lines(self): for tok in ('foo', 'bar', 'baz'): self.assertIn(tok, result[0]) + def test_sfc_sice_style_missing_trailing_ampersand(self): + """Fixed-form continuation where the second-to-last line has NO + trailing ``&`` but the next line has a column-6 ``&`` is a + valid F77 continuation. CCPP physics has this in the wild + (``SFC_Models/SeaIce/CICE/sfc_sice.f::sfc_sice_run``). Without + look-ahead at the next line's column-6 marker the parser ends + the logical line one step early and the closing ``)`` lands on + its own — args list never closes, signature regex captures 0 + args, validator reports a bogus arg-count mismatch.""" + src_lines = [ + ' subroutine sfc_sice_run &\n', + ' & ( im, kice, ps, t1, &\n', + ' & errmsg, errflg\n', # NO trailing ``&`` + ' & )\n', # column-6 ``&`` only + ] + result = _join_continuation(src_lines) + self.assertEqual(len(result), 1) + # Closing ``)`` must be present in the joined line. + self.assertIn(')', result[0]) + # And no stray leading ``&`` remained in the joined output. + import re as _re + self.assertIsNotNone( + _re.search(r'subroutine\s+sfc_sice_run\s*\([^)]*\)', result[0]), + 'joined signature lacks ``subroutine NAME (args)`` shape — got: {!r}' + .format(result[0]), + ) + def test_rrtmg_style_signature_round_trip(self): # The real-world failure mode that motivated the fix: a # fixed-form subroutine signature with 57 args spread across diff --git a/unit-tests/test_variable_resolver.py b/unit-tests/test_variable_resolver.py index c7aa33db..16daf6e6 100644 --- a/unit-tests/test_variable_resolver.py +++ b/unit-tests/test_variable_resolver.py @@ -173,14 +173,29 @@ def test_empty_index(self): class TestInstanceSubscript(unittest.TestCase): - def test_instance_dimension(self): - v = _make_ddt_instance_var('gs', 'gst', 'gs_type', '(instance_dimension)') - self.assertEqual(_instance_subscript(v), '(instance_number)') - def test_number_of_instances(self): v = _make_ddt_instance_var('gs', 'gst', 'gs_type', '(number_of_instances)') self.assertEqual(_instance_subscript(v), '(instance_number)') + def test_number_of_threads(self): + """Registered scalar-index dim 'number_of_threads' pairs with + the host's 'thread_number' control variable. Regression for + the per-thread DDT-container pattern (e.g. SCM's + physics%Interstitial(thread_number)).""" + v = _make_ddt_instance_var('inst', 'inst_std', 'gs_type', + '(number_of_threads)') + self.assertEqual(_instance_subscript(v), '(thread_number)') + + def test_multiple_registered_dims(self): + """A DDT instance with two registered scalar-index dims emits + both index placeholders in declared order.""" + v = _make_ddt_instance_var('foo', 'foo_std', 'foo_type', + '(number_of_instances, number_of_threads)') + self.assertEqual( + _instance_subscript(v), + '(instance_number, thread_number)', + ) + def test_scalar_no_subscript(self): v = _make_ddt_instance_var('gs', 'gst', 'gs_type', '()') self.assertEqual(_instance_subscript(v), '') @@ -330,6 +345,82 @@ def test_multiple_ddts_in_one_file(self): def test_empty(self): self.assertEqual(build_ddt_module_map([]), {}) + # ---- Precedence cases for the DDT module map ------------------------ + # Truth table (see build_ddt_module_map docstring): + # + # DDT.module_name | co-located resolved | Result + # ------------------|---------------------|-------- + # X (set) | Y (any) | X (DDT wins) + # unset | Y (any) | Y + # X (set) | (no co-located tbl) | X + # unset | (no co-located tbl) | (skipped) + # + # The "co-located resolved" column is itself the same + # ``module_name or table_name`` rule used by build_flat_host_dict. + + def test_explicit_module_name_used_when_no_colocated_table(self): + """Real-world fixture: CCPP-physics ``radsw_param.meta`` declares + ``cmpfsw_type`` in a file containing only DDT tables, where the + Fortran module name differs from any table name and is supplied + via ``module_name = …`` in ``[ccpp-table-properties]``. Without + this resolution path the suite_types emitter can't find the + module and raises CCPPError on pointer-wrapper generation.""" + ctx = _ctx() + tbl = MetadataTable('cmpfsw_type', 'ddt', 'radsw_param.meta', ctx) + tbl.module_name = 'module_radsw_parameters' + self.assertEqual( + build_ddt_module_map([tbl]), + {'cmpfsw_type': 'module_radsw_parameters'}, + ) + + def test_ddt_module_name_overrides_colocated_table_name(self): + """``module_name`` on a DDT table beats the implicit co-located + scheme/host name — the DDT may genuinely live in a different + Fortran module than the scheme its .meta is paired with.""" + ctx = _ctx() + ddt = MetadataTable('cmpfsw_type', 'ddt', 'rad.meta', ctx) + ddt.module_name = 'module_radsw_parameters' + sch = MetadataTable('radsw_main', 'scheme', 'rad.meta', ctx) + result = build_ddt_module_map([ddt, sch]) + self.assertEqual(result['cmpfsw_type'], 'module_radsw_parameters') + + def test_ddt_module_name_wins_over_colocated_module_name(self): + """Both the DDT and a co-located scheme declare module_name; the + DDT's takes precedence (most-specific-wins). Documents the + truth-table row "X / Y / X".""" + ctx = _ctx() + ddt = MetadataTable('cmpfsw_type', 'ddt', 'rad.meta', ctx) + ddt.module_name = 'module_radsw_parameters' + sch = MetadataTable('radsw_main', 'scheme', 'rad.meta', ctx) + sch.module_name = 'mod_radsw_main' + result = build_ddt_module_map([ddt, sch]) + self.assertEqual(result['cmpfsw_type'], 'module_radsw_parameters') + + def test_colocated_module_name_used_when_ddt_has_none(self): + """When the DDT has no ``module_name`` but the co-located + scheme/host carries one, the co-located ``module_name`` + (NOT its table name) wins. Documents "unset / Y / Y" where + Y comes from the co-located override. This is the bug we + fixed when refactoring build_ddt_module_map — the old code + used the co-located table_name and silently ignored its + own module_name override.""" + ctx = _ctx() + ddt = MetadataTable('inner_t', 'ddt', 'a.meta', ctx) + # No DDT override. + sch = MetadataTable('scheme_a', 'scheme', 'a.meta', ctx) + sch.module_name = 'mod_a' # Fortran module differs from table name + result = build_ddt_module_map([ddt, sch]) + self.assertEqual(result['inner_t'], 'mod_a') + + def test_colocated_table_name_used_when_neither_has_override(self): + """When neither carries module_name, the implicit "module = table + name" convention applies to the co-located scheme/host.""" + ctx = _ctx() + ddt = MetadataTable('inner_t', 'ddt', 'a.meta', ctx) + sch = MetadataTable('scheme_a', 'scheme', 'a.meta', ctx) + result = build_ddt_module_map([ddt, sch]) + self.assertEqual(result['inner_t'], 'scheme_a') + ######################################################################## # Tests: _flatten_ddt_instance @@ -754,6 +845,201 @@ def test_empty_inputs(self): ''' +class TestRule2LeafScalarDimRejection(unittest.TestCase): + """Rule 2 of the registered-scalar-index-dimension contract (see + capgen-ng/metadata/registered_dimensions.py): leaf data variables — + intrinsic-typed or external-typed, the kind a scheme binds to — + MUST NOT declare a registered scalar-index dim like + ``number_of_threads``. ``build_flat_host_dict`` is the validation + site; the error must name the variable, the offending dim, the + paired index, and point at the registered_dimensions module. + """ + + _HOST_SRC = ''' +[ccpp-table-properties] + name = host_data + type = host + +[ccpp-arg-table] + name = host_data + type = host +[ leaf_field ] + standard_name = bad_leaf + long_name = leaf with a registered scalar-index dim — illegal + units = K + dimensions = (number_of_threads, horizontal_dimension) + type = real + kind = kind_phys +''' + + def test_leaf_with_registered_dim_rejected(self): + host_tbls = _parse_lines(self._HOST_SRC.splitlines(keepends=True), + 'host_bad.meta') + with self.assertRaises(CCPPError) as ctx: + build_flat_host_dict(host_tbls, [], []) + msg = str(ctx.exception) + # Names the offending variable. + self.assertIn("'leaf_field'", msg) + # Names the offending dim. + self.assertIn("'number_of_threads'", msg) + # Names the paired index. + self.assertIn("'thread_number'", msg) + # Points at the source module for further reading. + self.assertIn('registered_dimensions.py', msg) + # Tells the user how to fix it. + self.assertIn('container DDT', msg) + + def test_leaf_with_instances_dim_rejected(self): + src = self._HOST_SRC.replace( + 'number_of_threads, horizontal_dimension', + 'number_of_instances, horizontal_dimension', + ) + host_tbls = _parse_lines(src.splitlines(keepends=True), + 'host_bad.meta') + with self.assertRaises(CCPPError) as ctx: + build_flat_host_dict(host_tbls, [], []) + msg = str(ctx.exception) + self.assertIn("'number_of_instances'", msg) + self.assertIn("'instance_number'", msg) + + def test_ddt_instance_with_non_registered_dim_skips_field_flatten(self): + """A DDT-instance variable dimensioned by a non-registered dim + (e.g. ``horizontal_dimension`` on a per-column DDT array like + ``fluxLW(horizontal_dimension)`` of type ``ty_rad_lw``) is a + legitimate pattern: schemes take the whole sliced DDT array as + a single arg, not individual flattened inner fields. + + capgen-ng must NOT flatten the inner fields in this case — + attempting to bake a scalar subscript would emit invalid + Fortran like ``parent%var%field(...)``. Instead, only the + DDT-instance's own entry is recorded; schemes that take it + whole resolve via that entry, and schemes that ask for inner + fields by std_name trip the existing "not found" error. + + Regression for the nested_suite + var_compat end-to-end fixtures + which use exactly this pattern. + """ + ddt_src = ''' +[ccpp-table-properties] + name = ty_rad_lw + type = ddt +[ccpp-arg-table] + name = ty_rad_lw + type = ddt +[ sfc_up_lw ] + standard_name = surface_upwelling_longwave_radiation_flux + units = W m-2 + dimensions = () + type = real + kind = kind_phys +''' + host_src = ''' +[ccpp-table-properties] + name = phys_state + type = host +[ccpp-arg-table] + name = phys_state + type = host +[ fluxLW ] + standard_name = longwave_radiation_fluxes + units = W m-2 + dimensions = (horizontal_dimension) + type = ty_rad_lw +''' + ddt_tbls = _parse_lines(ddt_src.splitlines(keepends=True), + 'module_rad_ddt.meta') + host_tbls = _parse_lines(host_src.splitlines(keepends=True), + 'phys_state.meta') + # No exception — the DDT-instance entry alone is enough for + # schemes that take the whole sliced DDT as an arg. + d = build_flat_host_dict(host_tbls, [], ddt_tbls) + self.assertIn('longwave_radiation_fluxes', d) + # Inner field is NOT flattened (would have required a scalar + # subscript capgen-ng can't synthesize). + self.assertNotIn('surface_upwelling_longwave_radiation_flux', d) + + def test_ddt_instance_with_non_registered_dim_no_fields_accepted(self): + """An empty DDT (no fields) dimensioned by a non-registered dim + should NOT trigger the flatten-time error — there's nothing to + flatten, so no broken access pattern is possible. Real-world + case: ``ccpp_constituent_prop_ptr_t(:)`` field on a host's + constituent object. This DDT is accessed via the dedicated + constituent resolver, not via field-flattening.""" + ddt_src = ''' +[ccpp-table-properties] + name = empty_ddt + type = ddt +[ccpp-arg-table] + name = empty_ddt + type = ddt +''' + host_src = ''' +[ccpp-table-properties] + name = my_host + type = host +[ccpp-arg-table] + name = my_host + type = host +[ payload_arr ] + standard_name = some_payload_array + units = DDT + dimensions = (number_of_ccpp_constituents) + type = empty_ddt +''' + ddt_tbls = _parse_lines(ddt_src.splitlines(keepends=True), + 'empty_ddt.meta') + host_tbls = _parse_lines(host_src.splitlines(keepends=True), + 'my_host.meta') + # Should not raise. + result = build_flat_host_dict(host_tbls, [], ddt_tbls) + self.assertIn('some_payload_array', result) + + def test_container_ddt_with_registered_dim_accepted(self): + """The same dim on a DDT-instance container variable is fine — + Rule 2 only applies to leaves.""" + src = ''' +[ccpp-table-properties] + name = my_ddt + type = ddt + +[ccpp-arg-table] + name = my_ddt + type = ddt +[ field ] + standard_name = inner_field + units = K + dimensions = (horizontal_dimension) + type = real + kind = kind_phys +''' + host_src = ''' +[ccpp-table-properties] + name = host_data + type = host + +[ccpp-arg-table] + name = host_data + type = host +[ inst_array ] + standard_name = instance_array + units = DDT + dimensions = (number_of_threads) + type = my_ddt +''' + ddt_tbls = _parse_lines(src.splitlines(keepends=True), 'ddt.meta') + host_tbls = _parse_lines(host_src.splitlines(keepends=True), + 'host.meta') + # Should not raise — the dim is on a container. + result = build_flat_host_dict(host_tbls, [], ddt_tbls) + self.assertIn('inner_field', result) + # The flattened field's access path carries the (thread_number) + # placeholder. + self.assertEqual( + result['inner_field'].access_path, + 'inst_array(thread_number)%field', + ) + + class TestSchemeStore(unittest.TestCase): def _build(self, src=_SIMPLE_SCHEME_SRC): From f2c5918597179a807ff1a15da465828c58cb184e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 13 May 2026 20:36:16 -0600 Subject: [PATCH 12/74] Update doc/* and add doc/briefing.md --- doc/briefing.md | 365 +++++++++++++++++++++++++++++++++++ doc/constituents_overhaul.md | 2 +- doc/migration.md | 192 +++++++++++++++--- doc/redesign_prompt.md | 93 +++++++-- 4 files changed, 615 insertions(+), 37 deletions(-) create mode 100644 doc/briefing.md diff --git a/doc/briefing.md b/doc/briefing.md new file mode 100644 index 00000000..237ef706 --- /dev/null +++ b/doc/briefing.md @@ -0,0 +1,365 @@ +# capgen-ng — Briefing for CCPP Framework Developers & Power Users + +*Prepared for the 2026-05-14 walk-through. Companion document to +`doc/migration.md` (the detailed migration guide) and +`doc/redesign_prompt.md` (the implementation spec).* + +--- + +## 1. Why a new generator? + +The CCPP Framework runs two code generators today: + +- **`ccpp-prebuild`** — simple, procedural Python; fast; DDT-argument + passing; in production use by NOAA UFS Weather Model, Navy NEPTUNE, + and CCPP-SCM. Reliable but feature-light. No framework-owned + variables. +- **`ccpp-capgen`** — complex, deeply object-oriented Python; flat-field + argument passing; in use by NCAR CAM-SIMA. Many advanced features + designed but never implemented; flat-field passing infeasible at + UFS/NEPTUNE scale (1200+ variables, breaks under `-check all`); + nobody on the team fully understands it. + +**`capgen-ng`** starts fresh, drawing lessons from both. Guiding +principle: **simplicity of prebuild, feature set of capgen**. + +What we wanted to fix: + +1. **Flat fields → DDT arguments at all scales.** No "flat-field + group cap" failure mode. +2. **No scope-chain variable promotion.** Variables flow through + metadata, not through a runtime synthetic dictionary stacking. +3. **Code anyone can read and extend.** No 10-deep class hierarchy. +4. **One generator, one CLI, one query tool** for both prebuild-style + and capgen-style hosts. + +--- + +## 2. What capgen-ng is (in one paragraph) + +capgen-ng reads metadata for the **host model**, the **physics +schemes**, and the **suite definition files** (SDFs), produces a +small set of Fortran cap modules that bridge them, and writes a +`datatable.xml` describing the result for CMake / Make to consume. +At runtime the host calls a small set of public entry points +(`ccpp_register`, `ccpp_init`, `ccpp_physics_init`, +`ccpp_physics_run`, `ccpp_physics_*_init`/`_final`, `ccpp_final`); the +generated caps dispatch by `suite_name` (and optionally `group_name`) +to the right scheme. + +--- + +## 3. Core concepts + +### 3.1 Five metadata table types + +| `type = ` | Owner | How it reaches the cap | +|-------------|------------------|----------------------------| +| `scheme` | Physics scheme | Intent args on scheme subs | +| `host` | Host model | Module USE (direct / DDT) | +| `control` | Framework runtime layer | `ccpp_physics_*` args | +| `ddt` | Type definition | Structural — fields only | +| `suite` | Generated suite cap | Module USE | + +### 3.2 Three layers of generated cap + +- **Static API** (`ccpp_static_api.F90`) — public entry points; one per + build. Dispatches by `suite_name` → suite cap. +- **Suite cap** (`ccpp__cap.F90`) — per-suite state machine, plus + dispatch by `group_name` → group cap. Suite-owned interstitial data + lives in a sibling `ccpp__data.F90`. +- **Group cap** (`ccpp___cap.F90`) — scheme call sites + with full argument lists, unit/kind/vertical-flip transforms, + optional-arg pointer wrappers, subcycle `do` loops. + +### 3.3 Two-level integer state machine + +Replaces both the boolean `initialized(:)` array from prebuild and +the string-based `ccpp_suite_state` from CAM-SIMA capgen. Per +instance: + +- **Suite-level**: `UNREGISTERED → REGISTERED → FRAMEWORK_INITIALIZED`. +- **Group-level**: `UNINITIALIZED → INITIALIZED → IN_TIMESTEP`. + +### 3.4 Six scheme phases + +`register`, `init`, `timestep_init`, `run`, `timestep_final`, `final`. +`register` is new — schemes that contribute to the constituent table +do so here. `final` replaces the older `finalize` (breaking change, +intentional). + +### 3.5 Variable resolution + +For each scheme arg: + +1. Found in host+control metadata → use the access path. If units / + kind / vertical orientation differ, generate a transform. +2. Not found, first use is `intent(out)` → **suite-owned** variable + (interstitial); add to `ccpp__data.F90`. +3. Not found, first use is `intent(in/inout)` → **error**. +4. Found in suite data (a prior scheme provided it) → use suite data + access path. + +### 3.6 Two tools, one parser + +- `ccpp_capgen_ng.py` — the code generator. Trusts metadata; no + Fortran parsing. +- `ccpp_validator.py` — the standalone Fortran-vs-metadata checker. + The ONE place capgen-ng parses Fortran. Run by developers / + CMake before generation. + +Both share the same metadata-parsing library (`metadata/`). + +--- + +## 4. How capgen-ng differs from `ccpp-prebuild` + +| Topic | prebuild | capgen-ng | +|-----------------------------|-----------------------------------|---------------------------------------------------| +| Host metadata mechanism | Hard-coded Python dict (`TYPEDEFS_NEW_METADATA`) | Regular `type = ddt` + `type = host` tables | +| Framework-owned variables | Not supported | First-class (suite-owned interstitial via Case 2) | +| Constituents | Hand-rolled, host-specific glue | Standardised opt-in mechanism with auto-provision | +| `register` phase | Doesn't exist | First phase; schemes declare dynamic constituents | +| Multi-instance API | Implicit, ad-hoc | Paired-opt-in (`instance_number` / `number_of_instances`) | +| Subcycle loop counter | Host plumbs it manually | Registered std names `ccpp_loop_counter` / `ccpp_loop_extent` resolve to the do-loop locals automatically inside `` | +| Suite introspection | Limited | Five runtime queries (`ccpp_physics_suite_list`, `_part_list`, `_schemes`, `_variables`, `_host_data`) | + +--- + +## 5. How capgen-ng differs from `ccpp-capgen` + +| Topic | capgen | capgen-ng | +|-----------------------------|-----------------------------------|---------------------------------------------------| +| Group-cap arguments | Flat fields (1200+ at UFS scale) | DDT arguments (as in prebuild) | +| Variable matching algorithm | Scope-chain promotion | Flat host+control dict + suite-owned discovery | +| `type = module` in metadata | Yes | Renamed `type = host` | +| `is_constituent` scheme args | Auto-cloned by generator | Schemes register constituents explicitly in the `register` phase via `ccpp_constituent_properties_t(:)` | +| `ConstituentVarDict` | Synthetic scope between suite + host | Removed; constituents are one of four sources (`control`/`host`/`suite`/`constituent`) on `ResolvedArg` | +| `_state` runtime check | String | Integer (named parameters) | +| Fortran-vs-metadata check | Inside the generator | Separate tool (`ccpp_validator.py`) | +| Code complexity | Deep OO hierarchy | Flat data classes + procedural resolver | + +--- + +## 6. Breaking metadata changes hosts must make + +Comprehensive list — see `doc/migration.md` for full detail. + +### 6.1 Table types + +- `type = module` → **`type = host`**. + +### 6.2 Phase names + +- `_finalize` → **`_final`** in both metadata and + Fortran source. + +### 6.3 Standard names + +- `horizontal_loop_extent` → **`horizontal_dimension`** uniformly in + scheme metadata. (The chunk-vs-full-domain distinction is driven + by what the host passes for `horizontal_loop_begin` / + `horizontal_loop_end`.) +- `number_of_openmp_threads` → **`number_of_threads`** (matches the + `thread_number` control variable convention). + +Both are rewritten on the fly by **`--legacy-mode`** for a transition +period; the shim prints a banner listing every rewrite it performs +and is marked for clean removal. + +### 6.4 Required host `type = control` table + +Every host MUST declare scalar integers (and one character) with +these CCPP standard names: + +- `suite_name`, `horizontal_loop_begin`, `horizontal_loop_end`, + `thread_number`, `number_of_threads`, `number_of_physics_threads`, + `ccpp_error_code`, `ccpp_error_message`. + +Optional (paired): `instance_number` (control) + +`number_of_instances` (host). + +### 6.5 DDT-instance variables with scalar-index dims + +Container DDT-instance variables (`physics%Interstitial`, +`physics%Coupling`, ...) dimensioned by a count standard name +(`number_of_threads`, `number_of_instances`) get their scalar index +inserted **automatically** by capgen-ng. The host metadata declares +the dim; the generator emits +`physics%Interstitial(thread_number)%alpha(...)` at every call site. + +The host's Fortran can keep its existing OpenMP-thread-private DDT +layout — no glue code needed on the host side. + +### 6.6 Leaf variables MUST NOT carry registered scalar-index dims + +Rule 2 of the registered-scalar-index-dimension contract: scalar +variables (real / integer / character / DDT-typed leaves the scheme +binds to) cannot declare `number_of_threads` or `number_of_instances` +as a dimension. Wrap them in a container DDT instead. This is +enforced at parse time with an explicit remediation message; existing +CCPP-physics, UFS-WM, and CAM-SIMA host metadata is already +compliant. + +### 6.7 No more `cdata` / `ccpp_t` struct passing + +The framework-owned bag-of-state struct is replaced by explicit +control-variable arguments to the public entry points. + +--- + +## 7. What capgen-ng does NOT support (yet) + +### 7.1 Deferred — to be resolved in upcoming work + +- **Constituents overhaul.** Three reform proposals on the table + (`doc/constituents_overhaul.md`); decision pending an upcoming + meeting. Pieces involved: framework setter additions + (`set_advected`, `set_diagnostic_name`, `set_default_value`), + `is_match` relaxation, Class A vs Class B property classification. +- **Validator host-metadata check.** `ccpp_validator.py` currently + validates scheme metadata only; host-metadata-vs-Fortran is on + hold until the e2e test suite settles. +- **Codegen-time scheme-registration cross-check.** Today's + registration check is at runtime + (`ccpp_initialize_constituents`). Stronger options: new metadata + attribute `registers_std_names = a, b, c` on register-phase + tables; cross-check at codegen. +- **Nested-subcycle `ccpp_loop_counter` semantics.** When a scheme + inside a deeply nested subcycle asks for `ccpp_loop_counter`, it + currently resolves to the **outermost** loop's counter. None of + the in-tree physics catalogs uses the inner-counter case. +- **`ccpp_datafile.py --host-files` repurpose.** The current + `--host-files` returns the generated host-API file; should be a + filtered list of *input* host metadata files (parallel to the new + `--scheme-files`). Deferred. +- **`ccpp_host_constituents.F90` suppression** when no suite touches + constituents (file is correct-but-empty under host-wins; should + not be emitted at all). +- **Python linter / formatter pass.** Pick `ruff`, apply across + `capgen-ng/`. + +### 7.2 Intentionally NOT supported + +- **`_finalize` phase spelling.** Use `_final`. No legacy-mode + shim — rename in metadata + Fortran. +- **`type = module`.** Use `type = host`. +- **Flat-field scheme call arguments** (capgen's failure mode). +- **`character(len=*)` as a DDT component** (Fortran disallows it; + we error at parse time with a remediation pointing at + `character(len=:)` deferred-length). +- **Multiple registration sources for the same constituent** with + silent dedup. Today's behaviour is to error on conflict; the + proposed reform sets a clear precedence rule (host-set Class B + properties win) — pending the constituents-overhaul decision. +- **`ConstituentVarDict`** synthetic scope between suite and host. + Gone for good. + +--- + +## 8. Validation and error reporting + +A deliberate design choice across capgen-ng: **errors are loud, +specific, and actionable**. Examples surfaced during the SCM +shake-down: + +- Empty `units =` line → error names file, line, variable, + attribute, raw value, AND inner reason. +- Scheme metadata file passed via `--scheme-files` but missing from + the SDF → silently ignored (and dropped from `` so + CMake doesn't compile orphan code). +- Scheme listed in the SDF but its metadata not supplied → single + CCPPError listing every missing scheme + pointer to + `--scheme-files`. Replaces silent empty-cap emission. +- DDT-instance variable with a non-registered scalar-index dim AND + flattenable fields → error shows the broken access pattern + capgen-ng WOULD have emitted and quotes the Fortran compiler + error verbatim ("Component to the right of a part reference with + nonzero rank must not have the POINTER attribute"). +- Generated `case default` on `select case(suite_name)` / + `select case(group_name)` → unknown suite or group at runtime + produces a clear errflg + errmsg, not silent fall-through. + +--- + +## 9. Build-system integration (capsule view) + +```cmake +# In your CMakeLists.txt +set(SCHEME_METADATA_FILES …list of .meta paths…) +set(HOST_METADATA_FILES …list of host .meta paths…) +set(SUITE_FILES …list of suite XML paths…) + +# Validate before generation (developer step, optional in CI) +ccpp_validator(SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) + +# Run the code generator +ccpp_capgen(HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT ${OUTPUT_ROOT}) + +# Pull the manifest from the datatable +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES ${CCPP_FILES}) + +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) + +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +add_library(scm-ccpp STATIC + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES}) +``` + +Regenerating on every CMake configure is cheap — `write_if_changed` +preserves mtimes when content hasn't changed, so `make` / `ninja` +don't rebuild downstream objects unless something actually moved. + +--- + +## 10. Where things stand right now + +- **Unit tests**: 1208 passing on `main`. +- **End-to-end tests passing**: `advection`, `unit_conv`, + `nested_suite`, `variable_transform`, `instances`, `ddt`. +- **CCPP-SCM**: actively driving development this week — every build + / runtime failure surfaced this week landed as a fix in capgen-ng + (rather than being patched around in the host). Most of the + `phys_ps` group now builds end-to-end via `--legacy-mode`. +- **CAM-SIMA**: not yet reconnected; pending the constituents + overhaul decision. +- **UFS Weather Model / NEPTUNE**: not yet attempted; SCM is the + proving ground first. + +--- + +## 11. Walk-through outline (suggested order for the meeting) + +1. Live `ccpp_capgen_ng.py --help` (CLI shape). +2. Show one scheme's `.meta` + its generated group-cap fragment. +3. Run the generator twice — note the `Unchanged: …` messages on the + second pass (write-if-changed in action). +4. Run `ccpp_datafile.py --scheme-files datatable.xml` to show the + filtered manifest. +5. Demonstrate a deliberately-broken metadata (`units =` empty, or + missing scheme, or invalid `case default` group) to show the + error UX. +6. Walk through the registered scalar-index dimension table and the + two rules. +7. Open the floor — focus areas for the audience: + - **Host metadata maintainers**: anything in §6 that surprises + you for your model? + - **Scheme metadata maintainers**: anything in §6.2 / §6.3 that + can't be migrated cleanly? + - **Framework devs**: §7.1 — which deferred items block your + downstream work? diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index 2adb34dd..04c6b8c2 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -33,7 +33,7 @@ but it carries: have no setters; `is_match` is overly strict about properties hosts should be free to change. - **Two registration models** coexist — original capgen's auto-clone of - is_constituent scheme args, and capgen-ng's explicit register-phase + + is_constituent scheme args, and capgen's/capgen-ng's explicit register-phase + host-side declaration. Capgen-ng deliberately dropped auto-clone. This document is a structured brief for a discussion this week. It does diff --git a/doc/migration.md b/doc/migration.md index 58c951f6..d222c6f3 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -6,7 +6,7 @@ Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to **capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and `doc/redesign_analysis.md` (analysis of the old systems). -*Last revised: 2026-05-13.* Current unit-test suite: 1127 passing. +*Last revised: 2026-05-13 (late evening).* Current unit-test suite: 1208 passing. **Repository layout** (post-2026-05-13 cleanup): tooling lives under `capgen-ng/` (top-level of this repo). Unit tests live at the top @@ -125,25 +125,46 @@ These two control variables are now **paired optional**: Hosts that don't need multi-instance bookkeeping can drop both declarations. -### 1.8 `horizontal_loop_extent` → `horizontal_dimension` +### 1.8 Deprecated standard names rewritten by `--legacy-mode` -ccpp-prebuild / original ccpp-capgen used `horizontal_loop_extent` as -the horizontal-axis std name in scheme metadata. capgen-ng uses -`horizontal_dimension` uniformly — the run-vs-non-run distinction -isn't expressed in scheme metadata anymore (host passes -`horizontal_loop_begin`/`horizontal_loop_end` as control vars and the -generated cap slices accordingly). +`--legacy-mode` is a transient migration shim that rewrites a small +set of deprecated standard names to their canonical capgen-ng +equivalents at parse time. The full table currently covers: + +| Deprecated (legacy) | Canonical (capgen-ng) | +|--------------------------------|--------------------------| +| `horizontal_loop_extent` | `horizontal_dimension` | +| `number_of_openmp_threads` | `number_of_threads` | + +Why each entry: + +* `horizontal_loop_extent` — ccpp-prebuild / original ccpp-capgen used + this for the horizontal-axis std name in scheme metadata. capgen-ng + uses `horizontal_dimension` uniformly; the run-vs-non-run distinction + isn't expressed in scheme metadata anymore (host passes + `horizontal_loop_begin` / `horizontal_loop_end` as control vars and + the generated cap slices accordingly). +* `number_of_openmp_threads` — legacy CCPP-physics hosts (CCPP-SCM + 17p8 in particular) size per-thread DDT containers by + `number_of_openmp_threads` (e.g. `physics%Interstitial`). capgen-ng + uses `number_of_threads`, which matches the `thread_number` control + variable, so the registered scalar-index dim table can substitute + `physics%Interstitial(thread_number)%…` automatically (see §3.4). + +The rewrite fires for both standard-name attributes AND dimension +tokens (so a host's `dimensions = (number_of_openmp_threads)` becomes +`dimensions = (number_of_threads)` before any further processing). Migration paths: -1. **Edit the metadata** (recommended) — search-and-replace - `horizontal_loop_extent` → `horizontal_dimension` in every scheme - `.meta` you maintain. +1. **Edit the metadata** (recommended) — search-and-replace the + legacy names in every host / scheme `.meta` you maintain. 2. **Use `--legacy-mode`** (transient) — pass `--legacy-mode` to both - `ccpp_capgen_ng.py` and `ccpp_validator.py` and the rename happens - at parse time. A loud warning banner prints at startup so the - rewrite is never invisible. This shim *will be removed*; treat - it as a runway, not a destination. + `ccpp_capgen_ng.py` and `ccpp_validator.py` and the renames happen + at parse time. A loud warning banner prints at startup, listing + every pair the shim is rewriting, so the substitution is never + invisible. This shim *will be removed*; treat it as a runway, + not a destination. --- @@ -194,6 +215,63 @@ counter variables follow the convention: Effective iteration count = product of every level's `loop=` value. `effr_calc` in the example runs 3·2 = 6 times. +### 2.3.1 Passing the loop counter / extent to a scheme + +A scheme inside a `` block may consume the current iteration +counter and the total iteration count via two CCPP standard names: + +| Standard name | Fortran type | Meaning | +|----------------------|--------------|----------------------------------------------------------| +| `ccpp_loop_counter` | integer | Current subcycle iteration (1 … `ccpp_loop_extent`) | +| `ccpp_loop_extent` | integer | Total iterations — the `loop=` value on the `` | + +These are **loop-context control variables**: the host model does **not** +declare them. capgen-ng emits them automatically as locals in the +generated group cap (the `do` loop's induction variable for the counter, +the loop bound for the extent), and resolves any scheme arg requesting +them against those locals. + +Example scheme metadata fragment: + +``` +[iter] + standard_name = ccpp_loop_counter + units = index + dimensions = () + type = integer + intent = in +[niter] + standard_name = ccpp_loop_extent + units = index + dimensions = () + type = integer + intent = in +``` + +Place the scheme in a `` in the SDF: + +```xml + + sfc_diff + GFS_surface_loop_control_part1 + sfc_nst + +``` + +The generated group cap will emit `do ccpp_loop_counter = 1, 2` and call +the scheme with `iter = ccpp_loop_counter, niter = 2` (or the loop's +resolved local name when `loop=` is used). + +**Scope is the subcycle body.** A scheme that requests +`ccpp_loop_counter` / `ccpp_loop_extent` but is NOT inside a +`` block raises a clear parse-time error pointing at this +contract. + +**Nested-subcycle nuance** (see §8): nested-subcycle schemes that ask +for `ccpp_loop_counter` currently get the **outermost** loop's counter, +not the innermost. None of the in-tree physics catalogs use the +inner-counter case yet; revisit when one needs it. + ### 2.4 Suite-level `` and `` schemes ```xml @@ -289,6 +367,56 @@ typically named after the table. When that's not the case, use the module_name = mod_test_host_data ``` +### 3.4 Registered scalar-index dimensions + +A small set of CCPP standard-name dimensions are *registered*: each +one is a count that capgen-ng auto-collapses to a paired scalar index +variable at every access site. + +| Count dim (in `dimensions = (...)`) | Index var (capgen-ng substitutes) | +|---|---| +| `number_of_instances` | `instance_number` | +| `number_of_threads` | `thread_number` | + +**Where these may appear**: ONLY on container DDT-instance variables in +the access path. Example: + +``` +[Interstitial] + standard_name = GFS_interstitial_type_instance + type = GFS_interstitial_type + dimensions = (number_of_threads) +``` + +Every scheme that reaches into `Interstitial%` will see the +generator emit `physics%Interstitial(thread_number)%` at the +call site — no metadata work required on the scheme side. + +**Two rules govern this:** + +1. *(generalized)* A container DDT-instance variable may carry any + registered scalar-index dim — single (`(number_of_threads)`) or + paired (`(number_of_instances, number_of_threads)`). Dims that + AREN'T registered flow through the normal slice machinery + (`horizontal_loop_begin:horizontal_loop_end`, `1:vertical_*`, …) + just like flat-array dims. +2. *(enforced — hard parse-time error)* A **leaf** variable + (intrinsic-typed or `external:` — the kind a scheme binds to) + **MUST NOT** declare a registered scalar-index dim. If you write:: + + [my_array] + type = real | kind = kind_phys + dimensions = (number_of_threads, horizontal_dimension) # ILLEGAL + + capgen-ng will reject it at parse time with a message pointing + at the wrap-in-DDT remediation pattern. Wrap the leaf in a + container DDT instead. + +The registered table lives in +[`capgen-ng/metadata/registered_dimensions.py`](../capgen-ng/metadata/registered_dimensions.py). +It carries a four-step recipe at the top of the file for adding new +pairings. + --- ## 4. Generator CLI and build integration @@ -313,16 +441,18 @@ and the module defaults to `iso_fortran_env`. `kind_phys` is auto-defaulted to `iso_fortran_env:REAL64` when not supplied. `--legacy-mode` (transient migration shim, will be removed): silently -rewrites legacy CCPP standard names that ccpp-prebuild / original -ccpp-capgen used to their capgen-ng equivalents at parse time. -Currently translates `horizontal_loop_extent` → `horizontal_dimension`. -Prints a loud warning banner at startup so the rewrite is never -invisible. Available on both `ccpp_capgen_ng.py` and `ccpp_validator.py` -(keep the flag consistent between the two when both are invoked from -CMake). All translation logic is isolated in -`metadata/legacy_compat.py` and tagged with `# legacy-compat:` comments -at every touchpoint, so the shim can be cleanly removed when migration -is complete. +rewrites a small set of deprecated CCPP standard names to their +capgen-ng equivalents at parse time — see §1.8 for the full table +(`horizontal_loop_extent` → `horizontal_dimension`, +`number_of_openmp_threads` → `number_of_threads`). The rewrite fires +for both standard-name attributes AND dimension tokens. Prints a +loud warning banner at startup, enumerating every pair the shim is +rewriting, so the substitution is never invisible. Available on both +`ccpp_capgen_ng.py` and `ccpp_validator.py` (keep the flag consistent +between the two when both are invoked from CMake). All translation +logic is isolated in `metadata/legacy_compat.py` and tagged with +`# legacy-compat:` comments at every touchpoint, so the shim can be +cleanly removed when migration is complete. ### 4.2 `ccpp_datafile.py` query CLI @@ -362,6 +492,18 @@ query stays useful. `ccpp_capgen(...)` and `ccpp_validator(...)` macros. `ccpp_datafile(...)` queries datatable.xml at configure time. +### 4.4 No-op regeneration preserves mtimes + +Every generated file (caps, `datatable.xml`, `ccpp_kinds.F90`, expanded +SDFs, `.meta` artifacts) goes through `write_if_changed`: the new content +is staged to a sibling temp file under the output root and atomically +replaces the target only when the bytes actually differ. Reruns with +identical inputs therefore leave on-disk mtimes untouched, so CMake / +Make / Ninja do not trigger a downstream rebuild cascade. Matches the +behaviour of legacy `ccpp-prebuild` / `ccpp-capgen`. The staging temp +file lives in the target's parent directory (always under +`--output-root`), so no `/tmp` access is required. + --- ## 5. Generated cap layout — what's new and what changed diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index 0ac76cde..4b33d5f2 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -1,6 +1,6 @@ # CCPP Framework Code Generator — Redesign Specification -*Last revised: 2026-05-13.* +*Last revised: 2026-05-13 (late evening — SCM-driven session).* ## Purpose @@ -277,7 +277,7 @@ The generator has built-in semantic knowledge of these dimension standard names: | Standard name | Indexing semantic | |---|---| -| `instance_dimension` | Scalar extraction: `var(instance_number)` — `instance_number` is the control variable | +| Any key of `SCALAR_INDEX_DIMS` (currently `number_of_instances`, `number_of_threads`) | Scalar extraction: substitute the paired index variable's local Fortran name (currently `instance_number`, `thread_number`). See `capgen-ng/metadata/registered_dimensions.py` for the full table and the contract. | | `horizontal_dimension` | **At scheme call sites**: always `horizontal_loop_begin:horizontal_loop_end` (using control variable local names), for all phases. **For suite-owned array allocation sizing**: local name of `horizontal_dimension` from the host `type=host` table (accessed via module USE, not the control variable). | | `vertical_*` | Slice: `1:` | @@ -289,10 +289,11 @@ dimensions — by looking up the variable with that standard name in the host me All other dimension standard names are resolved identically: look up the variable with that standard name, get its local Fortran name, emit `1:local_name`. -The timing of `instance_dimension` substitution — whether at parse time (when building -the flat dict access path) or at call-string generation time (like other registered -dimensions) — is an implementation decision left to the developer. Either is correct; -choose whichever is easier to implement, understand, and maintain. +Registered scalar-index dims are subject to one hard contract (Rule 2 in +`metadata/registered_dimensions.py`): they may appear only on **container +DDT-instance variables** in the access path, never on leaf data variables +(intrinsic- or `external:`-typed). Leaves that declare them are rejected +at parse time with a remediation pointer. See `doc/migration.md` §3.4. --- @@ -676,7 +677,12 @@ is maintained during code generation. For each variable, the generator constructs the call-site expression by applying dimension rules to each dimension in order: -1. **`instance_dimension`** → substitute `instance_number` (scalar extraction) +1. **Registered scalar-index dim** (key in `SCALAR_INDEX_DIMS`; currently + `number_of_instances` → `instance_number`, + `number_of_threads` → `thread_number`) → scalar extraction using the + paired index variable's local Fortran name. Only permitted on + container DDT-instance variables, never on leaves (Rule 2; see + `capgen-ng/metadata/registered_dimensions.py`). 2. **`horizontal_dimension`** → always substitute `horizontal_loop_begin:horizontal_loop_end` (using control variable local names) at scheme call sites. For suite-owned array allocation sizing, `horizontal_dimension` from the host `type=host` table is used directly. @@ -1166,7 +1172,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` SCM) keep using their own short local names (e.g. `ntcw`) without blowing Fortran's 63-char identifier limit. -### Landed 2026-05-13 +### Landed 2026-05-13 (morning + afternoon) - **`--legacy-mode` shim** — transient parse-time rewrite of legacy CCPP standard names (`horizontal_loop_extent` → @@ -1183,12 +1189,77 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` the canonical lowercase host_dict keys (Fortran is case-insensitive, so embedded logical operators are unaffected). +### Landed 2026-05-13 (late evening — SCM-driven session) + +- **`SCALAR_INDEX_DIMS` registered table** — `metadata/registered_dimensions.py` + carries the single source of truth for count-dim ↔ scalar-index pairings + (`number_of_instances → instance_number`, `number_of_threads → thread_number`). + Drops the old `instance_dimension` placeholder. Rule 2 (leaves never carry + registered dims) enforced at parse time with rich error messages. +- **Loop-context resolver wired** — scheme args declaring + `ccpp_loop_counter` / `ccpp_loop_extent` resolve inside `` to + the generated do-loop locals (or the loop's literal/host-resolved + bound for the extent). Outside-subcycle raises a clear parse-time + error pointing at the SDF contract. +- **Write-if-changed** — every generated cap file goes through + `metadata/parse_tools/io_helpers.py::write_if_changed`; unchanged + files keep their mtime so CMake/Make/Ninja don't trigger a rebuild + cascade on regenerate. Staging temp lives next to the target under + `--output-root`, never `/tmp`. Logger emits `"Wrote …"` vs + `"Unchanged: …"`. +- **`--scheme-files` query** — `datatable.xml` carries a `` + section listing the user-supplied scheme `.F90` source paths actually + referenced by some loaded suite (group phases + suite-level + `` / `` hooks). Companion `` filter applied + to scheme tables (host/control/ddt deps still flow unconditionally). +- **`ccpp__cap.F90` group dispatch + `ccpp_physics_*` suite + dispatch case-default** — unknown `group_name` / `suite_name` now + sets `errflg=1` and writes a clear message, no silent fall-through. +- **Missing-scheme parse-time detection** — `resolve_suite` walks every + scheme reference in the SDF (group phases + `` + ``) + and raises with the full list when any aren't in the scheme store. + Replaces silent empty-group-cap emission. +- **Validator continuation look-ahead** — `_join_continuation` now + detects continuation when the *current* line has no trailing `&` + but the next line has a column-6 `&` marker (fixed-form F77). + Fixes `sfc_sice.f::sfc_sice_run` and similar legacy CCPP-physics + signatures. +- **Metadata error enrichment** — `MetaVar.set_attr` wraps every + `check_X` helper failure with variable name + attribute name + raw + value + source location, so `'' is not a valid unit` becomes + actionable across a 60+ file load. +- **Character pointer-wrapper name encodes len** — `_ptr_type_name` + bakes the length into the wrapper name so two `character(len=N)` + args of different lengths don't collide on a single + `character_rank1_ptr_type` symbol. `len=:` → `_deferred`; `len=*` + rejected; unparseable lengths rejected. +- **DDT-instance non-registered-dim diagnostic** — a DDT-instance + variable with dims none of which are registered scalar-index AND + with flattenable fields raises at parse time with the concrete + would-be-broken access pattern. Empty DDTs (e.g. + `ccpp_constituent_prop_ptr_t`) flow through unchanged. +- **`build_ddt_module_map` honors per-DDT `module_name` override** — + CCPP-physics `radsw_param.meta` declares `cmpfsw_type` in a + scheme-less file with explicit `module_name`; previously skipped. + Precedence: DDT's own `module_name` wins > co-located non-DDT + table's resolved module > skipped. +- **`_resolve_single_bound` substitutes scalar-idx placeholders** — + dim bounds that resolve through a per-thread/per-instance DDT + field no longer leak the std-name placeholder in nested + subscripts. +- **Legacy-mode adds second pair** — `--legacy-mode` now also + rewrites `number_of_openmp_threads → number_of_threads`. Banner + enumerates every pair automatically; no hard-coded text per + pairing. + ### Test status -- **Unit tests**: 1127 passing (`python -m pytest unit-tests/`). +- **Unit tests**: 1208 passing (`python -m pytest unit-tests/`). - **End-to-end tests**: `advection`, `unit_conv`, `nested_suite`, - `variable_transform` covered. Tree is off-limits for in-session - edits — user-driven. + `variable_transform`, `instances`, `ddt` covered. SCM running + against ccpp-physics is the active driver right now — most of the + late-evening landings were surfaced by SCM build/runtime failures. + Tree is off-limits for in-session edits — user-driven. ### Still deferred From 2a9892be53425bd877137ecc66e8c5d970f07186 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 May 2026 11:41:45 -0600 Subject: [PATCH 13/74] Temporarily add end-to-end-tests.sh --- end-to-end-tests.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 end-to-end-tests.sh diff --git a/end-to-end-tests.sh b/end-to-end-tests.sh new file mode 100755 index 00000000..c930887f --- /dev/null +++ b/end-to-end-tests.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +rm -fr build/* +cd build +cmake ../end-to-end-tests +make +ctest +cd .. From 8fd7b3ecc8b23284c4d6c10ed42177b4f22a7391 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 May 2026 11:43:20 -0600 Subject: [PATCH 14/74] Update capgen-ng and unit-tests: all GFS v17 p8 suites in SCM --- capgen-ng/ccpp_capgen_ng.py | 49 ++- capgen-ng/ccpp_validator.py | 85 ++++- capgen-ng/generator/datatable.py | 66 ++-- capgen-ng/generator/group_cap.py | 108 +++--- capgen-ng/generator/host_constituents.py | 12 +- capgen-ng/generator/static_api.py | 301 ++++++++++++----- capgen-ng/generator/suite_cap.py | 130 +++---- capgen-ng/generator/suite_data.py | 75 +++-- capgen-ng/generator/suite_resolver.py | 138 ++++---- capgen-ng/generator/suite_types.py | 8 +- capgen-ng/generator/suite_xml.py | 10 +- unit-tests/test_ccpp_datafile.py | 14 +- unit-tests/test_datatable.py | 22 +- unit-tests/test_host_constituents.py | 30 +- unit-tests/test_integration.py | 22 +- unit-tests/test_static_api.py | 254 +++++++++++--- unit-tests/test_suite_cap.py | 48 +-- unit-tests/test_suite_data.py | 8 +- unit-tests/test_suite_resolver.py | 410 +++++++++++------------ unit-tests/test_suite_xml.py | 52 +-- unit-tests/test_validator.py | 69 ++++ 21 files changed, 1173 insertions(+), 738 deletions(-) diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index 1856ff48..7a947b7e 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -248,6 +248,22 @@ def _build_arg_parser() -> argparse.ArgumentParser: "loud warning at startup. Will be removed." ), ) + parser.add_argument( + '--no-host-introspection', + action='store_true', + help=( + "Stub the five suite-introspection routines in " + "ccpp_static_api.F90 (ccpp_physics_suite_list / " + "suite_part_list / suite_schemes / suite_variables / " + "suite_host_data). Signatures remain so callers still " + "link, but bodies set errflg=1 with a clear errmsg " + "(suite_list, which has no errflg, writes to error_unit " + "and returns an empty list). Use this to shrink the " + "generated ccpp_static_api.F90 from ~33000 lines to ~800 " + "for multi-suite builds where -O3 cannot finish compiling " + "the introspection case-blocks." + ), + ) return parser @@ -779,6 +795,7 @@ def capgen( output_root: str, kind_types: Dict[str, Tuple[str, str]], logger: Optional[logging.Logger] = None, + no_host_introspection: bool = False, ) -> None: """Programmatic entry point for the cap generator. @@ -903,9 +920,9 @@ def capgen( suite_resolutions.append(suite_res) # Group caps - for rg in suite_res.groups: + for resolved_group in suite_res.groups: write_group_cap( - suite.name, rg.group_name, rg, host_dict, output_root, + suite.name, resolved_group.group_name, resolved_group, host_dict, output_root, logger=log, ) @@ -936,6 +953,7 @@ def capgen( write_static_api( suite_names, suite_resolutions, output_root, host_dict, scheme_store, logger=log, + no_host_introspection=no_host_introspection, ) # ---- host-wide constituent module (only when any suite touches @@ -961,7 +979,7 @@ def capgen( ] suite_file_paths = [] suite_meta_paths = [] - for sname, sr in zip(suite_names, suite_resolutions): + for sname, suite_resolution in zip(suite_names, suite_resolutions): suite_file_paths.append( os.path.join(abs_root, 'ccpp_{}_cap.F90'.format(sname)) ) @@ -972,15 +990,15 @@ def capgen( types_file = os.path.join(abs_root, 'ccpp_{}_types.F90'.format(sname)) if os.path.isfile(types_file): suite_file_paths.append(types_file) - for rg in sr.groups: + for resolved_group in suite_resolution.groups: suite_file_paths.append( os.path.join( abs_root, - 'ccpp_{}_{}_cap.F90'.format(sname, rg.group_name), + 'ccpp_{}_{}_cap.F90'.format(sname, resolved_group.group_name), ) ) suite_meta_paths.append( - os.path.join(abs_root, 'ccpp_{}.meta'.format(sname)) + os.path.join(abs_root, 'ccpp_{}_data.meta'.format(sname)) ) # Expanded SDFs (one per parsed suite) are inspection artifacts; carry # the paths set by parse_suite_xml() forward into datatable.xml. @@ -997,15 +1015,15 @@ def capgen( # don't want its dependencies to leak into datatable.xml. Duplicates # are collapsed by ``write_datatable``. used_scheme_names: set = set() - for sr in suite_resolutions: - for rg in sr.groups: - for items in rg.phase_calls.values(): - for rc in iter_phase_calls(items): - used_scheme_names.add(rc.scheme_name) - if sr.suite_init_call is not None: - used_scheme_names.add(sr.suite_init_call.scheme_name) - if sr.suite_final_call is not None: - used_scheme_names.add(sr.suite_final_call.scheme_name) + for suite_resolution in suite_resolutions: + for resolved_group in suite_resolution.groups: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + used_scheme_names.add(resolved_call.scheme_name) + if suite_resolution.suite_init_call is not None: + used_scheme_names.add(suite_resolution.suite_init_call.scheme_name) + if suite_resolution.suite_final_call is not None: + used_scheme_names.add(suite_resolution.suite_final_call.scheme_name) dependency_paths = [] for tbl in host_tables: @@ -1117,6 +1135,7 @@ def main(argv: Optional[List[str]] = None) -> int: suite_files=_split_file_list(args.suites), output_root=args.output_root, kind_types=kind_types, + no_host_introspection=args.no_host_introspection, ) except CCPPError as exc: _LOGGER.error("%s", exc) diff --git a/capgen-ng/ccpp_validator.py b/capgen-ng/ccpp_validator.py index 7e57e8cb..486f3543 100755 --- a/capgen-ng/ccpp_validator.py +++ b/capgen-ng/ccpp_validator.py @@ -70,6 +70,11 @@ # on the prior line for portability with free-form parsers. Only # applied when we know we are mid-continuation (the buffer is non-empty). _LEAD_CONT_RE = re.compile(r'^\s*&\s?') +# Matches an identifier-character anywhere in a string. Used by the +# decoration-repair branch to distinguish "stray punctuation past a +# trailing ``&``" (safe to drop) from "real tokens past a ``&``" +# (leave alone so the parser surfaces a real error). +_IDENT_CHAR_RE = re.compile(r'[A-Za-z_0-9]') class _SubSig(NamedTuple): @@ -155,10 +160,13 @@ def _line_optional_names(line: str) -> List[str]: return names -def _join_continuation(lines: List[str]) -> List[str]: +def _join_continuation( + lines: List[str], + filename: Optional[str] = None, +) -> List[str]: """Join Fortran continuation lines (ending with ``&``) into single logical lines. - Handles three continuation conventions seen in real CCPP physics code: + Handles four continuation conventions seen in real CCPP physics code: * **Free-form**: ``&`` only at the trailing end of the prior line. * **Dual-form**: ``&`` at the trailing end of the prior line *and* @@ -172,6 +180,23 @@ def _join_continuation(lines: List[str]) -> List[str]: signature, where the line before the closing ``)`` has no trailing ``&``). Detected by look-ahead at the next non-blank, non-comment line. + * **Decorated trailing ``&``** (repair): a ``&`` near the end of the + line is followed by stray non-identifier characters (commas, + parens, whitespace) — typically a typo or hand-edit artefact that + compilers silently ignore because it lives past column 72 in + strict fixed-form mode. When the next line's column-6 ``&`` + already proves we are mid-continuation, treat the last ``&`` as + the continuation marker, discard the decoration, and emit a + ``logger.warning`` naming *filename* so the user knows their + source has decoration past the statement end. + + Parameters + ---------- + lines : list of str + Source lines, each ending in ``\\n`` (as from ``splitlines(keepends=True)``). + filename : str, optional + Source path used only in the decoration-repair warning message. + Defaults to ```` when not supplied. Examples -------- @@ -220,8 +245,12 @@ def _next_starts_with_lead_cont(start_idx: int) -> bool: continue # No trailing ``&`` — but a fixed-form continuation may still # be implied by the next line's column-6 ``&``. If so, keep - # buffering rather than flushing. + # buffering rather than flushing, and try the decoration-repair + # in case the trailing ``&`` was decorated with stray punctuation + # that lives past column 72 (compilers silently drop it; we'd + # otherwise glue it into the joined statement). if _next_starts_with_lead_cont(i): + stripped = _repair_decorated_trailing_amp(stripped, filename, i + 1) buf += stripped continue buf += stripped @@ -232,7 +261,49 @@ def _next_starts_with_lead_cont(start_idx: int) -> bool: return result -def _parse_subroutines(source: str) -> Dict[str, _SubSig]: +def _repair_decorated_trailing_amp( + line: str, + filename: Optional[str], + line_no: int, +) -> str: + """Strip a decorated trailing ``&`` from *line*. + + Called from the fixed-form look-ahead branch of + :func:`_join_continuation`, where the next line's column-6 ``&`` has + already established that we are mid-continuation. If *line* + contains a ``&`` followed only by non-identifier characters + (commas, parens, semicolons, whitespace), the ``&`` is the + decorated continuation marker — drop everything from it onward and + emit a single ``WARNING`` so the user sees that their source has + decoration the compiler is silently ignoring. + + If *line* contains no ``&`` (true fixed-form-leading-only + continuation), or if any token past the last ``&`` looks like a + real Fortran identifier, the line is returned unchanged so the + parser can surface a real error. + """ + amp_idx = line.rfind('&') + if amp_idx < 0: + return line + trailing = line[amp_idx + 1:].strip() + if not trailing: + # ``_CONT_RE`` should have caught this; defensive no-op. + return line + if _IDENT_CHAR_RE.search(trailing): + return line + _LOGGER.warning( + "%s:%d: dropping decoration past trailing '&' (%r); " + "compiler silently ignores this but the parser would otherwise " + "glue it into the statement", + filename or '', line_no, line[amp_idx:], + ) + return line[:amp_idx] + + +def _parse_subroutines( + source: str, + filename: Optional[str] = None, +) -> Dict[str, _SubSig]: """Extract subroutine signatures from Fortran *source*. Returns a mapping ``{subroutine_name_lower: _SubSig}`` where each @@ -276,7 +347,9 @@ def _parse_subroutines(source: str) -> Dict[str, _SubSig]: >>> sorted(sig.optional) ['y', 'z'] """ - logical = _join_continuation(source.splitlines(keepends=True)) + logical = _join_continuation( + source.splitlines(keepends=True), filename=filename, + ) args_by_name: Dict[str, List[str]] = {} optional_by_name: Dict[str, Set[str]] = {} # Stack of names whose body we are currently scanning. Each entry is @@ -335,7 +408,7 @@ def _load_source_tree(source_files: List[str]) -> Dict[str, _SubSig]: for fpath in source_files: with open(fpath) as fh: src = fh.read() - for name, sig in _parse_subroutines(src).items(): + for name, sig in _parse_subroutines(src, filename=fpath).items(): if name not in merged: merged[name] = sig return merged diff --git a/capgen-ng/generator/datatable.py b/capgen-ng/generator/datatable.py index 87819b63..efec3fcf 100644 --- a/capgen-ng/generator/datatable.py +++ b/capgen-ng/generator/datatable.py @@ -226,11 +226,11 @@ def _build_schemes( schemes_elem = ET.SubElement(root, 'schemes') seen_scheme_phases: Dict[str, Set[str]] = {} - for sr in suite_resolutions: - for rg in sr.groups: - for phase_name, items in rg.phase_calls.items(): - for rc in iter_phase_calls(items): - sname = rc.scheme_name + for suite_resolution in suite_resolutions: + for resolved_group in suite_resolution.groups: + for phase_name, items in resolved_group.phase_calls.items(): + for resolved_call in iter_phase_calls(items): + sname = resolved_call.scheme_name if sname not in seen_scheme_phases: seen_scheme_phases[sname] = set() seen_scheme_phases[sname].add(phase_name) @@ -238,13 +238,13 @@ def _build_schemes( # phase_calls — fold them in explicitly so the schemes they # name appear in (and downstream queries pick them # up). Their phase is fixed by the SDF element. - for rc, phase_name in ( - (sr.suite_init_call, 'init'), - (sr.suite_final_call, 'final'), + for resolved_call, phase_name in ( + (suite_resolution.suite_init_call, 'init'), + (suite_resolution.suite_final_call, 'final'), ): - if rc is None: + if resolved_call is None: continue - seen_scheme_phases.setdefault(rc.scheme_name, set()).add(phase_name) + seen_scheme_phases.setdefault(resolved_call.scheme_name, set()).add(phase_name) for sname in sorted(seen_scheme_phases): scheme_elem = ET.SubElement(schemes_elem, 'scheme') @@ -277,19 +277,19 @@ def _build_api( api_elem = ET.SubElement(root, 'api') suites_elem = ET.SubElement(api_elem, 'suites') - for sr in suite_resolutions: + for suite_resolution in suite_resolutions: suite_elem = ET.SubElement(suites_elem, 'suite') - suite_elem.set('name', sr.suite_name) - for rg in sr.groups: + suite_elem.set('name', suite_resolution.suite_name) + for resolved_group in suite_resolution.groups: group_elem = ET.SubElement(suite_elem, 'group') - group_elem.set('name', rg.group_name) + group_elem.set('name', resolved_group.group_name) seen: Set[str] = set() - for items in rg.phase_calls.values(): - for rc in iter_phase_calls(items): - if rc.scheme_name not in seen: - seen.add(rc.scheme_name) + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + if resolved_call.scheme_name not in seen: + seen.add(resolved_call.scheme_name) sch = ET.SubElement(group_elem, 'scheme') - sch.text = rc.scheme_name + sch.text = resolved_call.scheme_name def _build_var_dictionaries( @@ -335,30 +335,30 @@ def _build_var_dictionaries( api_d.set('parent', host_name) ET.SubElement(api_d, 'variables') - for sr in suite_resolutions: + for suite_resolution in suite_resolutions: suite_d = ET.SubElement(dicts, 'var_dictionary') - suite_d.set('name', sr.suite_name) + suite_d.set('name', suite_resolution.suite_name) suite_d.set('type', 'suite') suite_d.set('parent', _API_DICT_NAME) ET.SubElement(suite_d, 'variables') - for rg in sr.groups: + for resolved_group in suite_resolution.groups: group_d = ET.SubElement(dicts, 'var_dictionary') - group_d.set('name', rg.group_name) + group_d.set('name', resolved_group.group_name) group_d.set('type', 'group') - group_d.set('parent', sr.suite_name) + group_d.set('parent', suite_resolution.suite_name) ET.SubElement(group_d, 'variables') call_d = ET.SubElement(dicts, 'var_dictionary') - call_d.set('name', '{}_call_list'.format(rg.group_name)) + call_d.set('name', '{}_call_list'.format(resolved_group.group_name)) call_d.set('type', 'group_call_list') - call_d.set('parent', rg.group_name) + call_d.set('parent', resolved_group.group_name) cl_vars = ET.SubElement(call_d, 'variables') seen: Set[Tuple[str, str]] = set() - for items in rg.phase_calls.values(): - for rc in iter_phase_calls(items): - for arg in rc.args: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: intent = arg.intent or '' key = (arg.standard_name, intent) if key in seen: @@ -430,12 +430,12 @@ def write_datatable( >>> import tempfile, os >>> from generator.datatable import write_datatable >>> from unittest.mock import MagicMock - >>> sr = MagicMock() - >>> sr.suite_name = 'test' - >>> sr.groups = [] + >>> suite_resolution = MagicMock() + >>> suite_resolution.suite_name = 'test' + >>> suite_resolution.groups = [] >>> store = MagicMock() >>> with tempfile.TemporaryDirectory() as d: - ... path = write_datatable([sr], store, [], [], d) + ... path = write_datatable([suite_resolution], store, [], [], d) ... os.path.basename(path) 'datatable.xml' """ diff --git a/capgen-ng/generator/group_cap.py b/capgen-ng/generator/group_cap.py index 71aa5f72..185c5962 100644 --- a/capgen-ng/generator/group_cap.py +++ b/capgen-ng/generator/group_cap.py @@ -249,7 +249,7 @@ def _active_std_names(active: str) -> Set[str]: def _collect_group_uses( - rg: ResolvedGroup, + resolved_group: ResolvedGroup, host_dict, ) -> Dict[str, Set[str]]: """Collect ``{module: {symbol, ...}}`` across all phases of a group. @@ -259,7 +259,7 @@ def _collect_group_uses( Parameters ---------- - rg : ResolvedGroup + resolved_group : ResolvedGroup host_dict : dict Flat host+control variable dictionary (for dimension look-ups). @@ -273,9 +273,9 @@ def _add(mod: Optional[str], sym: str) -> None: if mod is not None: uses.setdefault(mod, set()).add(sym) - for items in rg.phase_calls.values(): - for rc in iter_phase_calls(items): - for arg in rc.args: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: # Direct argument symbol. if arg.source != 'control': _add(arg.module_name, arg.root_symbol) @@ -300,13 +300,13 @@ def _add(mod: Optional[str], sym: str) -> None: _add(arg.constituent_module_name, sym) # Also add dim_uses already collected during resolution. - for mod, syms in rg.dim_uses.items(): + for mod, syms in resolved_group.dim_uses.items(): uses.setdefault(mod, set()).update(syms) return uses -def _collect_kinds_used(rg: ResolvedGroup) -> List[str]: +def _collect_kinds_used(resolved_group: ResolvedGroup) -> List[str]: """Collect kind parameter *names* referenced by transformation temporaries. Mirrors the kind-resolution logic in :func:`_generate_phase_subroutine` @@ -321,9 +321,9 @@ def _collect_kinds_used(rg: ResolvedGroup) -> List[str]: (``1.0_8``) unchanged; only the USE list needs to filter them out. """ kinds: Set[str] = set() - for items in rg.phase_calls.values(): - for rc in iter_phase_calls(items): - for arg in rc.args: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: if not arg.temp_name: continue kind = arg.kind_scheme or ( @@ -343,15 +343,15 @@ def _collect_kinds_used(rg: ResolvedGroup) -> List[str]: # Control variable dummy argument handling ######################################################################## -def _collect_control_args(rg: ResolvedGroup) -> List[ResolvedArg]: +def _collect_control_args(resolved_group: ResolvedGroup) -> List[ResolvedArg]: """Return deduplicated control-variable arguments for the group subroutine. Returns one ResolvedArg per unique standard_name, in a consistent order. """ seen: Dict[str, ResolvedArg] = {} - for items in rg.phase_calls.values(): - for rc in iter_phase_calls(items): - for arg in rc.args: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: if arg.source == 'control' and arg.standard_name not in seen: seen[arg.standard_name] = arg return list(seen.values()) @@ -379,8 +379,8 @@ def _extra_dim_ctrl_entries( has_suite_vars = any( arg.source == 'suite' - for rc in iter_phase_calls(phase_items) - for arg in rc.args + for resolved_call in iter_phase_calls(phase_items) + for arg in resolved_call.args ) needs_inst = ( phase in ('init', 'final', 'timestep_init', 'timestep_final') @@ -391,8 +391,8 @@ def _extra_dim_ctrl_entries( if inst_entry is not None and 'instance_number' not in already: extras['instance_number'] = inst_entry - for rc in iter_phase_calls(phase_items): - for arg in rc.args: + for resolved_call in iter_phase_calls(phase_items): + for arg in resolved_call.args: for dim_std in arg.used_dim_std_names: if dim_std in already or dim_std in extras: continue @@ -436,7 +436,7 @@ def _use_statements(uses: Dict[str, Set[str]]) -> List[str]: return result -def _collect_scheme_uses(rg: ResolvedGroup) -> List[Tuple[str, str, List[str]]]: +def _collect_scheme_uses(resolved_group: ResolvedGroup) -> List[Tuple[str, str, List[str]]]: """Return ``[(scheme_name, module_name, [phase_routine, ...]), ...]``. Schemes are listed in first-seen order across phases (in canonical phase @@ -452,16 +452,16 @@ def _collect_scheme_uses(rg: ResolvedGroup) -> List[Tuple[str, str, List[str]]]: scheme_modules: Dict[str, str] = {} order: List[str] = [] for phase in _GROUP_PHASE_ORDER: - for rc in iter_phase_calls(rg.phase_calls.get(phase, [])): - if rc.scheme_name not in seen_schemes: - seen_schemes[rc.scheme_name] = set() - order.append(rc.scheme_name) - seen_schemes[rc.scheme_name].add(phase) - # rc.scheme_module is empty for old/legacy ResolvedCall objects - # built in tests; fall back to the scheme name so emission still - # works. - scheme_modules[rc.scheme_name] = ( - rc.scheme_module or rc.scheme_name + for resolved_call in iter_phase_calls(resolved_group.phase_calls.get(phase, [])): + if resolved_call.scheme_name not in seen_schemes: + seen_schemes[resolved_call.scheme_name] = set() + order.append(resolved_call.scheme_name) + seen_schemes[resolved_call.scheme_name].add(phase) + # resolved_call.scheme_module is empty for old/legacy ResolvedCall + # objects built in tests; fall back to the scheme name so emission + # still works. + scheme_modules[resolved_call.scheme_name] = ( + resolved_call.scheme_module or resolved_call.scheme_name ) result: List[Tuple[str, str, List[str]]] = [] for sname in order: @@ -471,7 +471,7 @@ def _collect_scheme_uses(rg: ResolvedGroup) -> List[Tuple[str, str, List[str]]]: return result -def _scheme_use_statements(rg: ResolvedGroup) -> List[str]: +def _scheme_use_statements(resolved_group: ResolvedGroup) -> List[str]: """Generate ``use , only: _, ...`` lines. Schemes are emitted in first-seen order; phase routines within each @@ -481,7 +481,7 @@ def _scheme_use_statements(rg: ResolvedGroup) -> List[str]: """ return [ '{}use {}, only: {}'.format(_INDENT, mod, ', '.join(syms)) - for _sname, mod, syms in _collect_scheme_uses(rg) + for _sname, mod, syms in _collect_scheme_uses(resolved_group) if syms ] @@ -664,20 +664,20 @@ def _emit_phase_items( def _emit_one_call( - rc: ResolvedCall, + resolved_call: ResolvedCall, indent: str, lines: List[str], ) -> None: """Append Fortran lines for a single scheme call (with transforms + errcheck).""" # Pre-call transformations. - for arg in rc.args: + for arg in resolved_call.args: lines.extend(_pre_call_lines(arg)) call_args_exprs = [ '{}={}'.format(a.scheme_local_name, _call_arg_expr(a)) - for a in rc.args + for a in resolved_call.args ] - call_name = '{}_{}'.format(rc.scheme_name, rc.phase) + call_name = '{}_{}'.format(resolved_call.scheme_name, resolved_call.phase) if call_args_exprs: lines.append('{}call {}( &'.format(indent, call_name)) @@ -688,12 +688,12 @@ def _emit_one_call( lines.append('{}call {}()'.format(indent, call_name)) errflg_arg = next( - (a for a in rc.args if a.standard_name == 'ccpp_error_code'), None + (a for a in resolved_call.args if a.standard_name == 'ccpp_error_code'), None ) if errflg_arg is not None: lines.append('{}if ({} /= 0) return'.format(indent, _call_arg_expr(errflg_arg))) - for arg in rc.args: + for arg in resolved_call.args: lines.extend(_post_call_lines(arg)) lines.append('') @@ -836,8 +836,8 @@ def _generate_phase_subroutine( seen_temp_names: Set[str] = set() seen_ptr_names: Set[str] = set() - for rc in iter_phase_calls(phase_items): - for arg in rc.args: + for resolved_call in iter_phase_calls(phase_items): + for arg in resolved_call.args: if arg.temp_name and arg.temp_name not in seen_temp_names: seen_temp_names.add(arg.temp_name) t = _fortran_type_str( @@ -1030,7 +1030,7 @@ def _generate_state_dealloc(suite_name: str, group_name: str) -> List[str]: def _generate_group_cap( suite_name: str, group_name: str, - rg: ResolvedGroup, + resolved_group: ResolvedGroup, host_dict, ) -> List[str]: """Generate the full group cap module source lines. @@ -1039,7 +1039,7 @@ def _generate_group_cap( ---------- suite_name : str group_name : str - rg : ResolvedGroup + resolved_group : ResolvedGroup host_dict : dict Flat host+control dictionary. @@ -1062,13 +1062,13 @@ def _generate_group_cap( lines.append('') # ---- USE statements ------------------------------------------------- - uses = _collect_group_uses(rg, host_dict) + uses = _collect_group_uses(resolved_group, host_dict) # Add USE for types module when optional pointer args are present. ptr_type_names: Set[str] = set() - for items in rg.phase_calls.values(): - for rc in iter_phase_calls(items): - for arg in rc.args: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: if arg.ptr_name: type_, kind, rank = _ptr_type_for_arg(arg) ptr_type_names.add(_ptr_type_name(type_, kind, rank)) @@ -1078,12 +1078,12 @@ def _generate_group_cap( # USE ccpp_kinds for any kind parameter referenced in transformation # temporaries declared in this group (e.g. ``real(kind=kind_phys)``). - kind_names = _collect_kinds_used(rg) + kind_names = _collect_kinds_used(resolved_group) if kind_names: uses['ccpp_kinds'] = set(kind_names) use_lines = _use_statements(uses) - use_lines.extend(_scheme_use_statements(rg)) + use_lines.extend(_scheme_use_statements(resolved_group)) lines.extend(use_lines) if use_lines: lines.append('') @@ -1121,7 +1121,7 @@ def _generate_group_cap( host_dict, exclude={'suite_name', 'group_name'} ) for phase in _GROUP_PHASE_ORDER: - phase_items = rg.phase_calls.get(phase, []) + phase_items = resolved_group.phase_calls.get(phase, []) sub_lines = _generate_phase_subroutine( suite_name, group_name, phase, phase_items, ctrl_sig_entries, host_dict ) @@ -1138,11 +1138,11 @@ def _generate_group_cap( return lines -def _ctrl_args_for_phase(rg: ResolvedGroup, phase: str) -> List[ResolvedArg]: +def _ctrl_args_for_phase(resolved_group: ResolvedGroup, phase: str) -> List[ResolvedArg]: """Return control args used in a specific phase, deduplicated.""" seen: Dict[str, ResolvedArg] = {} - for rc in iter_phase_calls(rg.phase_calls.get(phase, [])): - for arg in rc.args: + for resolved_call in iter_phase_calls(resolved_group.phase_calls.get(phase, [])): + for arg in resolved_call.args: if arg.source == 'control' and arg.standard_name not in seen: seen[arg.standard_name] = arg return list(seen.values()) @@ -1155,7 +1155,7 @@ def _ctrl_args_for_phase(rg: ResolvedGroup, phase: str) -> List[ResolvedArg]: def write_group_cap( suite_name: str, group_name: str, - rg: ResolvedGroup, + resolved_group: ResolvedGroup, host_dict, output_root: str, logger: Optional[logging.Logger] = None, @@ -1166,7 +1166,7 @@ def write_group_cap( ---------- suite_name : str group_name : str - rg : ResolvedGroup + resolved_group : ResolvedGroup Resolved call information for this group. host_dict : dict Flat host+control variable dictionary. @@ -1182,7 +1182,7 @@ def write_group_cap( filename = 'ccpp_{}_{}_cap.F90'.format(suite_name, group_name) out_path = os.path.join(output_root, filename) - lines = _generate_group_cap(suite_name, group_name, rg, host_dict) + lines = _generate_group_cap(suite_name, group_name, resolved_group, host_dict) with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return out_path diff --git a/capgen-ng/generator/host_constituents.py b/capgen-ng/generator/host_constituents.py index 114fac40..c0c5302d 100644 --- a/capgen-ng/generator/host_constituents.py +++ b/capgen-ng/generator/host_constituents.py @@ -53,24 +53,24 @@ def _any_constituent_state(suite_results: List[SuiteResolution]) -> bool: """Return True iff any suite either uses or registers constituents.""" return any( - sr.uses_constituents or sr.constituent_register_calls - for sr in suite_results + suite_resolution.uses_constituents or suite_resolution.constituent_register_calls + for suite_resolution in suite_results ) def _all_index_names(suite_results: List[SuiteResolution]) -> List[str]: """Return sorted unique base std-names that need an ``index_of_``.""" names: Set[str] = set() - for sr in suite_results: - names.update(sr.constituent_index_names) + for suite_resolution in suite_results: + names.update(suite_resolution.constituent_index_names) return sorted(names) def _suites_with_register_consts( suite_results: List[SuiteResolution], ) -> List[str]: - return [sr.suite_name for sr in suite_results - if sr.constituent_register_calls] + return [suite_resolution.suite_name for suite_resolution in suite_results + if suite_resolution.constituent_register_calls] def _dyn_const_array_name(suite_name: str) -> str: diff --git a/capgen-ng/generator/static_api.py b/capgen-ng/generator/static_api.py index ebe83a9c..003c0595 100644 --- a/capgen-ng/generator/static_api.py +++ b/capgen-ng/generator/static_api.py @@ -76,6 +76,39 @@ _INDENT = ' ' +# Message bodied into stubbed suite-introspection routines when the +# generator was invoked with ``--no-host-introspection``. Kept here so +# every introspection routine produces the same wording and tests can +# assert against a single constant. +_INTROSPECTION_DISABLED_MSG = ( + 'suite introspection disabled at code-generation time; ' + 'regenerate caps without --no-host-introspection' +) + + +def _emit_introspection_stub_body( + routine_name: str, + list_arg_name: str, + indent: str, +) -> List[str]: + """Emit the stub body shared by the four errflg-bearing introspection + routines (``suite_part_list``, ``suite_schemes``, ``suite_variables``, + ``suite_host_data``). + + Sets ``errflg = 1`` and ``errmsg`` to the canonical disabled-message + prefixed with *routine_name*, then allocates *list_arg_name* to a + zero-length array so callers can safely ``size()`` / iterate without + a NULL-deref crash. Indentation level *indent* matches the body + block of the calling routine. + """ + lines: List[str] = [] + lines.append("{}errmsg = '{}: {}'".format( + indent, routine_name, _INTROSPECTION_DISABLED_MSG, + )) + lines.append('{}errflg = 1'.format(indent)) + lines.append('{}allocate({}(0))'.format(indent, list_arg_name)) + return lines + ######################################################################## # Helpers @@ -90,8 +123,8 @@ def _all_ctrl_args_for_phase( Deduplicated by standard_name, first-seen order. """ seen: Dict[str, ResolvedArg] = {} - for sr in suite_resolutions: - for arg in _suite_ctrl_args_for_phase(sr, phase): + for suite_resolution in suite_resolutions: + for arg in _suite_ctrl_args_for_phase(suite_resolution, phase): if arg.standard_name not in seen: seen[arg.standard_name] = arg return list(seen.values()) @@ -110,8 +143,8 @@ def _all_extra_ctrl_entries_for_phase( return [] seen = set(ctrl_std_names) result: Dict[str, HostVarEntry] = {} - for sr in suite_resolutions: - for entry in _suite_extra_ctrl_entries_for_phase(sr, phase, seen, host_dict): + for suite_resolution in suite_resolutions: + for entry in _suite_extra_ctrl_entries_for_phase(suite_resolution, phase, seen, host_dict): if entry.standard_name not in seen and entry.standard_name not in result: result[entry.standard_name] = entry return list(result.values()) @@ -190,13 +223,13 @@ def _arg_top_level_name( def _collect_host_io( - sr: SuiteResolution, + suite_resolution: SuiteResolution, host_dict=None, collapse_ddts: bool = False, ) -> Tuple[List[str], List[str]]: """Collect (inputs, outputs) standard names for the introspection routines. - Walks every phase of every group of *sr*. Includes scheme args from + Walks every phase of every group of *suite_resolution*. Includes scheme args from every ``source`` category EXCEPT ``'suite'`` — suite-owned vars are internal data flow between schemes (one scheme writes them, another reads them) and are not part of the host-facing variable list. @@ -250,7 +283,7 @@ def _collapse_std(std_name: str) -> str: inputs: Set[str] = set() outputs: Set[str] = set() - for group in sr.groups: + for group in suite_resolution.groups: for items in group.phase_calls.values(): # Subcycle loop bounds named by a CCPP standard name (e.g. # ````) are pure @@ -260,9 +293,9 @@ def _collapse_std(std_name: str) -> str: # the full subcycle tree. Without this, the host's # compile-time bookkeeping (ccpp_physics_suite_variables / # _suite_host_data) would silently omit required inputs. - for sc in iter_phase_subcycles(items): - if sc.loop_std_name: - inputs.add(_collapse_std(sc.loop_std_name)) + for subcycle in iter_phase_subcycles(items): + if subcycle.loop_std_name: + inputs.add(_collapse_std(subcycle.loop_std_name)) for call in iter_phase_calls(items): for arg in call.args: # Suite-owned vars are internal scheme-to-scheme @@ -589,11 +622,21 @@ def _physics_subroutine( # Suite-introspection subroutines ######################################################################## -def _suite_list_subroutine(suite_names: List[str]) -> List[str]: +def _suite_list_subroutine( + suite_names: List[str], + stub_body: bool = False, +) -> List[str]: """Generate ``ccpp_physics_suite_list(suites)``. Allocates ``suites`` to the number of compiled-in suites and assigns each entry to the suite name as a literal string. + + When *stub_body* is true, emit a stub: write a clear message to + ``error_unit`` and return an empty list. ``ccpp_physics_suite_list`` + has no errflg/errmsg arguments, so ``error_unit`` is the only + available error channel. The module-level ``use iso_fortran_env`` + that this references is added by ``_generate_static_api`` when + ``no_host_introspection`` is on. """ i1 = _INDENT i2 = _INDENT * 2 @@ -605,7 +648,17 @@ def _suite_list_subroutine(suite_names: List[str]) -> List[str]: '{}character(len=*), allocatable, intent(out) :: suites(:)'.format(i2) ) lines.append('') - lines.extend(_emit_var_set_loop('suites', suite_names, i2)) + if stub_body: + lines.append( + "{}write(error_unit, '(a)') 'ccpp_physics_suite_list: ' &" + .format(i2) + ) + lines.append( + "{} // '{}'".format(i2, _INTROSPECTION_DISABLED_MSG) + ) + lines.append('{}allocate(suites(0))'.format(i2)) + else: + lines.extend(_emit_var_set_loop('suites', suite_names, i2)) lines.append('') lines.append('{}end subroutine ccpp_physics_suite_list'.format(i1)) return lines @@ -614,6 +667,7 @@ def _suite_list_subroutine(suite_names: List[str]) -> List[str]: def _suite_part_list_subroutine( suite_names: List[str], suite_resolutions: List[SuiteResolution], + stub_body: bool = False, ) -> List[str]: """Generate ``ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errflg)``. @@ -644,21 +698,28 @@ def _suite_part_list_subroutine( '{}integer, intent(out) :: errflg'.format(i2) ) lines.append('') - lines.append("{}errmsg = ''".format(i2)) - lines.append('{}errflg = 0'.format(i2)) - lines.append('') - lines.append('{}select case (trim(suite_name))'.format(i2)) - for sname, sr in zip(suite_names, suite_resolutions): - groups = [g.group_name for g in sr.groups] - lines.append("{}case ('{}')".format(i2, sname)) - lines.extend(_emit_var_set_loop('part_list', groups, i3)) - lines.append('{}case default'.format(i2)) - lines.append('{}errflg = 1'.format(i3)) - lines.append( - "{}errmsg = 'ccpp_physics_suite_part_list: unknown suite: ' " - "// trim(suite_name)".format(i3) - ) - lines.append('{}end select'.format(i2)) + if stub_body: + lines.extend( + _emit_introspection_stub_body( + 'ccpp_physics_suite_part_list', 'part_list', i2, + ) + ) + else: + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + lines.append('{}select case (trim(suite_name))'.format(i2)) + for sname, suite_resolution in zip(suite_names, suite_resolutions): + groups = [g.group_name for g in suite_resolution.groups] + lines.append("{}case ('{}')".format(i2, sname)) + lines.extend(_emit_var_set_loop('part_list', groups, i3)) + lines.append('{}case default'.format(i2)) + lines.append('{}errflg = 1'.format(i3)) + lines.append( + "{}errmsg = 'ccpp_physics_suite_part_list: unknown suite: ' " + "// trim(suite_name)".format(i3) + ) + lines.append('{}end select'.format(i2)) lines.append('') lines.append('{}end subroutine ccpp_physics_suite_part_list'.format(i1)) return lines @@ -667,6 +728,7 @@ def _suite_part_list_subroutine( def _suite_schemes_subroutine( suite_names: List[str], suite_resolutions: List[SuiteResolution], + stub_body: bool = False, ) -> List[str]: """Generate ``ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errflg)``. @@ -696,26 +758,33 @@ def _suite_schemes_subroutine( '{}integer, intent(out) :: errflg'.format(i2) ) lines.append('') - lines.append("{}errmsg = ''".format(i2)) - lines.append('{}errflg = 0'.format(i2)) - lines.append('') - lines.append('{}select case (trim(suite_name))'.format(i2)) - for sname, sr in zip(suite_names, suite_resolutions): - schemes = sorted({ - call.scheme_name - for group in sr.groups - for items in group.phase_calls.values() - for call in iter_phase_calls(items) - }) - lines.append("{}case ('{}')".format(i2, sname)) - lines.extend(_emit_var_set_loop('scheme_list', schemes, i3)) - lines.append('{}case default'.format(i2)) - lines.append('{}errflg = 1'.format(i3)) - lines.append( - "{}errmsg = 'ccpp_physics_suite_schemes: unknown suite: ' " - "// trim(suite_name)".format(i3) - ) - lines.append('{}end select'.format(i2)) + if stub_body: + lines.extend( + _emit_introspection_stub_body( + 'ccpp_physics_suite_schemes', 'scheme_list', i2, + ) + ) + else: + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + lines.append('{}select case (trim(suite_name))'.format(i2)) + for sname, suite_resolution in zip(suite_names, suite_resolutions): + schemes = sorted({ + call.scheme_name + for group in suite_resolution.groups + for items in group.phase_calls.values() + for call in iter_phase_calls(items) + }) + lines.append("{}case ('{}')".format(i2, sname)) + lines.extend(_emit_var_set_loop('scheme_list', schemes, i3)) + lines.append('{}case default'.format(i2)) + lines.append('{}errflg = 1'.format(i3)) + lines.append( + "{}errmsg = 'ccpp_physics_suite_schemes: unknown suite: ' " + "// trim(suite_name)".format(i3) + ) + lines.append('{}end select'.format(i2)) lines.append('') lines.append('{}end subroutine ccpp_physics_suite_schemes'.format(i1)) return lines @@ -726,6 +795,7 @@ def _suite_io_subroutine( suite_resolutions: List[SuiteResolution], host_dict=None, collapse_ddts: bool = False, + stub_body: bool = False, ) -> List[str]: """Generate ``ccpp_physics_suite_variables`` or ``ccpp_physics_suite_host_data``. @@ -775,44 +845,56 @@ def _suite_io_subroutine( '{}logical, optional, intent(in) :: output_vars'.format(i2) ) lines.append('') - lines.append('{}logical :: input_vars_use'.format(i2)) - lines.append('{}logical :: output_vars_use'.format(i2)) - lines.append('') - lines.append("{}errmsg = ''".format(i2)) - lines.append('{}errflg = 0'.format(i2)) - lines.append('') - lines.append('{}if (present(input_vars)) then'.format(i2)) - lines.append('{}input_vars_use = input_vars'.format(i3)) - lines.append('{}else'.format(i2)) - lines.append('{}input_vars_use = .true.'.format(i3)) - lines.append('{}end if'.format(i2)) - lines.append('{}if (present(output_vars)) then'.format(i2)) - lines.append('{}output_vars_use = output_vars'.format(i3)) - lines.append('{}else'.format(i2)) - lines.append('{}output_vars_use = .true.'.format(i3)) - lines.append('{}end if'.format(i2)) - lines.append('') - lines.append('{}select case (trim(suite_name))'.format(i2)) - for sname, sr in zip(suite_names, suite_resolutions): - inputs, outputs = _collect_host_io(sr, host_dict, collapse_ddts) - union = sorted(set(inputs) | set(outputs)) - lines.append("{}case ('{}')".format(i2, sname)) - lines.append('{}if (input_vars_use .and. output_vars_use) then'.format(i3)) - lines.extend(_emit_var_set_loop('variable_list', union, i4)) - lines.append('{}else if (input_vars_use) then'.format(i3)) - lines.extend(_emit_var_set_loop('variable_list', inputs, i4)) - lines.append('{}else if (output_vars_use) then'.format(i3)) - lines.extend(_emit_var_set_loop('variable_list', outputs, i4)) - lines.append('{}else'.format(i3)) - lines.append('{}allocate(variable_list(0))'.format(i4)) - lines.append('{}end if'.format(i3)) - lines.append('{}case default'.format(i2)) - lines.append('{}errflg = 1'.format(i3)) - lines.append( - "{}errmsg = '{}: unknown suite: ' " - "// trim(suite_name)".format(i3, sub_name) - ) - lines.append('{}end select'.format(i2)) + if stub_body: + # Stubbed body: set errflg + clear errmsg and allocate an empty + # list. ``input_vars`` / ``output_vars`` are intentionally left + # unreferenced — they're declared ``intent(in), optional`` so + # compilers may warn about the unused dummy arg, but warning is + # the right outcome: the host built against introspection and + # is calling with filter flags that no longer matter. + lines.append('') + lines.extend( + _emit_introspection_stub_body(sub_name, 'variable_list', i2) + ) + else: + lines.append('{}logical :: input_vars_use'.format(i2)) + lines.append('{}logical :: output_vars_use'.format(i2)) + lines.append('') + lines.append("{}errmsg = ''".format(i2)) + lines.append('{}errflg = 0'.format(i2)) + lines.append('') + lines.append('{}if (present(input_vars)) then'.format(i2)) + lines.append('{}input_vars_use = input_vars'.format(i3)) + lines.append('{}else'.format(i2)) + lines.append('{}input_vars_use = .true.'.format(i3)) + lines.append('{}end if'.format(i2)) + lines.append('{}if (present(output_vars)) then'.format(i2)) + lines.append('{}output_vars_use = output_vars'.format(i3)) + lines.append('{}else'.format(i2)) + lines.append('{}output_vars_use = .true.'.format(i3)) + lines.append('{}end if'.format(i2)) + lines.append('') + lines.append('{}select case (trim(suite_name))'.format(i2)) + for sname, suite_resolution in zip(suite_names, suite_resolutions): + inputs, outputs = _collect_host_io(suite_resolution, host_dict, collapse_ddts) + union = sorted(set(inputs) | set(outputs)) + lines.append("{}case ('{}')".format(i2, sname)) + lines.append('{}if (input_vars_use .and. output_vars_use) then'.format(i3)) + lines.extend(_emit_var_set_loop('variable_list', union, i4)) + lines.append('{}else if (input_vars_use) then'.format(i3)) + lines.extend(_emit_var_set_loop('variable_list', inputs, i4)) + lines.append('{}else if (output_vars_use) then'.format(i3)) + lines.extend(_emit_var_set_loop('variable_list', outputs, i4)) + lines.append('{}else'.format(i3)) + lines.append('{}allocate(variable_list(0))'.format(i4)) + lines.append('{}end if'.format(i3)) + lines.append('{}case default'.format(i2)) + lines.append('{}errflg = 1'.format(i3)) + lines.append( + "{}errmsg = '{}: unknown suite: ' " + "// trim(suite_name)".format(i3, sub_name) + ) + lines.append('{}end select'.format(i2)) lines.append('') lines.append('{}end subroutine {}'.format(i1, sub_name)) return lines @@ -827,6 +909,7 @@ def _generate_static_api( suite_resolutions: List[SuiteResolution], host_dict=None, scheme_store: Optional[SchemeStore] = None, + no_host_introspection: bool = False, ) -> List[str]: """Generate the full ``ccpp_static_api.F90`` module source lines. @@ -862,6 +945,15 @@ def _generate_static_api( lines.append('module ccpp_static_api') lines.append('') + # Pull in ``error_unit`` for the stubbed ``ccpp_physics_suite_list`` + # body (which has no errflg/errmsg arg, so error_unit is the only + # available channel). Only emitted when --no-host-introspection is + # on, to keep the module imports minimal in the normal case. + if no_host_introspection: + lines.append( + '{}use iso_fortran_env, only: error_unit'.format(_INDENT) + ) + # USE each suite cap module. ``_register`` is now mandatory and # always emitted in the suite cap, so always import it here too. for sname in suite_names: @@ -882,8 +974,8 @@ def _generate_static_api( # constituent state (the ccpp_host_constituents module is only emitted # in that case too). uses_consts = any( - sr.uses_constituents or sr.constituent_register_calls - for sr in suite_resolutions + suite_resolution.uses_constituents or suite_resolution.constituent_register_calls + for suite_resolution in suite_resolutions ) constituent_pub_syms = [ 'ccpp_model_constituents_obj', @@ -937,14 +1029,26 @@ def _generate_static_api( lines.extend(_physics_subroutine(phase, suite_names, suite_resolutions, host_dict)) lines.extend(_final_subroutine(suite_names, host_dict)) # Introspection routines (do not advance state, no scheme calls). - lines.extend(_suite_list_subroutine(suite_names)) - lines.extend(_suite_part_list_subroutine(suite_names, suite_resolutions)) - lines.extend(_suite_schemes_subroutine(suite_names, suite_resolutions)) + # With --no-host-introspection, each routine retains its signature + # but the body is replaced with an errflg=1 stub (or, for + # suite_list, an error_unit write + empty allocation), shrinking + # ccpp_static_api.F90 dramatically for multi-suite builds. + lines.extend(_suite_list_subroutine( + suite_names, stub_body=no_host_introspection, + )) + lines.extend(_suite_part_list_subroutine( + suite_names, suite_resolutions, stub_body=no_host_introspection, + )) + lines.extend(_suite_schemes_subroutine( + suite_names, suite_resolutions, stub_body=no_host_introspection, + )) lines.extend(_suite_io_subroutine( suite_names, suite_resolutions, host_dict, collapse_ddts=False, + stub_body=no_host_introspection, )) lines.extend(_suite_io_subroutine( suite_names, suite_resolutions, host_dict, collapse_ddts=True, + stub_body=no_host_introspection, )) lines.append('') @@ -963,6 +1067,7 @@ def write_static_api( host_dict=None, scheme_store: Optional[SchemeStore] = None, logger: Optional[logging.Logger] = None, + no_host_introspection: bool = False, ) -> str: """Write ``ccpp_static_api.F90`` to *output_root*. @@ -980,6 +1085,17 @@ def write_static_api( scheme; only those drive emission of ``ccpp_register`` and the constituent module USE. When omitted, ``ccpp_register`` is omitted entirely. + no_host_introspection : bool, optional + When True, replace the bodies of the five suite-introspection + routines (``ccpp_physics_suite_list`` / ``..._suite_part_list`` / + ``..._suite_schemes`` / ``..._suite_variables`` / + ``..._suite_host_data``) with stubs that set ``errflg=1`` and a + clear ``errmsg`` (or write to ``error_unit`` for + ``ccpp_physics_suite_list``, which has no error channel). + Signatures remain so existing callers still link. Use this to + shrink ``ccpp_static_api.F90`` from ~33k lines to ~800 for + multi-suite builds where the introspection case-blocks make + ``-O3`` compilation impractical. Returns ------- @@ -990,7 +1106,10 @@ def write_static_api( filename = 'ccpp_static_api.F90' out_path = os.path.join(output_root, filename) - lines = _generate_static_api(suite_names, suite_resolutions, host_dict, scheme_store) + lines = _generate_static_api( + suite_names, suite_resolutions, host_dict, scheme_store, + no_host_introspection=no_host_introspection, + ) with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return out_path diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 1a4b949c..2fdc133c 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -75,19 +75,19 @@ def _all_suite_scheme_names(suite_res: SuiteResolution) -> List[str]: order within each group's phase call list). >>> from generator.suite_resolver import SuiteResolution, ResolvedGroup, ResolvedCall - >>> rg = ResolvedGroup('grp', phase_calls={'run': [ResolvedCall('sch_a', 'run'), ResolvedCall('sch_b', 'run')]}) - >>> sr = SuiteResolution('s', groups=[rg]) - >>> _all_suite_scheme_names(sr) + >>> resolved_group = ResolvedGroup('grp', phase_calls={'run': [ResolvedCall('sch_a', 'run'), ResolvedCall('sch_b', 'run')]}) + >>> suite_resolution = SuiteResolution('s', groups=[resolved_group]) + >>> _all_suite_scheme_names(suite_resolution) ['sch_a', 'sch_b'] """ seen: Set[str] = set() names: List[str] = [] - for rg in suite_res.groups: - for items in rg.phase_calls.values(): - for rc in iter_phase_calls(items): - if rc.scheme_name not in seen: - seen.add(rc.scheme_name) - names.append(rc.scheme_name) + for resolved_group in suite_res.groups: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + if resolved_call.scheme_name not in seen: + seen.add(resolved_call.scheme_name) + names.append(resolved_call.scheme_name) return names @@ -115,27 +115,27 @@ def _suite_ctrl_args_for_phase( The result is deduplicated by standard_name and preserves first-seen order. """ seen: Dict[str, ResolvedArg] = {} - for rg in suite_res.groups: - for arg in _ctrl_args_for_phase(rg, phase): + for resolved_group in suite_res.groups: + for arg in _ctrl_args_for_phase(resolved_group, phase): if arg.standard_name not in seen: seen[arg.standard_name] = arg return list(seen.values()) -def _group_ctrl_arg_names(rg: ResolvedGroup, phase: str, host_dict=None) -> List[str]: +def _group_ctrl_arg_names(resolved_group: ResolvedGroup, phase: str, host_dict=None) -> List[str]: """Return the local_name list for the control args of a group phase. These are the keyword names passed when calling the group cap subroutine. Includes extra control vars needed for state indexing and dimension subscripts (instance_number for suite-var access, control vars used only in dim subscripts). """ - ctrl_args = _ctrl_args_for_phase(rg, phase) + ctrl_args = _ctrl_args_for_phase(resolved_group, phase) names = [ a.host_entry.local_name for a in ctrl_args if a.host_entry is not None ] - phase_items = rg.phase_calls.get(phase, []) + phase_items = resolved_group.phase_calls.get(phase, []) for entry in _extra_dim_ctrl_entries(phase_items, phase, ctrl_args, host_dict): if entry.local_name not in names: names.append(entry.local_name) @@ -159,9 +159,9 @@ def _suite_extra_ctrl_entries_for_phase( return [] seen = set(ctrl_std_names) result: Dict[str, HostVarEntry] = {} - for rg in suite_res.groups: - phase_items = rg.phase_calls.get(phase, []) - ctrl_args = _ctrl_args_for_phase(rg, phase) + for resolved_group in suite_res.groups: + phase_items = resolved_group.phase_calls.get(phase, []) + ctrl_args = _ctrl_args_for_phase(resolved_group, phase) for entry in _extra_dim_ctrl_entries(phase_items, phase, ctrl_args, host_dict): if entry.standard_name not in seen and entry.standard_name not in result: result[entry.standard_name] = entry @@ -178,9 +178,9 @@ def _register_calls(suite_res: SuiteResolution): Groups are visited in suite-XML order; within each group the calls follow the resolver's ordering (which mirrors the suite XML). """ - for rg in suite_res.groups: - for rc in iter_phase_calls(rg.phase_calls.get('register', [])): - yield rg.group_name, rc + for resolved_group in suite_res.groups: + for resolved_call in iter_phase_calls(resolved_group.phase_calls.get('register', [])): + yield resolved_group.group_name, resolved_call def _register_uses( @@ -200,16 +200,16 @@ def _register_uses( """ uses: Dict[str, Set[str]] = {} seen_schemes: Set[str] = set() - for _gname, rc in _register_calls(suite_res): - if rc.scheme_name not in seen_schemes: - seen_schemes.add(rc.scheme_name) + for _gname, resolved_call in _register_calls(suite_res): + if resolved_call.scheme_name not in seen_schemes: + seen_schemes.add(resolved_call.scheme_name) # Module is metadata-declared (``module_name`` in table props) # when present; otherwise falls back to the scheme name. - scheme_module = rc.scheme_module or rc.scheme_name + scheme_module = resolved_call.scheme_module or resolved_call.scheme_name uses.setdefault(scheme_module, set()).add( - '{}_register'.format(rc.scheme_name) + '{}_register'.format(resolved_call.scheme_name) ) - for arg in rc.args: + for arg in resolved_call.args: if arg.is_constituent_arg: continue # local temp, not a USE'd var mod = arg.module_name @@ -225,7 +225,7 @@ def _register_uses( return uses -def _add_call_uses(uses: Dict[str, Set[str]], rc) -> None: +def _add_call_uses(uses: Dict[str, Set[str]], resolved_call) -> None: """Merge USE-statement requirements for a single :class:`ResolvedCall`. Adds: @@ -239,29 +239,29 @@ def _add_call_uses(uses: Dict[str, Set[str]], rc) -> None: ``_final`` to integrate the suite-level / scheme calls into the USE block. """ - scheme_module = rc.scheme_module or rc.scheme_name + scheme_module = resolved_call.scheme_module or resolved_call.scheme_name uses.setdefault(scheme_module, set()).add( - '{}_{}'.format(rc.scheme_name, rc.phase) + '{}_{}'.format(resolved_call.scheme_name, resolved_call.phase) ) - for arg in rc.args: + for arg in resolved_call.args: mod = arg.module_name if mod is not None: uses.setdefault(mod, set()).add(arg.root_symbol) -def _emit_register_call(rc, indent: str, errflg_local: str, lines: List[str]) -> None: +def _emit_register_call(resolved_call, indent: str, errflg_local: str, lines: List[str]) -> None: """Emit one scheme ``_register`` call with keyword args + error guard. Register-phase calls are kept simple: no transformations (transform code paths are physics-phase only), keyword-arg style for clarity. """ - sub = '{}_register'.format(rc.scheme_name) - if not rc.args: + sub = '{}_register'.format(resolved_call.scheme_name) + if not resolved_call.args: lines.append('{}call {}()'.format(indent, sub)) else: lines.append('{}call {}( &'.format(indent, sub)) - for i, arg in enumerate(rc.args): - sep = ', &' if i < len(rc.args) - 1 else ')' + for i, arg in enumerate(resolved_call.args): + sep = ', &' if i < len(resolved_call.args) - 1 else ')' lines.append('{} {}={}{}'.format( indent, arg.scheme_local_name, arg.call_expr, sep )) @@ -371,7 +371,7 @@ def _register_lines( # only the first instance to enter does the two-pass count+pack. # Subsequent instances reuse the same buffer. The state-array # transition still runs per instance (after this block). - const_scheme_names = {sn for sn, _ in suite_res.constituent_register_calls} + const_scheme_names = {scheme_name for scheme_name, _ in suite_res.constituent_register_calls} buf = '{}_dynamic_constituents'.format(suite_name) lines.append( @@ -379,9 +379,9 @@ def _register_lines( ) lines.append('{}num_consts = 0'.format(i2 + _INDENT)) lines.append('{}! First pass: count constituents'.format(i2 + _INDENT)) - for _gname, rc in _register_calls(suite_res): - if rc.scheme_name in const_scheme_names: - _emit_register_call(rc, i2 + _INDENT, errflg_local, lines) + for _gname, resolved_call in _register_calls(suite_res): + if resolved_call.scheme_name in const_scheme_names: + _emit_register_call(resolved_call, i2 + _INDENT, errflg_local, lines) lines.append( '{}num_consts = num_consts + size(scheme_consts, 1)'.format( i2 + _INDENT, @@ -393,9 +393,9 @@ def _register_lines( lines.append('{}num_consts = 0'.format(i2 + _INDENT)) lines.append('') lines.append('{}! Second pass: copy into per-suite buffer'.format(i2 + _INDENT)) - for _gname, rc in _register_calls(suite_res): - if rc.scheme_name in const_scheme_names: - _emit_register_call(rc, i2 + _INDENT, errflg_local, lines) + for _gname, resolved_call in _register_calls(suite_res): + if resolved_call.scheme_name in const_scheme_names: + _emit_register_call(resolved_call, i2 + _INDENT, errflg_local, lines) lines.append('{}do i = 1, size(scheme_consts, 1)'.format(i2 + _INDENT)) lines.append( '{}{}(num_consts + i) = scheme_consts(i)'.format( @@ -412,13 +412,13 @@ def _register_lines( lines.append('{}end if'.format(i2)) lines.append('') # Emit any non-constituent register calls in addition (always, per instance). - for _gname, rc in _register_calls(suite_res): - if rc.scheme_name not in const_scheme_names: - _emit_register_call(rc, i2, errflg_local, lines) + for _gname, resolved_call in _register_calls(suite_res): + if resolved_call.scheme_name not in const_scheme_names: + _emit_register_call(resolved_call, i2, errflg_local, lines) else: # No constituent merge — emit register calls in suite-XML order. - for _gname, rc in _register_calls(suite_res): - _emit_register_call(rc, i2, errflg_local, lines) + for _gname, resolved_call in _register_calls(suite_res): + _emit_register_call(resolved_call, i2, errflg_local, lines) lines.append('') lines.append( @@ -529,8 +529,8 @@ def _init_lines( ] # Group state allocators (idempotent). - for rg in suite_res.groups: - alloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, rg.group_name, 'state_alloc') + for resolved_group in suite_res.groups: + alloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'state_alloc') lines.append('{}call {}({}, {}, {})'.format( i2, alloc_sub, ninstances_arg, errmsg_local, errflg_local )) @@ -682,8 +682,8 @@ def _final_lines( lines.append( '{}if (all(ccpp_suite_state == CCPP_SUITE_UNREGISTERED)) then'.format(i2) ) - for rg in suite_res.groups: - dealloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, rg.group_name, 'state_dealloc') + for resolved_group in suite_res.groups: + dealloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'state_dealloc') lines.append('{} call {}({}, {})'.format( i2, dealloc_sub, errmsg_local, errflg_local )) @@ -807,10 +807,10 @@ def _physics_dispatch_lines( '', ] - def _emit_group_call(rg, indent): + def _emit_group_call(resolved_group, indent): # Group phase subroutines are always emitted (so the per-group state # machine transitions through every phase), so we always dispatch. - cap_sub = 'ccpp_{}_{}_{}'.format(suite_name, rg.group_name, phase) + cap_sub = 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, phase) if group_ctrl_local: lines.append('{}call {}( &'.format(indent, cap_sub)) for idx, lname in enumerate(group_ctrl_local): @@ -824,12 +824,12 @@ def _emit_group_call(rg, indent): lines.append('{}select case(trim({}))'.format(i2, grp_local)) # '' or 'all' → call all groups. lines.append("{}case('', 'all')".format(i2)) - for rg in suite_res.groups: - _emit_group_call(rg, i3) + for resolved_group in suite_res.groups: + _emit_group_call(resolved_group, i3) # Individual group cases. - for rg in suite_res.groups: - lines.append("{}case('{}')".format(i2, rg.group_name)) - _emit_group_call(rg, i3) + for resolved_group in suite_res.groups: + lines.append("{}case('{}')".format(i2, resolved_group.group_name)) + _emit_group_call(resolved_group, i3) # case default: anything other than '', 'all', or a known group # is a runtime error — caller asked for a group this suite # doesn't define. Without ccpp_error_code/_message in the host @@ -848,8 +848,8 @@ def _emit_group_call(rg, indent): lines.append('{}end select'.format(i2)) else: # No group_name control var: call all groups unconditionally. - for rg in suite_res.groups: - _emit_group_call(rg, i2) + for resolved_group in suite_res.groups: + _emit_group_call(resolved_group, i2) lines.append('') lines.append('{}end subroutine {}'.format(i1, sub_name)) @@ -979,14 +979,14 @@ def _generate_suite_cap( lines.append('') # USE statements: one per group cap (all phase + state subroutines). - for rg in suite_res.groups: - group_cap_mod = 'ccpp_{}_{}_{}'.format(suite_name, rg.group_name, 'cap') + for resolved_group in suite_res.groups: + group_cap_mod = 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'cap') syms_list = [ - 'ccpp_{}_{}_{}'.format(suite_name, rg.group_name, p) + 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, p) for p in _PHYSICS_PHASES ] - syms_list.append('ccpp_{}_{}_{}'.format(suite_name, rg.group_name, 'state_alloc')) - syms_list.append('ccpp_{}_{}_{}'.format(suite_name, rg.group_name, 'state_dealloc')) + syms_list.append('ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'state_alloc')) + syms_list.append('ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'state_dealloc')) lines.append('{}use {}, only: {}'.format( _INDENT, group_cap_mod, ', '.join(syms_list) )) diff --git a/capgen-ng/generator/suite_data.py b/capgen-ng/generator/suite_data.py index faf198f6..e0e2a0ec 100644 --- a/capgen-ng/generator/suite_data.py +++ b/capgen-ng/generator/suite_data.py @@ -59,8 +59,8 @@ def _collect_dim_uses( uses: Dict[str, List] = {} seen: set = set() suite_var_std_names = set(suite_vars.keys()) - for sv in sorted(suite_vars.values(), key=lambda v: v.standard_name): - for dim_std in sv.dimensions: + for suite_var in sorted(suite_vars.values(), key=lambda v: v.standard_name): + for dim_std in suite_var.dimensions: if dim_std in seen: continue seen.add(dim_std) @@ -71,7 +71,7 @@ def _collect_dim_uses( raise CCPPError( "Suite-owned variable '{}' has dimension '{}' but no host " "metadata was provided to resolve it".format( - sv.standard_name, dim_std + suite_var.standard_name, dim_std ) ) entry = host_dict.get(dim_std) @@ -79,14 +79,14 @@ def _collect_dim_uses( raise CCPPError( "Suite-owned variable '{}' dimension '{}' not found in " "host metadata or in suite-owned variables".format( - sv.standard_name, dim_std + suite_var.standard_name, dim_std ) ) if entry.is_control: raise CCPPError( "Suite-owned variable '{}' dimension '{}' is a control " "variable; suite data must use host-module or " - "suite-owned dimensions".format(sv.standard_name, dim_std) + "suite-owned dimensions".format(suite_var.standard_name, dim_std) ) mod = entry.module_name if mod not in uses: @@ -131,8 +131,8 @@ def _collect_ddt_uses( """ uses: Dict[str, List[str]] = {} seen: set = set() - for sv in sorted(suite_vars.values(), key=lambda v: v.standard_name): - t = sv.type_.strip() + for suite_var in sorted(suite_vars.values(), key=lambda v: v.standard_name): + t = suite_var.type_.strip() tlow = t.lower() if tlow in _INTRINSICS or tlow.startswith('external:'): continue @@ -146,7 +146,7 @@ def _collect_ddt_uses( "Suite-owned variable '{}' has DDT type '{}' but its " "defining Fortran module is unknown; the DDT must appear " "in a metadata file alongside a scheme/host/control " - "table".format(sv.standard_name, t) + "table".format(suite_var.standard_name, t) ) mod = ddt_module_map[t] uses.setdefault(mod, []) @@ -190,8 +190,8 @@ def _generate_suite_data( # USE ccpp_kinds for any kind parameters referenced in suite-var # declarations (e.g. ``real(kind=kind_phys)``). kind_names = sorted({ - sv.kind for sv in suite_vars.values() - if sv.kind and not sv.kind.startswith('len=') + suite_var.kind for suite_var in suite_vars.values() + if suite_var.kind and not suite_var.kind.startswith('len=') }) if kind_names: lines.append( @@ -223,18 +223,18 @@ def _generate_suite_data( # for optional-arg passing and transformation temporaries. lines.append('{}type, public :: {}'.format(i1, type_name)) if suite_vars: - for sv in sorted(suite_vars.values(), key=lambda v: v.standard_name): - t = _type_str(sv.type_, sv.kind) - if sv.dimensions: - rank = len(sv.dimensions) + for suite_var in sorted(suite_vars.values(), key=lambda v: v.standard_name): + t = _type_str(suite_var.type_, suite_var.kind) + if suite_var.dimensions: + rank = len(suite_var.dimensions) deferred = '({})'.format(','.join([':'] * rank)) lines.append( '{}{}, allocatable :: {}{}'.format( - i2, t, sv.local_name, deferred, + i2, t, suite_var.local_name, deferred, ) ) else: - lines.append('{}{} :: {}'.format(i2, t, sv.local_name)) + lines.append('{}{} :: {}'.format(i2, t, suite_var.local_name)) else: lines.append('{}! (no suite-owned variables)'.format(i2)) lines.append('{}end type {}'.format(i1, type_name)) @@ -324,15 +324,15 @@ def _generate_suite_data( "{}errmsg = ''".format(i2), '{}errflg = 0'.format(i2), ] - for sv in sorted_svs: - if sv.dimensions: + for suite_var in sorted_svs: + if suite_var.dimensions: dim_exprs = [ _dim_local_expr(d, suite_vars, host_dict) - for d in sv.dimensions + for d in suite_var.dimensions ] lines.append( '{}allocate(ccpp_suite_data(i)%{}({}))'.format( - i2, sv.local_name, ', '.join(dim_exprs) + i2, suite_var.local_name, ', '.join(dim_exprs) ) ) lines += [ @@ -354,12 +354,12 @@ def _generate_suite_data( "{}errmsg = ''".format(i2), '{}errflg = 0'.format(i2), ] - for sv in sorted_svs: - if sv.dimensions: + for suite_var in sorted_svs: + if suite_var.dimensions: lines.append( '{}if (allocated(ccpp_suite_data(i)%{})) ' 'deallocate(ccpp_suite_data(i)%{})'.format( - i2, sv.local_name, sv.local_name + i2, suite_var.local_name, suite_var.local_name ) ) lines += [ @@ -395,10 +395,13 @@ def _generate_suite_meta( suite_name: str, suite_vars: Dict[str, SuiteVar], ) -> List[str]: - """Generate metadata lines for ``ccpp_.meta``. + """Generate metadata lines for ``ccpp__data.meta``. The file documents all suite-owned variables in the standard ``.meta`` format so that downstream tools can inspect what each suite provides. + The ``_data`` suffix matches the companion Fortran file + ``ccpp__data.F90``, satisfying the ``.meta`` ↔ ``.F90`` pairing + convention. >>> lines = _generate_suite_meta('mysuite', {}) >>> lines[0].startswith('!') @@ -410,7 +413,7 @@ def _generate_suite_meta( i1 = _INDENT lines: List[str] = [] lines.append( - '! ccpp_{}.meta -- generated by ccpp_capgen_ng, do not edit'.format(suite_name) + '! ccpp_{}_data.meta -- generated by ccpp_capgen_ng, do not edit'.format(suite_name) ) lines.append('[ccpp-table-properties]') lines.append('{}name = {}'.format(i1, mod_name)) @@ -419,17 +422,17 @@ def _generate_suite_meta( lines.append('[ccpp-arg-table]') lines.append('{}name = {}'.format(i1, mod_name)) lines.append('{}type = suite'.format(i1)) - for sv in sorted(suite_vars.values(), key=lambda v: v.standard_name): + for suite_var in sorted(suite_vars.values(), key=lambda v: v.standard_name): lines.append('') - lines.append('[ {} ]'.format(sv.local_name)) - lines.append('{}standard_name = {}'.format(i1, sv.standard_name)) - lines.append('{}long_name = {}'.format(i1, sv.standard_name)) - lines.append('{}units = {}'.format(i1, sv.units)) - dim_str = '({})'.format(', '.join(sv.dimensions)) if sv.dimensions else '()' + lines.append('[ {} ]'.format(suite_var.local_name)) + lines.append('{}standard_name = {}'.format(i1, suite_var.standard_name)) + lines.append('{}long_name = {}'.format(i1, suite_var.standard_name)) + lines.append('{}units = {}'.format(i1, suite_var.units)) + dim_str = '({})'.format(', '.join(suite_var.dimensions)) if suite_var.dimensions else '()' lines.append('{}dimensions = {}'.format(i1, dim_str)) - lines.append('{}type = {}'.format(i1, sv.type_)) - if sv.kind: - lines.append('{}kind = {}'.format(i1, sv.kind)) + lines.append('{}type = {}'.format(i1, suite_var.type_)) + if suite_var.kind: + lines.append('{}kind = {}'.format(i1, suite_var.kind)) return lines @@ -439,9 +442,9 @@ def write_suite_meta( output_root: str, logger: Optional[logging.Logger] = None, ) -> str: - """Write ``ccpp_.meta`` to *output_root* and return its path.""" + """Write ``ccpp__data.meta`` to *output_root* and return its path.""" os.makedirs(output_root, exist_ok=True) - filename = 'ccpp_{}.meta'.format(suite_name) + filename = 'ccpp_{}_data.meta'.format(suite_name) out_path = os.path.join(output_root, filename) lines = _generate_suite_meta(suite_name, suite_vars) with open_if_changed(out_path, logger=logger) as fh: diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index f7d2e3b6..6a102969 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -273,8 +273,8 @@ def _format_available_std_names( else: rows.append((std, 'host')) if suite_vars: - for std, sv in suite_vars.items(): - rows.append((std, 'suite: {}'.format(sv.suite_module_name))) + for std, suite_var in suite_vars.items(): + rows.append((std, 'suite: {}'.format(suite_var.suite_module_name))) rows.sort(key=lambda t: t[0]) if not rows: @@ -353,10 +353,10 @@ def _resolve_single_bound( # code (Fortran rejects it as "no IMPLICIT type"). return _substitute_scalar_idx(entry.access_path, host_dict) if suite_vars: - sv = suite_vars.get(bound) - if sv is not None: + suite_var = suite_vars.get(bound) + if suite_var is not None: used.add(bound) - return sv.access_path + return suite_var.access_path return None @@ -965,8 +965,8 @@ def _resolve_subcycle_loop_bound( # in an instance-dimensioned array; resolve that template here). return _substitute_instance_idx(entry.access_path, host_dict), key if suite_vars and key in suite_vars: - sv = suite_vars[key] - return sv.access_path, key + suite_var = suite_vars[key] + return suite_var.access_path, key raise CCPPError( "Subcycle loop=\"{}\" is not an integer literal and does not " "resolve to a CCPP standard name in the host/control metadata or " @@ -1311,18 +1311,18 @@ def _resolve_one_arg( # active is a host-model-only attribute; read it from the host entry only. active = host_entry.active if host_entry is not None else '' - sv: Optional[SuiteVar] = suite_vars.get(std_name) + suite_var: Optional[SuiteVar] = suite_vars.get(std_name) - if host_entry is not None and sv is None: + if host_entry is not None and suite_var is None: source = 'control' if host_entry.is_control else 'host' - elif sv is not None and host_entry is None: + elif suite_var is not None and host_entry is None: source = 'suite' - elif host_entry is None and sv is None: + elif host_entry is None and suite_var is None: # Case 2 or 3. if intent == 'out': inst_entry = host_dict.get('instance_number') inst_access = '({})'.format(inst_entry.local_name) if inst_entry else '(1)' - sv = SuiteVar( + suite_var = SuiteVar( standard_name=std_name, local_name=local, type_=scheme_var.type, @@ -1335,7 +1335,7 @@ def _resolve_one_arg( inst_access=inst_access, allocatable=scheme_var.allocatable, ) - suite_vars[std_name] = sv + suite_vars[std_name] = suite_var source = 'suite' else: raise CCPPError( @@ -1362,11 +1362,11 @@ def _resolve_one_arg( host_kind = host_entry.kind host_allocatable = host_entry.allocatable else: - base_expr = sv.access_path - host_dims = sv.dimensions - host_units = sv.units - host_kind = sv.kind - host_allocatable = sv.allocatable + base_expr = suite_var.access_path + host_dims = suite_var.dimensions + host_units = suite_var.units + host_kind = suite_var.kind + host_allocatable = suite_var.allocatable # ---- allocatable compatibility check --------------------------------- # An actual argument that is not allocatable cannot be passed to an @@ -1534,7 +1534,7 @@ def _resolve_one_arg( active_local=active_local, source=source, host_entry=host_entry, - suite_var=sv if source == 'suite' else None, + suite_var=suite_var if source == 'suite' else None, base_expr=base_expr, subscript=subscript, call_expr=call_expr, @@ -1899,7 +1899,7 @@ def resolve_suite( resolved_groups: List[ResolvedGroup] = [] for group in suite.groups: - rg = ResolvedGroup(group_name=group.name) + resolved_group = ResolvedGroup(group_name=group.name) for phase in phases: used_local_names_phase: Set[str] = set() @@ -1928,22 +1928,22 @@ def resolve_suite( ) if items_for_phase: - rg.phase_calls[phase] = items_for_phase + resolved_group.phase_calls[phase] = items_for_phase # Collect dimension variable USE info for this group. - rg.dim_uses = _collect_dim_uses(rg, host_dict, suite_vars=suite_vars) - resolved_groups.append(rg) + resolved_group.dim_uses = _collect_dim_uses(resolved_group, host_dict, suite_vars=suite_vars) + resolved_groups.append(resolved_group) # Constituent register calls: gather the (scheme_name, scheme_local_name) # pairs for every register-phase arg that was flagged as a constituent. # The suite cap uses these to emit two-pass merge logic. constituent_calls: List[Tuple[str, str]] = [] - for rg in resolved_groups: - for rc in iter_phase_calls(rg.phase_calls.get('register', [])): - for arg in rc.args: + for resolved_group in resolved_groups: + for resolved_call in iter_phase_calls(resolved_group.phase_calls.get('register', [])): + for arg in resolved_call.args: if arg.is_constituent_arg: constituent_calls.append( - (rc.scheme_name, arg.scheme_local_name) + (resolved_call.scheme_name, arg.scheme_local_name) ) # Walk every constituent-sourced arg (excluding the legacy # register-phase ccpp_constituent_properties_t case) and collect: @@ -1951,10 +1951,10 @@ def resolve_suite( # * constituent_index_names — base std names X needing an index_of_X uses_constituents = False index_names: Set[str] = set() - for rg in resolved_groups: - for items in rg.phase_calls.values(): - for rc in iter_phase_calls(items): - for arg in rc.args: + for resolved_group in resolved_groups: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: if arg.source != 'constituent' or arg.is_constituent_arg: continue uses_constituents = True @@ -2024,8 +2024,8 @@ def _collect_scheme_names(group) -> List[str]: if isinstance(item, SuiteScheme): names.append(item.name) elif isinstance(item, (SuiteSubcycle, SuiteSubcol)): - for sn in item.scheme_names(): - names.append(sn) + for scheme_name in item.scheme_names(): + names.append(scheme_name) return names @@ -2039,11 +2039,11 @@ def _dedup_scheme_names(scheme_names: List[str]) -> List[str]: """ seen: Set[str] = set() deduped: List[str] = [] - for sn in scheme_names: - if sn in seen: + for scheme_name in scheme_names: + if scheme_name in seen: continue - seen.add(sn) - deduped.append(sn) + seen.add(scheme_name) + deduped.append(scheme_name) return deduped @@ -2068,17 +2068,17 @@ def _resolve_one_call( vars_list = scheme_store.variables_for(scheme_name, phase) if vars_list is None: return None - rc = ResolvedCall( + resolved_call = ResolvedCall( scheme_name=scheme_name, phase=phase, scheme_module=scheme_store.module_for(scheme_name), ) - for sv in vars_list: + for scheme_var in vars_list: arg = _resolve_one_arg( - sv, phase, host_dict, suite_vars, scheme_name, used_local_names, + scheme_var, phase, host_dict, suite_vars, scheme_name, used_local_names, suite_name=suite_name, loop_context=loop_context, ) - rc.args.append(arg) - return rc + resolved_call.args.append(arg) + return resolved_call def _resolve_flat_phase( @@ -2092,12 +2092,12 @@ def _resolve_flat_phase( ) -> List[ResolvedCall]: """Resolve a flat (non-subcycle) phase into a list of ResolvedCall.""" result: List[ResolvedCall] = [] - for sn in scheme_names: - rc = _resolve_one_call(sn, phase, scheme_store, host_dict, + for scheme_name in scheme_names: + resolved_call = _resolve_one_call(scheme_name, phase, scheme_store, host_dict, suite_vars, used_local_names, suite_name=suite_name) - if rc is not None: - result.append(rc) + if resolved_call is not None: + result.append(resolved_call) return result @@ -2137,14 +2137,14 @@ def _resolve_items( out: List[PhaseItem] = [] for sub in suite_items: if isinstance(sub, SuiteScheme): - rc = _resolve_one_call( + resolved_call = _resolve_one_call( sub.name, phase, scheme_store, host_dict, suite_vars, used_local_names, suite_name=suite_name, loop_context=loop_context, ) - if rc is not None: - out.append(rc) + if resolved_call is not None: + out.append(resolved_call) elif isinstance(sub, SuiteSubcycle): loop_count, loop_std = _resolve_subcycle_loop_bound( sub.loop, host_dict, suite_vars=suite_vars, @@ -2160,27 +2160,27 @@ def _resolve_items( elif isinstance(sub, SuiteSubcol): # SuiteSubcol is flattened in place — the framework # doesn't render it as a separate loop level. - for sn in sub.scheme_names(): - rc = _resolve_one_call( - sn, phase, scheme_store, host_dict, + for scheme_name in sub.scheme_names(): + resolved_call = _resolve_one_call( + scheme_name, phase, scheme_store, host_dict, suite_vars, used_local_names, suite_name=suite_name, loop_context=loop_context, ) - if rc is not None: - out.append(rc) + if resolved_call is not None: + out.append(resolved_call) return out result: List[PhaseItem] = [] for item in group.items: if isinstance(item, SuiteScheme): - rc = _resolve_one_call(item.name, phase, scheme_store, host_dict, + resolved_call = _resolve_one_call(item.name, phase, scheme_store, host_dict, suite_vars, used_local_names, suite_name=suite_name, loop_context=[]) - if rc is not None: - result.append(rc) + if resolved_call is not None: + result.append(resolved_call) elif isinstance(item, SuiteSubcycle): loop_count, loop_std = _resolve_subcycle_loop_bound( item.loop, host_dict, suite_vars=suite_vars, @@ -2194,19 +2194,19 @@ def _resolve_items( loop_std_name=loop_std, )) elif isinstance(item, SuiteSubcol): - for sn in item.scheme_names(): - rc = _resolve_one_call(sn, phase, scheme_store, host_dict, + for scheme_name in item.scheme_names(): + resolved_call = _resolve_one_call(scheme_name, phase, scheme_store, host_dict, suite_vars, used_local_names, suite_name=suite_name, loop_context=[]) - if rc is not None: - result.append(rc) + if resolved_call is not None: + result.append(resolved_call) return result def _collect_dim_uses( - rg: ResolvedGroup, + resolved_group: ResolvedGroup, host_dict: Dict[str, HostVarEntry], suite_vars: Optional[Dict[str, 'SuiteVar']] = None, ) -> Dict[str, Set[str]]: @@ -2219,9 +2219,9 @@ def _collect_dim_uses( expressions. """ dim_uses: Dict[str, Set[str]] = {} - for items in rg.phase_calls.values(): - for rc in iter_phase_calls(items): - for arg in rc.args: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: for dim_std in arg.used_dim_std_names: entry = host_dict.get(dim_std) if entry is not None and entry.module_name is not None: @@ -2235,8 +2235,8 @@ def _collect_dim_uses( sym = _root_symbol(entry.access_path) dim_uses.setdefault(mod, set()).add(sym) elif suite_vars and dim_std in suite_vars: - sv = suite_vars[dim_std] - dim_uses.setdefault(sv.module_name, set()).add( + suite_var = suite_vars[dim_std] + dim_uses.setdefault(suite_var.module_name, set()).add( 'ccpp_suite_data') # Subcycle loop bounds resolved from CCPP standard names also need # a USE entry (or, for control vars, a dummy arg — handled elsewhere). @@ -2254,8 +2254,8 @@ def _collect_dim_uses( _root_symbol(entry.access_path) ) elif suite_vars and item.loop_std_name in suite_vars: - sv = suite_vars[item.loop_std_name] - dim_uses.setdefault(sv.module_name, set()).add( + suite_var = suite_vars[item.loop_std_name] + dim_uses.setdefault(suite_var.module_name, set()).add( 'ccpp_suite_data' ) return dim_uses diff --git a/capgen-ng/generator/suite_types.py b/capgen-ng/generator/suite_types.py index 20c0dea7..7f46247e 100644 --- a/capgen-ng/generator/suite_types.py +++ b/capgen-ng/generator/suite_types.py @@ -216,10 +216,10 @@ def _collect_ptr_type_combos( set of (type_, kind, rank) """ combos: Set[Tuple[str, str, int]] = set() - for rg in suite_res.groups: - for items in rg.phase_calls.values(): - for rc in iter_phase_calls(items): - for arg in rc.args: + for resolved_group in suite_res.groups: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: if arg.ptr_name: combos.add(_ptr_type_for_arg(arg)) return combos diff --git a/capgen-ng/generator/suite_xml.py b/capgen-ng/generator/suite_xml.py index 0dfee524..fefbcf8a 100644 --- a/capgen-ng/generator/suite_xml.py +++ b/capgen-ng/generator/suite_xml.py @@ -155,13 +155,13 @@ class SuiteSubcycle: Examples -------- - >>> sc = SuiteSubcycle(loop='2', items=[SuiteScheme('sch_a')]) - >>> sc.loop + >>> subcycle = SuiteSubcycle(loop='2', items=[SuiteScheme('sch_a')]) + >>> subcycle.loop '2' - >>> sc.is_literal_count + >>> subcycle.is_literal_count True - >>> sc2 = SuiteSubcycle(loop='num_subcycles_for_ag', items=[]) - >>> sc2.is_literal_count + >>> subcycle2 = SuiteSubcycle(loop='num_subcycles_for_ag', items=[]) + >>> subcycle2.is_literal_count False """ diff --git a/unit-tests/test_ccpp_datafile.py b/unit-tests/test_ccpp_datafile.py index ee304436..e66dc345 100644 --- a/unit-tests/test_ccpp_datafile.py +++ b/unit-tests/test_ccpp_datafile.py @@ -48,9 +48,9 @@ def _build_datatable(tmpdir, host_name='test_host', hd[first].protected = True store = _load_scheme_store() suite = _parse_suite('suite_test_simple.xml') - sr = resolve_suite(suite, store, hd) + suite_resolution = resolve_suite(suite, store, hd) return write_datatable( - [sr], + [suite_resolution], store, utility_paths or ['/out/ccpp_kinds.F90'], suite_file_paths or ['/out/ccpp_test_simple_cap.F90', @@ -121,7 +121,7 @@ def setUp(self): self._tmpdir = tempfile.mkdtemp() self._datatable = _build_datatable( self._tmpdir, - suite_meta_paths=['/out/ccpp_test_simple.meta'], + suite_meta_paths=['/out/ccpp_test_simple_data.meta'], expanded_sdf_paths=['/out/ccpp_test_simple_expanded.xml'], ) @@ -132,14 +132,14 @@ def test_inspection_files_returns_meta_and_expanded(self): out = datatable_report(self._datatable, DatatableReport('inspection_files'), ',') items = out.split(',') - self.assertIn('/out/ccpp_test_simple.meta', items) + self.assertIn('/out/ccpp_test_simple_data.meta', items) self.assertIn('/out/ccpp_test_simple_expanded.xml', items) def test_inspection_files_excluded_from_capgen_files(self): out = datatable_report(self._datatable, DatatableReport('capgen_files'), ',') items = out.split(',') - self.assertNotIn('/out/ccpp_test_simple.meta', items) + self.assertNotIn('/out/ccpp_test_simple_data.meta', items) self.assertNotIn('/out/ccpp_test_simple_expanded.xml', items) def test_inspection_files_empty_when_none_given(self): @@ -345,8 +345,8 @@ def test_main_suite_list(self): import contextlib buf = io.StringIO() with contextlib.redirect_stdout(buf): - rc = cdf.main([self._datatable, '--suite-list']) - self.assertEqual(rc, 0) + resolved_call = cdf.main([self._datatable, '--suite-list']) + self.assertEqual(resolved_call, 0) self.assertEqual(buf.getvalue().strip(), 'test_simple') def test_main_mutually_exclusive(self): diff --git a/unit-tests/test_datatable.py b/unit-tests/test_datatable.py index b4a54227..a1811296 100644 --- a/unit-tests/test_datatable.py +++ b/unit-tests/test_datatable.py @@ -27,10 +27,10 @@ def _write(tmpdir, suite_xml='suite_test_simple.xml', utility_paths=None, suite_file_paths=None, host_file_paths=None, host_dict=None, host_name='test_host', suite_meta_paths=None, expanded_sdf_paths=None): - sr, store = _resolve(suite_xml) + suite_resolution, store = _resolve(suite_xml) return ( write_datatable( - [sr], + [suite_resolution], store, utility_paths or ['/out/ccpp_kinds.F90', '/out/ccpp_static_api.F90'], suite_file_paths or ['/out/ccpp_test_simple_cap.F90'], @@ -41,7 +41,7 @@ def _write(tmpdir, suite_xml='suite_test_simple.xml', utility_paths=None, host_dict=host_dict, host_name=host_name, ), - sr, + suite_resolution, store, ) @@ -138,7 +138,7 @@ def setUp(self): self._tmpdir = tempfile.mkdtemp() path, _, _ = _write( self._tmpdir, - suite_meta_paths=['/out/ccpp_test_simple.meta'], + suite_meta_paths=['/out/ccpp_test_simple_data.meta'], expanded_sdf_paths=['/out/ccpp_test_simple_expanded.xml'], ) self._root = ET.parse(path).getroot() @@ -157,7 +157,7 @@ def test_suite_meta_files_subsection(self): meta = self._inspection().find('suite_meta_files') self.assertIsNotNone(meta) files = [f.text for f in meta.findall('file')] - self.assertEqual(files, ['/out/ccpp_test_simple.meta']) + self.assertEqual(files, ['/out/ccpp_test_simple_data.meta']) def test_expanded_sdf_files_subsection(self): exp = self._inspection().find('expanded_sdf_files') @@ -335,9 +335,9 @@ class TestDependenciesPopulated(unittest.TestCase): def setUp(self): self._tmpdir = tempfile.mkdtemp() - sr, store = _resolve() + suite_resolution, store = _resolve() self._path = write_datatable( - [sr], + [suite_resolution], store, [], [], @@ -415,9 +415,9 @@ def test_explicit_diagnostic_name_emitted(self): target = next(mv for mv in mvars if mv.standard_name == 'air_temperature') target._diagnostic_name = 'temperature' - sr, _ = _resolve() + suite_resolution, _ = _resolve() path = write_datatable( - [sr], store, + [suite_resolution], store, ['/out/ccpp_kinds.F90'], ['/out/ccpp_test_simple_cap.F90'], d, @@ -430,13 +430,13 @@ def test_explicit_diagnostic_name_emitted(self): def test_diagnostic_name_fixed_emitted(self): with tempfile.TemporaryDirectory() as d: - sr, store = _resolve() + suite_resolution, store = _resolve() mvars = store.variables_for('temp_calc_adjust', 'run') target = next(mv for mv in mvars if mv.standard_name == 'air_temperature') target.diagnostic_name_fixed = 'Q' path = write_datatable( - [sr], store, + [suite_resolution], store, ['/out/ccpp_kinds.F90'], ['/out/ccpp_test_simple_cap.F90'], d, diff --git a/unit-tests/test_host_constituents.py b/unit-tests/test_host_constituents.py index c57b8566..16a5433f 100644 --- a/unit-tests/test_host_constituents.py +++ b/unit-tests/test_host_constituents.py @@ -14,7 +14,7 @@ def _resolve_consumer(): - """Resolve the consume_constituent fixture; return (sr, host_dict).""" + """Resolve the consume_constituent fixture; return (suite_resolution, host_dict).""" from test_suite_resolver import ( _load_constituent_host_dict, _load_constituent_consumer_store, @@ -27,7 +27,7 @@ def _resolve_consumer(): def _resolve_register(): - """Resolve the register_constituents fixture; return (sr, host_dict).""" + """Resolve the register_constituents fixture; return (suite_resolution, host_dict).""" from test_suite_resolver import ( _load_constituent_host_dict, _load_constituent_scheme_store, @@ -40,7 +40,7 @@ def _resolve_register(): def _resolve_simple(): - """Resolve the no-constituent fixture; return (sr, host_dict).""" + """Resolve the no-constituent fixture; return (suite_resolution, host_dict).""" from test_suite_resolver import ( _load_full_host_dict, _load_scheme_store, @@ -53,29 +53,29 @@ def _resolve_simple(): def _render_consumer(): - sr, hd = _resolve_consumer() - return '\n'.join(_generate_host_constituents([sr], host_dict=hd)) + suite_resolution, hd = _resolve_consumer() + return '\n'.join(_generate_host_constituents([suite_resolution], host_dict=hd)) def _render_register(): - sr, hd = _resolve_register() - return '\n'.join(_generate_host_constituents([sr], host_dict=hd)) + suite_resolution, hd = _resolve_register() + return '\n'.join(_generate_host_constituents([suite_resolution], host_dict=hd)) class TestAggregationHelpers(unittest.TestCase): """``_any_constituent_state`` / ``_all_index_names`` / ``_suites_with_register_consts``.""" def test_any_when_consumer_only(self): - sr, _hd = _resolve_consumer() - self.assertTrue(_any_constituent_state([sr])) + suite_resolution, _hd = _resolve_consumer() + self.assertTrue(_any_constituent_state([suite_resolution])) def test_any_when_register_only(self): - sr, _hd = _resolve_register() - self.assertTrue(_any_constituent_state([sr])) + suite_resolution, _hd = _resolve_register() + self.assertTrue(_any_constituent_state([suite_resolution])) def test_any_when_neither(self): - sr, _hd = _resolve_simple() - self.assertFalse(_any_constituent_state([sr])) + suite_resolution, _hd = _resolve_simple() + self.assertFalse(_any_constituent_state([suite_resolution])) def test_index_names_aggregated(self): consumer, _ch = _resolve_consumer() @@ -100,8 +100,8 @@ class TestModuleSkippedWhenNoConstituents(unittest.TestCase): constituent state — the module is not emitted at all.""" def test_returns_none(self): - sr, _hd = _resolve_simple() - self.assertIsNone(_generate_host_constituents([sr])) + suite_resolution, _hd = _resolve_simple() + self.assertIsNone(_generate_host_constituents([suite_resolution])) class TestModuleHeaderAndUses(unittest.TestCase): diff --git a/unit-tests/test_integration.py b/unit-tests/test_integration.py index 1eb2a656..4642eab1 100644 --- a/unit-tests/test_integration.py +++ b/unit-tests/test_integration.py @@ -551,7 +551,7 @@ def test_suite_meta_in_inspection_files(self): meta_files = inspection.find('suite_meta_files') self.assertIsNotNone(meta_files) names = [os.path.basename(f.text) for f in meta_files.findall('file')] - self.assertIn('ccpp_test_simple.meta', names) + self.assertIn('ccpp_test_simple_data.meta', names) def test_suite_meta_not_in_capgen_files(self): capgen_files = self._root.find('capgen_files') @@ -2116,14 +2116,14 @@ def test_static_api_physics_run_has_inst_num(self): self.assertIn('inst_num', physics_run) def test_suite_meta_file_exists(self): - """Polish 2: ccpp_.meta must be generated.""" + """Polish 2: ccpp__data.meta must be generated.""" self.assertTrue( - os.path.isfile(os.path.join(self._tmpdir, 'ccpp_interstitial.meta')) + os.path.isfile(os.path.join(self._tmpdir, 'ccpp_interstitial_data.meta')) ) def test_suite_meta_contains_suite_var(self): """Suite meta file must list suite-owned variable.""" - with open(os.path.join(self._tmpdir, 'ccpp_interstitial.meta')) as fh: + with open(os.path.join(self._tmpdir, 'ccpp_interstitial_data.meta')) as fh: text = fh.read() self.assertIn('diagnostic_interstitial_field', text) self.assertIn('type = suite', text) @@ -2271,11 +2271,11 @@ def test_timestep_final_resets_to_initialized(self): # --------------------------------------------------------------------------- -# Test: Polish 2 — ccpp_.meta output file +# Test: Polish 2 — ccpp__data.meta output file # --------------------------------------------------------------------------- class TestSuiteMetaOutput(unittest.TestCase): - """Polish 2: ccpp_.meta must be written for every suite.""" + """Polish 2: ccpp__data.meta must be written for every suite.""" def setUp(self): self._tmpdir = tempfile.mkdtemp() @@ -2287,17 +2287,17 @@ def tearDown(self): def test_meta_file_created(self): self.assertTrue( - os.path.isfile(os.path.join(self._tmpdir, 'ccpp_test_simple.meta')) + os.path.isfile(os.path.join(self._tmpdir, 'ccpp_test_simple_data.meta')) ) def test_meta_header_comment(self): - with open(os.path.join(self._tmpdir, 'ccpp_test_simple.meta')) as fh: + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_data.meta')) as fh: text = fh.read() self.assertTrue(text.startswith('!')) self.assertIn('ccpp_test_simple', text.split('\n')[0]) def test_meta_table_properties(self): - with open(os.path.join(self._tmpdir, 'ccpp_test_simple.meta')) as fh: + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_data.meta')) as fh: text = fh.read() self.assertIn('[ccpp-table-properties]', text) self.assertIn('name = ccpp_test_simple_data', text) @@ -2305,7 +2305,7 @@ def test_meta_table_properties(self): def test_meta_empty_suite_has_no_var_entries(self): """simple suite has no suite-owned vars so no [ var ] blocks.""" - with open(os.path.join(self._tmpdir, 'ccpp_test_simple.meta')) as fh: + with open(os.path.join(self._tmpdir, 'ccpp_test_simple_data.meta')) as fh: text = fh.read() self.assertNotIn('standard_name =', text) @@ -2323,7 +2323,7 @@ def test_meta_with_suite_vars_lists_them(self): output_root=d, kind_types={}, ) - with open(os.path.join(d, 'ccpp_interstitial.meta')) as fh: + with open(os.path.join(d, 'ccpp_interstitial_data.meta')) as fh: text = fh.read() self.assertIn('standard_name = diagnostic_interstitial_field', text) self.assertIn('units = K', text) diff --git a/unit-tests/test_static_api.py b/unit-tests/test_static_api.py index b8463a23..22ae49e1 100644 --- a/unit-tests/test_static_api.py +++ b/unit-tests/test_static_api.py @@ -36,16 +36,16 @@ def _resolve(): def _generate(): - sr = _resolve() - return _generate_static_api(['test_simple'], [sr]) + suite_resolution = _resolve() + return _generate_static_api(['test_simple'], [suite_resolution]) class TestAllCtrlArgsForPhase(unittest.TestCase): def test_only_error_ctrl_args_in_test_case(self): - sr = _resolve() + suite_resolution = _resolve() # temp_calc_adjust uses errmsg/errflg which are now control vars. - args = _all_ctrl_args_for_phase([sr], 'run') + args = _all_ctrl_args_for_phase([suite_resolution], 'run') std_names = {a.standard_name for a in args} self.assertEqual(std_names, {'ccpp_error_message', 'ccpp_error_code'}) @@ -124,9 +124,9 @@ def setUp(self): hd = _load_constituent_host_dict() store = _load_constituent_consumer_store() suite = _parse_suite('suite_consume_constituent.xml') - sr = resolve_suite(suite, store, hd) + suite_resolution = resolve_suite(suite, store, hd) self.text = '\n'.join( - _generate_static_api(['consume_consts'], [sr], host_dict=hd, + _generate_static_api(['consume_consts'], [suite_resolution], host_dict=hd, scheme_store=store), ) @@ -253,8 +253,8 @@ class TestCcppPhysicsUnknownSuiteErrors(unittest.TestCase): def setUp(self): hd = _load_full_host_dict() - sr = _resolve() - self.text = '\n'.join(_generate_static_api(['test_simple'], [sr], hd)) + suite_resolution = _resolve() + self.text = '\n'.join(_generate_static_api(['test_simple'], [suite_resolution], hd)) def test_physics_run_has_default_case_with_errflg(self): run_block_start = self.text.index('subroutine ccpp_physics_run') @@ -278,11 +278,11 @@ class TestMultipleSuites(unittest.TestCase): """Static API with two suites uses select case for both.""" def setUp(self): - sr = _resolve() + suite_resolution = _resolve() from copy import deepcopy - sr2 = deepcopy(sr) + sr2 = deepcopy(suite_resolution) sr2.suite_name = 'suite_b' - lines = _generate_static_api(['test_simple', 'suite_b'], [sr, sr2]) + lines = _generate_static_api(['test_simple', 'suite_b'], [suite_resolution, sr2]) self.text = '\n'.join(lines) def test_both_suites_in_register(self): @@ -298,16 +298,16 @@ def test_both_suite_caps_used(self): class TestWriteStaticApi(unittest.TestCase): def test_writes_file(self): - sr = _resolve() + suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: - path = write_static_api(['test_simple'], [sr], tmpdir) + path = write_static_api(['test_simple'], [suite_resolution], tmpdir) self.assertTrue(os.path.isfile(path)) self.assertEqual(os.path.basename(path), 'ccpp_static_api.F90') def test_file_content(self): - sr = _resolve() + suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: - path = write_static_api(['test_simple'], [sr], tmpdir) + path = write_static_api(['test_simple'], [suite_resolution], tmpdir) with open(path) as fh: content = fh.read() self.assertIn('module ccpp_static_api', content) @@ -316,16 +316,16 @@ def test_file_content(self): self.assertTrue(content.endswith('\n')) def test_creates_output_dir(self): - sr = _resolve() + suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: subdir = os.path.join(tmpdir, 'api') - write_static_api(['test_simple'], [sr], subdir) + write_static_api(['test_simple'], [suite_resolution], subdir) self.assertTrue(os.path.isdir(subdir)) def test_returns_absolute_path(self): - sr = _resolve() + suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: - path = write_static_api(['test_simple'], [sr], tmpdir) + path = write_static_api(['test_simple'], [suite_resolution], tmpdir) self.assertTrue(os.path.isabs(path)) @@ -335,8 +335,8 @@ class TestCcppInitMultiInstance(unittest.TestCase): def setUp(self): hd = _load_full_host_dict() - sr = _resolve() - lines = _generate_static_api(['test_simple'], [sr], hd) + suite_resolution = _resolve() + lines = _generate_static_api(['test_simple'], [suite_resolution], hd) self.text = '\n'.join(lines) def test_init_signature_has_instance_number(self): @@ -377,8 +377,8 @@ class TestCcppInitSingleInstance(unittest.TestCase): def setUp(self): hd = {k: v for k, v in _load_full_host_dict().items() if k not in ('number_of_instances', 'instance_number')} - sr = _resolve() - lines = _generate_static_api(['test_simple'], [sr], hd) + suite_resolution = _resolve() + lines = _generate_static_api(['test_simple'], [suite_resolution], hd) self.text = '\n'.join(lines) def test_init_signature_no_instance_args(self): @@ -493,13 +493,13 @@ class TestCollectHostIo(unittest.TestCase): """_collect_host_io: intent partitioning, control-var exclusion, sort.""" def setUp(self): - self.sr = _resolve() + self.suite_resolution = _resolve() self.hd = _load_full_host_dict() def test_includes_control_vars(self): # temp_calc_adjust declares errflg/errmsg with intent=out — they # appear in outputs. Matches original capgen's introspection. - inputs, outputs = _collect_host_io(self.sr, self.hd) + inputs, outputs = _collect_host_io(self.suite_resolution, self.hd) self.assertIn('ccpp_error_code', outputs) self.assertIn('ccpp_error_message', outputs) # …and not in inputs (intent=out only, not inout). @@ -507,28 +507,28 @@ def test_includes_control_vars(self): self.assertNotIn('ccpp_error_message', inputs) def test_includes_host_args(self): - inputs, outputs = _collect_host_io(self.sr, self.hd) + inputs, outputs = _collect_host_io(self.suite_resolution, self.hd) # air_temperature is intent=inout in run phase → both lists. self.assertIn('air_temperature', inputs) self.assertIn('air_temperature', outputs) def test_sorted_outputs(self): - inputs, outputs = _collect_host_io(self.sr, self.hd) + inputs, outputs = _collect_host_io(self.suite_resolution, self.hd) self.assertEqual(inputs, sorted(inputs)) self.assertEqual(outputs, sorted(outputs)) def test_collapse_ddts_no_ddts_unchanged(self): # host_full has no DDTs → collapse is a no-op. - flat_in, flat_out = _collect_host_io(self.sr, self.hd, collapse_ddts=False) - coll_in, coll_out = _collect_host_io(self.sr, self.hd, collapse_ddts=True) + flat_in, flat_out = _collect_host_io(self.suite_resolution, self.hd, collapse_ddts=False) + coll_in, coll_out = _collect_host_io(self.suite_resolution, self.hd, collapse_ddts=True) self.assertEqual(flat_in, coll_in) self.assertEqual(flat_out, coll_out) def test_no_host_dict_collapse_falls_back(self): # collapse_ddts=True without host_dict must not raise. # With no DDTs the result is identical to the non-collapsed view. - no_hd_in, _ = _collect_host_io(self.sr, None, collapse_ddts=True) - flat_in, _ = _collect_host_io(self.sr, self.hd, collapse_ddts=False) + no_hd_in, _ = _collect_host_io(self.suite_resolution, None, collapse_ddts=True) + flat_in, _ = _collect_host_io(self.suite_resolution, self.hd, collapse_ddts=False) self.assertEqual(no_hd_in, flat_in) @@ -636,14 +636,14 @@ def setUp(self): fh.write(suite_xml) suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), skip_validation=True) - self.sr = resolve_suite(suite, store, self.hd) + self.suite_resolution = resolve_suite(suite, store, self.hd) def test_number_of_ccpp_constituents_in_inputs(self): - inputs, _ = _collect_host_io(self.sr, self.hd) + inputs, _ = _collect_host_io(self.suite_resolution, self.hd) self.assertIn('number_of_ccpp_constituents', inputs) def test_horizontal_dim_not_in_inputs(self): - inputs, _ = _collect_host_io(self.sr, self.hd) + inputs, _ = _collect_host_io(self.suite_resolution, self.hd) # Sanity: the host-side dims are NOT included even though they # appear as scheme arg dimensions. self.assertNotIn('horizontal_dimension', inputs) @@ -717,10 +717,10 @@ def setUp(self): fh.write(suite_xml) suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), skip_validation=True) - self.sr = resolve_suite(suite, scheme_store, self.hd) + self.suite_resolution = resolve_suite(suite, scheme_store, self.hd) def test_subcycle_std_name_in_inputs(self): - inputs, _outputs = _collect_host_io(self.sr, self.hd) + inputs, _outputs = _collect_host_io(self.suite_resolution, self.hd) self.assertIn('num_subcycles_for_my_scheme', inputs) def test_integer_literal_subcycle_does_not_pollute(self): @@ -752,8 +752,8 @@ def test_integer_literal_subcycle_does_not_pollute(self): fh.write(suite_xml) suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), skip_validation=True) - sr = resolve_suite(suite, store, hd) - inputs, _ = _collect_host_io(sr, hd) + suite_resolution = resolve_suite(suite, store, hd) + inputs, _ = _collect_host_io(suite_resolution, hd) # No spurious integer / std-name additions from the literal loop. self.assertNotIn('3', inputs) @@ -845,15 +845,15 @@ def setUp(self): fh.write(suite_xml) suite = parse_suite_xml(xml_path, tmp, logging.getLogger('t'), skip_validation=True) - self.sr = resolve_suite(suite, store, self.hd) + self.suite_resolution = resolve_suite(suite, store, self.hd) def test_active_flag_in_inputs(self): - inputs, _outputs = _collect_host_io(self.sr, self.hd) + inputs, _outputs = _collect_host_io(self.suite_resolution, self.hd) self.assertIn('flag_for_passive_check', inputs) def test_active_flag_not_in_outputs(self): """The flag is a pure input — it must not leak into outputs.""" - _, outputs = _collect_host_io(self.sr, self.hd) + _, outputs = _collect_host_io(self.suite_resolution, self.hd) self.assertNotIn('flag_for_passive_check', outputs) @@ -888,8 +888,8 @@ def test_multi_suite(self): class TestSuitePartListSubroutine(unittest.TestCase): def setUp(self): - self.sr = _resolve() - self.text = '\n'.join(_suite_part_list_subroutine(['test_simple'], [self.sr])) + self.suite_resolution = _resolve() + self.text = '\n'.join(_suite_part_list_subroutine(['test_simple'], [self.suite_resolution])) def test_signature(self): self.assertIn('subroutine ccpp_physics_suite_part_list(', self.text) @@ -904,7 +904,7 @@ def test_signature(self): def test_dispatch_and_groups(self): self.assertIn("case ('test_simple')", self.text) # test_simple has one group named 'physics'. - group_names = [g.group_name for g in self.sr.groups] + group_names = [g.group_name for g in self.suite_resolution.groups] for i, gname in enumerate(group_names): self.assertIn("part_list({}) = '{}'".format(i + 1, gname), self.text) self.assertIn('allocate(part_list({}))'.format(len(group_names)), self.text) @@ -940,9 +940,9 @@ def test_errmsg_is_assumed_length(self): class TestSuiteSchemesSubroutine(unittest.TestCase): def setUp(self): - self.sr = _resolve() + self.suite_resolution = _resolve() self.text = '\n'.join( - _suite_schemes_subroutine(['test_simple'], [self.sr]) + _suite_schemes_subroutine(['test_simple'], [self.suite_resolution]) ) def test_signature(self): @@ -984,10 +984,10 @@ def test_errmsg_is_assumed_length(self): class TestSuiteVariablesSubroutine(unittest.TestCase): def setUp(self): - self.sr = _resolve() + self.suite_resolution = _resolve() self.hd = _load_full_host_dict() self.text = '\n'.join(_suite_io_subroutine( - ['test_simple'], [self.sr], self.hd, collapse_ddts=False, + ['test_simple'], [self.suite_resolution], self.hd, collapse_ddts=False, )) def test_subroutine_name(self): @@ -1056,10 +1056,10 @@ class TestSuiteHostDataSubroutine(unittest.TestCase): """Same shape as _variables; differs only in DDT collapsing.""" def setUp(self): - self.sr = _resolve() + self.suite_resolution = _resolve() self.hd = _load_full_host_dict() self.text = '\n'.join(_suite_io_subroutine( - ['test_simple'], [self.sr], self.hd, collapse_ddts=True, + ['test_simple'], [self.suite_resolution], self.hd, collapse_ddts=True, )) def test_subroutine_name(self): @@ -1077,7 +1077,7 @@ def test_no_ddts_matches_variables(self): # name and error message) should contain the same variable # literals as ..._variables. var_text = '\n'.join(_suite_io_subroutine( - ['test_simple'], [self.sr], self.hd, collapse_ddts=False, + ['test_simple'], [self.suite_resolution], self.hd, collapse_ddts=False, )) # A spot-check: any host-data variable in one is in the other. self.assertIn("'air_temperature'", self.text) @@ -1107,6 +1107,158 @@ def test_all_routines_present(self): self.assertIn('public :: {}'.format(sub), self.text) +######################################################################## +# Suite-introspection: --no-host-introspection stub bodies +######################################################################## + +class TestNoHostIntrospectionStubBodies(unittest.TestCase): + """When ``--no-host-introspection`` is set, each of the five + introspection routines retains its signature but the body is + replaced with an errflg=1 stub (or, for suite_list, an error_unit + write + empty allocation). Tests assert the stub shape per routine + and that signatures remain stable so existing host callers still + link.""" + + _DISABLED_MSG = ( + 'suite introspection disabled at code-generation time; ' + 'regenerate caps without --no-host-introspection' + ) + + @classmethod + def setUpClass(cls): + cls.suite_resolution = _resolve() + cls.hd = _load_full_host_dict() + + def test_suite_list_writes_error_unit_and_allocates_empty(self): + text = '\n'.join(_suite_list_subroutine( + ['a', 'b', 'c'], stub_body=True, + )) + # Signature preserved. + self.assertIn('subroutine ccpp_physics_suite_list(suites)', text) + # Stub body: error_unit message, empty allocation. + self.assertIn('write(error_unit,', text) + self.assertIn('ccpp_physics_suite_list:', text) + self.assertIn(self._DISABLED_MSG, text) + self.assertIn('allocate(suites(0))', text) + # The functional body must NOT appear. + self.assertNotIn("suites(1) = 'a'", text) + self.assertNotIn('allocate(suites(3))', text) + + def test_suite_part_list_stub(self): + text = '\n'.join(_suite_part_list_subroutine( + ['test_simple'], [self.suite_resolution], stub_body=True, + )) + self.assertIn('subroutine ccpp_physics_suite_part_list(', text) + self.assertIn('errflg = 1', text) + self.assertIn('ccpp_physics_suite_part_list: ' + self._DISABLED_MSG, + text) + self.assertIn('allocate(part_list(0))', text) + # Functional dispatch must not appear. + self.assertNotIn("case ('test_simple')", text) + self.assertNotIn('select case', text) + + def test_suite_schemes_stub(self): + text = '\n'.join(_suite_schemes_subroutine( + ['test_simple'], [self.suite_resolution], stub_body=True, + )) + self.assertIn('subroutine ccpp_physics_suite_schemes(', text) + self.assertIn('errflg = 1', text) + self.assertIn('ccpp_physics_suite_schemes: ' + self._DISABLED_MSG, + text) + self.assertIn('allocate(scheme_list(0))', text) + self.assertNotIn("scheme_list(1) =", text) + + def test_suite_variables_stub(self): + text = '\n'.join(_suite_io_subroutine( + ['test_simple'], [self.suite_resolution], self.hd, + collapse_ddts=False, stub_body=True, + )) + self.assertIn('subroutine ccpp_physics_suite_variables(', text) + self.assertIn('errflg = 1', text) + self.assertIn('ccpp_physics_suite_variables: ' + self._DISABLED_MSG, + text) + self.assertIn('allocate(variable_list(0))', text) + # The huge case-block must NOT appear. + self.assertNotIn('select case (trim(suite_name))', text) + self.assertNotIn("case ('test_simple')", text) + # Optional dummies are still declared so existing callers still + # type-check. + self.assertIn('logical, optional, intent(in) :: input_vars', + text) + self.assertIn('logical, optional, intent(in) :: output_vars', + text) + + def test_suite_host_data_stub(self): + text = '\n'.join(_suite_io_subroutine( + ['test_simple'], [self.suite_resolution], self.hd, + collapse_ddts=True, stub_body=True, + )) + self.assertIn('subroutine ccpp_physics_suite_host_data(', text) + self.assertIn('errflg = 1', text) + self.assertIn('ccpp_physics_suite_host_data: ' + self._DISABLED_MSG, + text) + self.assertIn('allocate(variable_list(0))', text) + self.assertNotIn('select case (trim(suite_name))', text) + + def test_module_imports_error_unit_only_when_stubbed(self): + # With stub on: iso_fortran_env appears for error_unit. + text_on = '\n'.join(_generate_static_api( + ['test_simple'], [self.suite_resolution], self.hd, + no_host_introspection=True, + )) + self.assertIn('use iso_fortran_env, only: error_unit', text_on) + # With stub off: no such import. + text_off = '\n'.join(_generate_static_api( + ['test_simple'], [self.suite_resolution], self.hd, + no_host_introspection=False, + )) + self.assertNotIn('use iso_fortran_env', text_off) + + def test_public_declarations_unchanged_when_stubbed(self): + # All five introspection routines remain public — callers must + # still link against them. + text = '\n'.join(_generate_static_api( + ['test_simple'], [self.suite_resolution], self.hd, + no_host_introspection=True, + )) + for sub in ( + 'ccpp_physics_suite_list', + 'ccpp_physics_suite_part_list', + 'ccpp_physics_suite_schemes', + 'ccpp_physics_suite_variables', + 'ccpp_physics_suite_host_data', + ): + self.assertIn('public :: {}'.format(sub), text) + self.assertIn('subroutine {}'.format(sub), text) + self.assertIn('end subroutine {}'.format(sub), text) + + def test_line_count_drops_dramatically(self): + """The motivating case: 33k+ lines → ~800. We don't have 80 + suites in unit-test fixtures, but even with one suite the + stubbed module must be strictly shorter than the full one.""" + full = _generate_static_api(['test_simple'], [self.suite_resolution], + self.hd, no_host_introspection=False) + stub = _generate_static_api(['test_simple'], [self.suite_resolution], + self.hd, no_host_introspection=True) + self.assertLess(len(stub), len(full), + 'stubbed module should be shorter than the full one ' + '(full={}, stub={})'.format(len(full), len(stub))) + + def test_write_static_api_passes_flag_through(self): + """``write_static_api(no_host_introspection=True, ...)`` must + produce a file containing stub bodies, not full ones.""" + with tempfile.TemporaryDirectory() as tmpdir: + out_path = write_static_api( + ['test_simple'], [self.suite_resolution], tmpdir, self.hd, + no_host_introspection=True, + ) + with open(out_path) as fh: + text = fh.read() + self.assertIn('ccpp_physics_suite_variables: ' + self._DISABLED_MSG, + text) + self.assertIn('use iso_fortran_env, only: error_unit', text) + + def load_tests(loader, tests, ignore): import generator.static_api as sa tests.addTests(doctest.DocTestSuite(sa)) diff --git a/unit-tests/test_suite_cap.py b/unit-tests/test_suite_cap.py index 5178fa99..a40e9fc7 100644 --- a/unit-tests/test_suite_cap.py +++ b/unit-tests/test_suite_cap.py @@ -31,28 +31,28 @@ def _resolve(): def _generate(): - sr, store = _resolve() - return _generate_suite_cap('test_simple', sr, store) + suite_resolution, store = _resolve() + return _generate_suite_cap('test_simple', suite_resolution, store) class TestAllSuiteSchemeNames(unittest.TestCase): def test_single_scheme(self): - sr, _ = _resolve() - names = _all_suite_scheme_names(sr) + suite_resolution, _ = _resolve() + names = _all_suite_scheme_names(suite_resolution) self.assertIn('temp_calc_adjust', names) def test_no_duplicates(self): - sr, _ = _resolve() - names = _all_suite_scheme_names(sr) + suite_resolution, _ = _resolve() + names = _all_suite_scheme_names(suite_resolution) self.assertEqual(len(names), len(set(names))) class TestSchemesWithRegister(unittest.TestCase): def test_none_have_register(self): - sr, store = _resolve() - names = _all_suite_scheme_names(sr) + suite_resolution, store = _resolve() + names = _all_suite_scheme_names(suite_resolution) reg = _schemes_with_register(names, store) # temp_calc_adjust has no register phase. self.assertEqual(reg, []) @@ -69,15 +69,15 @@ def test_scheme_with_register(self): class TestSuiteCtrlArgsForPhase(unittest.TestCase): def test_only_error_ctrl_args_in_test_case(self): - sr, _ = _resolve() + suite_resolution, _ = _resolve() # temp_calc_adjust uses errmsg/errflg which are now control vars. - args = _suite_ctrl_args_for_phase(sr, 'run') + args = _suite_ctrl_args_for_phase(suite_resolution, 'run') std_names = {a.standard_name for a in args} self.assertEqual(std_names, {'ccpp_error_message', 'ccpp_error_code'}) def test_unknown_phase_returns_empty(self): - sr, _ = _resolve() - args = _suite_ctrl_args_for_phase(sr, 'register') + suite_resolution, _ = _resolve() + args = _suite_ctrl_args_for_phase(suite_resolution, 'register') self.assertEqual(args, []) @@ -240,9 +240,9 @@ class TestGroupDispatchUnknownGroupError(unittest.TestCase): """ def setUp(self): - sr, store = _resolve() + suite_resolution, store = _resolve() self.text = '\n'.join( - _generate_suite_cap('test_simple', sr, store, _load_full_host_dict()) + _generate_suite_cap('test_simple', suite_resolution, store, _load_full_host_dict()) ) def _phase_block(self, phase): @@ -288,16 +288,16 @@ def test_all_phases_have_default_case(self): class TestWriteSuiteCap(unittest.TestCase): def test_writes_file(self): - sr, store = _resolve() + suite_resolution, store = _resolve() with tempfile.TemporaryDirectory() as tmpdir: - path = write_suite_cap('test_simple', sr, store, tmpdir) + path = write_suite_cap('test_simple', suite_resolution, store, tmpdir) self.assertTrue(os.path.isfile(path)) self.assertEqual(os.path.basename(path), 'ccpp_test_simple_cap.F90') def test_file_content(self): - sr, store = _resolve() + suite_resolution, store = _resolve() with tempfile.TemporaryDirectory() as tmpdir: - path = write_suite_cap('test_simple', sr, store, tmpdir) + path = write_suite_cap('test_simple', suite_resolution, store, tmpdir) with open(path) as fh: content = fh.read() self.assertIn('module ccpp_test_simple_cap', content) @@ -306,10 +306,10 @@ def test_file_content(self): self.assertTrue(content.endswith('\n')) def test_creates_output_dir(self): - sr, store = _resolve() + suite_resolution, store = _resolve() with tempfile.TemporaryDirectory() as tmpdir: subdir = os.path.join(tmpdir, 'caps') - write_suite_cap('test_simple', sr, store, subdir) + write_suite_cap('test_simple', suite_resolution, store, subdir) self.assertTrue(os.path.isdir(subdir)) @@ -355,9 +355,9 @@ def setUp(self): self.hd = _load_constituent_host_dict() self.store = _load_constituent_consumer_store() self.suite = _parse_suite('suite_consume_constituent.xml') - self.sr = resolve_suite(self.suite, self.store, self.hd) + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) self.text = '\n'.join( - _generate_suite_cap('consume_consts', self.sr, self.store, self.hd) + _generate_suite_cap('consume_consts', self.suite_resolution, self.store, self.hd) ) def test_no_kind_phys_import(self): @@ -404,8 +404,8 @@ def test_no_constituent_prop_ptr_type_import(self): def load_tests(loader, tests, ignore): - import generator.suite_cap as sc - tests.addTests(doctest.DocTestSuite(sc)) + import generator.suite_cap as subcycle + tests.addTests(doctest.DocTestSuite(subcycle)) return tests diff --git a/unit-tests/test_suite_data.py b/unit-tests/test_suite_data.py index 1e9d9f68..fcc438f1 100644 --- a/unit-tests/test_suite_data.py +++ b/unit-tests/test_suite_data.py @@ -129,8 +129,8 @@ class TestGenerateSuiteDataScalar(unittest.TestCase): """Suite var that is a scalar (no dimensions).""" def setUp(self): - sv = _make_sv('flag_var', 'flag', 'logical', '', '1', dims=[]) - self.lines = _generate_suite_data('s', {'flag_var': sv}) + suite_var = _make_sv('flag_var', 'flag', 'logical', '', '1', dims=[]) + self.lines = _generate_suite_data('s', {'flag_var': suite_var}) self.text = '\n'.join(self.lines) def test_no_allocatable_for_scalar(self): @@ -200,9 +200,9 @@ def test_missing_ddt_module_raises(self): def test_no_ddt_no_use(self): # When suite vars are all intrinsic, no DDT USE lines are emitted. - sv = _make_sv('temp', 't', 'real', 'kind_phys', 'K', dims=[]) + suite_var = _make_sv('temp', 't', 'real', 'kind_phys', 'K', dims=[]) lines = _generate_suite_data( - 'ds', {'temp': sv}, ddt_module_map=None, + 'ds', {'temp': suite_var}, ddt_module_map=None, ) text = '\n'.join(lines) self.assertNotIn('use make_ddt', text) diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 9304ef6f..b85732cc 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -786,8 +786,8 @@ def _scheme_var(self, local, std_name, intent='in', units='1', def test_case1_direct_host(self): """Case 1: scalar host variable, no transform.""" hd = self._host_dict() - sv = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count') - arg = _resolve_one_arg(sv, 'run', hd, {}, 'my_scheme', set()) + suite_var = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) self.assertEqual(arg.source, 'host') self.assertEqual(arg.transform_case, 1) self.assertEqual(arg.call_expr, 'im') @@ -796,18 +796,18 @@ def test_case1_direct_host(self): def test_case1_control_var(self): """Control variable → source='control', no USE module.""" hd = self._host_dict() - sv = self._scheme_var('thread_num', 'thread_number', 'in', '1') - arg = _resolve_one_arg(sv, 'run', hd, {}, 'my_scheme', set()) + suite_var = self._scheme_var('thread_num', 'thread_number', 'in', '1') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) self.assertEqual(arg.source, 'control') self.assertIsNone(arg.module_name) def test_case1_2d_array_run(self): """2D array in run phase → subscript applied.""" hd = self._host_dict() - sv = self._scheme_var('temp', 'air_temperature', 'inout', 'K', + suite_var = self._scheme_var('temp', 'air_temperature', 'inout', 'K', '(horizontal_loop_extent, vertical_layer_dimension)', 'real', 'kind_phys') - arg = _resolve_one_arg(sv, 'run', hd, {}, 'my_scheme', set()) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) # access_path = 'gt0', subscript = '(lb:ub, 1:nlev)' self.assertEqual(arg.call_expr, 'gt0(lb:ub, 1:nlev)') self.assertEqual(arg.transform_case, 1) @@ -815,10 +815,10 @@ def test_case1_2d_array_run(self): def test_case2_suite_owned(self): """Case 2: not in host, first use intent(out) → creates SuiteVar.""" hd = self._host_dict() - sv = self._scheme_var('new_var', 'brand_new_standard_name', 'out', 'K', + suite_var = self._scheme_var('new_var', 'brand_new_standard_name', 'out', 'K', '()', 'real', 'kind_phys') suite_vars: dict = {} - arg = _resolve_one_arg(sv, 'run', hd, suite_vars, 'my_scheme', set()) + arg = _resolve_one_arg(suite_var, 'run', hd, suite_vars, 'my_scheme', set()) self.assertEqual(arg.source, 'suite') self.assertIn('brand_new_standard_name', suite_vars) self.assertIsNotNone(arg.suite_var) @@ -826,9 +826,9 @@ def test_case2_suite_owned(self): def test_case3_not_found_intent_in_raises(self): """Case 3: not in host, intent(in) → CCPPError.""" hd = self._host_dict() - sv = self._scheme_var('missing', 'totally_missing_stdname', 'in', 'K') + suite_var = self._scheme_var('missing', 'totally_missing_stdname', 'in', 'K') with self.assertRaises(CCPPError) as cm: - _resolve_one_arg(sv, 'run', hd, {}, 'bad_scheme', set()) + _resolve_one_arg(suite_var, 'run', hd, {}, 'bad_scheme', set()) self.assertIn('totally_missing_stdname', str(cm.exception)) def test_case4_suite_data_reuse(self): @@ -870,8 +870,8 @@ def test_unit_transform_detected(self): extra_hd = build_flat_host_dict(extra_tbls, [], {}) combined = {**hd, **extra_hd} - sv = self._scheme_var('p_hpa', 'air_pressure', 'in', 'hPa', '()', 'real', 'kind_phys') - arg = _resolve_one_arg(sv, 'run', combined, {}, 'my_scheme', set()) + suite_var = self._scheme_var('p_hpa', 'air_pressure', 'in', 'hPa', '()', 'real', 'kind_phys') + arg = _resolve_one_arg(suite_var, 'run', combined, {}, 'my_scheme', set()) self.assertTrue(arg.needs_unit_transform) self.assertEqual(arg.transform_case, 3) self.assertIn('temp_name', arg.__dataclass_fields__) # has temp_name field @@ -880,8 +880,8 @@ def test_unit_transform_detected(self): def test_no_transform_same_units(self): """Identical units → no transformation.""" hd = self._host_dict() - sv = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count') - arg = _resolve_one_arg(sv, 'run', hd, {}, 'my_scheme', set()) + suite_var = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) self.assertFalse(arg.needs_transform) def test_unknown_unit_mismatch_raises(self): @@ -901,17 +901,17 @@ def test_unknown_unit_mismatch_raises(self): kind = kind_phys ''' hd = build_flat_host_dict(_parse(src), [], {}) - sv = self._scheme_var('v', 'some_value', 'in', 'abc_unit', '()', 'real', 'kind_phys') + suite_var = self._scheme_var('v', 'some_value', 'in', 'abc_unit', '()', 'real', 'kind_phys') with self.assertRaises(CCPPError) as cm: - _resolve_one_arg(sv, 'run', hd, {}, 'bad_scheme', set()) + _resolve_one_arg(suite_var, 'run', hd, {}, 'bad_scheme', set()) self.assertIn('xyz_unit', str(cm.exception)) def test_optional_sets_ptr_name(self): """Optional argument → ptr_name set.""" hd = self._host_dict() - sv = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count', + suite_var = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count', optional=True) - arg = _resolve_one_arg(sv, 'run', hd, {}, 'my_scheme', set()) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) self.assertTrue(arg.is_optional) self.assertTrue(arg.ptr_name) self.assertEqual(arg.transform_case, 2) @@ -989,35 +989,35 @@ def _build_host_and_scheme( from metadata.metadata_table import MetaVar ctx = _ctx() - sv = MetaVar('temp', ctx) - sv.set_attr('standard_name', 'air_temperature', ctx) - sv.set_attr('units', scheme_units, ctx) - sv.set_attr('dimensions', + suite_var = MetaVar('temp', ctx) + suite_var.set_attr('standard_name', 'air_temperature', ctx) + suite_var.set_attr('units', scheme_units, ctx) + suite_var.set_attr('dimensions', '(horizontal_loop_extent, vertical_layer_dimension)', ctx) - sv.set_attr('type', 'real', ctx) - sv.set_attr('kind', 'kind_phys', ctx) - sv.set_attr('intent', intent, ctx) + suite_var.set_attr('type', 'real', ctx) + suite_var.set_attr('kind', 'kind_phys', ctx) + suite_var.set_attr('intent', intent, ctx) if scheme_top_at_one: - sv.set_attr('top_at_one', 'True', ctx) - return hd, sv + suite_var.set_attr('top_at_one', 'True', ctx) + return hd, suite_var def test_no_flip_when_both_false(self): - hd, sv = self._build_host_and_scheme(False, False) - arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + hd, suite_var = self._build_host_and_scheme(False, False) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertFalse(arg.needs_vert_flip) self.assertEqual(arg.transform_case, 1) self.assertEqual(arg.call_expr, 'gt0(lb:ub, 1:nlev)') def test_no_flip_when_both_true(self): - hd, sv = self._build_host_and_scheme(True, True) - arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + hd, suite_var = self._build_host_and_scheme(True, True) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertFalse(arg.needs_vert_flip) self.assertEqual(arg.transform_case, 1) self.assertEqual(arg.call_expr, 'gt0(lb:ub, 1:nlev)') def test_flip_when_host_false_scheme_true(self): - hd, sv = self._build_host_and_scheme(False, True) - arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + hd, suite_var = self._build_host_and_scheme(False, True) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertTrue(arg.needs_vert_flip) self.assertTrue(arg.needs_transform) # Transform pipeline: temp local, transform_case 3, no unit conv. @@ -1033,8 +1033,8 @@ def test_flip_when_host_false_scheme_true(self): self.assertEqual(arg.unit_backward, 'temp_l') def test_flip_when_host_true_scheme_false(self): - hd, sv = self._build_host_and_scheme(True, False) - arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + hd, suite_var = self._build_host_and_scheme(True, False) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertTrue(arg.needs_vert_flip) self.assertEqual(arg.call_expr, 'gt0(lb:ub, nlev:1:-1)') @@ -1042,9 +1042,9 @@ def test_flip_composes_with_unit_conversion(self): """Mismatched top_at_one AND a unit conversion → the unit-forward formula is applied to the flipped call_expr; the temp pattern is a single combined assignment.""" - hd, sv = self._build_host_and_scheme(False, True, + hd, suite_var = self._build_host_and_scheme(False, True, host_units='Pa', scheme_units='hPa') - arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertTrue(arg.needs_vert_flip) self.assertTrue(arg.needs_unit_transform) self.assertEqual(arg.transform_case, 3) @@ -1057,15 +1057,15 @@ def test_flip_composes_with_unit_conversion(self): self.assertEqual(arg.call_expr, 'gt0(lb:ub, nlev:1:-1)') def test_intent_in_only_emits_forward(self): - hd, sv = self._build_host_and_scheme(False, True, intent='in') - arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + hd, suite_var = self._build_host_and_scheme(False, True, intent='in') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertTrue(arg.needs_vert_flip) self.assertEqual(arg.unit_forward, 'gt0(lb:ub, nlev:1:-1)') self.assertEqual(arg.unit_backward, '') def test_intent_out_only_emits_backward(self): - hd, sv = self._build_host_and_scheme(False, True, intent='out') - arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + hd, suite_var = self._build_host_and_scheme(False, True, intent='out') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertTrue(arg.needs_vert_flip) self.assertEqual(arg.unit_forward, '') self.assertEqual(arg.unit_backward, 'temp_l') @@ -1093,15 +1093,15 @@ def test_no_flip_on_scalar(self): from metadata.metadata_table import MetaVar ctx = _ctx() - sv = MetaVar('s', ctx) - sv.set_attr('standard_name', 'some_scalar', ctx) - sv.set_attr('units', '1', ctx) - sv.set_attr('dimensions', '()', ctx) - sv.set_attr('type', 'real', ctx) - sv.set_attr('kind', 'kind_phys', ctx) - sv.set_attr('intent', 'in', ctx) + suite_var = MetaVar('s', ctx) + suite_var.set_attr('standard_name', 'some_scalar', ctx) + suite_var.set_attr('units', '1', ctx) + suite_var.set_attr('dimensions', '()', ctx) + suite_var.set_attr('type', 'real', ctx) + suite_var.set_attr('kind', 'kind_phys', ctx) + suite_var.set_attr('intent', 'in', ctx) # Scheme leaves top_at_one at default (False). - arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertFalse(arg.needs_vert_flip) @@ -1146,8 +1146,8 @@ def _scheme_var_char(self, local, std_name, kind): def test_len_star_compatible_with_len_512(self): """len=* in scheme is always compatible — no transform, no error.""" hd = self._host_with_char('len=512') - sv = self._scheme_var_char('msg', 'my_message', 'len=*') - arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + suite_var = self._scheme_var_char('msg', 'my_message', 'len=*') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertFalse(arg.needs_kind_transform) self.assertEqual(arg.transform_case, 1) self.assertEqual(arg.temp_name, '') @@ -1155,32 +1155,32 @@ def test_len_star_compatible_with_len_512(self): def test_len_match_compatible(self): """Same specific len=N in both host and scheme — no transform.""" hd = self._host_with_char('len=512') - sv = self._scheme_var_char('msg', 'my_message', 'len=512') - arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + suite_var = self._scheme_var_char('msg', 'my_message', 'len=512') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertFalse(arg.needs_kind_transform) def test_len_star_in_host_no_error(self): """len=* in the host is also fine (assumed-length dummy everywhere).""" hd = self._host_with_char('len=*') - sv = self._scheme_var_char('msg', 'my_message', 'len=*') - arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + suite_var = self._scheme_var_char('msg', 'my_message', 'len=*') + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertFalse(arg.needs_kind_transform) def test_mismatched_specific_lengths_raises(self): """Specific len=128 vs len=512 is a metadata error.""" hd = self._host_with_char('len=512') - sv = self._scheme_var_char('msg', 'my_message', 'len=128') + suite_var = self._scheme_var_char('msg', 'my_message', 'len=128') with self.assertRaises(CCPPError) as cm: - _resolve_one_arg(sv, 'run', hd, {}, 'bad_scheme', set()) + _resolve_one_arg(suite_var, 'run', hd, {}, 'bad_scheme', set()) self.assertIn('len=512', str(cm.exception)) self.assertIn('len=128', str(cm.exception)) def test_len_star_host_specific_scheme_raises(self): """len=* in host but specific len=256 in scheme — error.""" hd = self._host_with_char('len=*') - sv = self._scheme_var_char('msg', 'my_message', 'len=256') + suite_var = self._scheme_var_char('msg', 'my_message', 'len=256') with self.assertRaises(CCPPError) as cm: - _resolve_one_arg(sv, 'run', hd, {}, 'bad_scheme', set()) + _resolve_one_arg(suite_var, 'run', hd, {}, 'bad_scheme', set()) self.assertIn('len=256', str(cm.exception)) @@ -1197,22 +1197,22 @@ def _resolve(self): return resolve_suite(suite, store, hd), hd def test_groups_present(self): - sr, _ = self._resolve() - self.assertEqual(sr.suite_name, 'test_simple') - self.assertEqual(len(sr.groups), 1) - self.assertEqual(sr.groups[0].group_name, 'physics') + suite_resolution, _ = self._resolve() + self.assertEqual(suite_resolution.suite_name, 'test_simple') + self.assertEqual(len(suite_resolution.groups), 1) + self.assertEqual(suite_resolution.groups[0].group_name, 'physics') def test_run_phase_calls(self): - sr, _ = self._resolve() - rg = sr.groups[0] - self.assertIn('run', rg.phase_calls) - calls = rg.phase_calls['run'] + suite_resolution, _ = self._resolve() + resolved_group = suite_resolution.groups[0] + self.assertIn('run', resolved_group.phase_calls) + calls = resolved_group.phase_calls['run'] self.assertEqual(len(calls), 1) self.assertEqual(calls[0].scheme_name, 'temp_calc_adjust') def test_run_phase_args(self): - sr, _ = self._resolve() - calls = sr.groups[0].phase_calls['run'] + suite_resolution, _ = self._resolve() + calls = suite_resolution.groups[0].phase_calls['run'] args = {a.scheme_local_name: a for a in calls[0].args} # im = horizontal_dimension: scalar arg is synthesised from the loop # bounds so the scheme sees the per-call chunk extent, not host ncols. @@ -1226,41 +1226,41 @@ def test_run_phase_args(self): self.assertIn('errflg', args) def test_init_phase_calls(self): - sr, _ = self._resolve() - rg = sr.groups[0] - self.assertIn('init', rg.phase_calls) - calls = rg.phase_calls['init'] + suite_resolution, _ = self._resolve() + resolved_group = suite_resolution.groups[0] + self.assertIn('init', resolved_group.phase_calls) + calls = resolved_group.phase_calls['init'] self.assertEqual(calls[0].scheme_name, 'temp_calc_adjust') def test_init_phase_horizontal_subscript(self): """In init phase, scalar horizontal_dimension does not produce an lb:ub slice.""" - sr, _ = self._resolve() - rg = sr.groups[0] + suite_resolution, _ = self._resolve() + resolved_group = suite_resolution.groups[0] # temp_calc_adjust_init doesn't have temp, but the general rule should hold # for any 2D variable in a non-run phase: no lb:ub slice in init args. # (Scalar horizontal_dimension args are synthesised as (ub - lb + 1), # which collapses to ncols in non-run phases but never contains # the substring 'lb:ub'.) - calls = rg.phase_calls.get('init', []) - for rc in calls: - for arg in rc.args: + calls = resolved_group.phase_calls.get('init', []) + for resolved_call in calls: + for arg in resolved_call.args: self.assertNotIn('lb:ub', arg.call_expr) def test_no_suite_vars(self): """All variables in temp_calc_adjust are provided by the host.""" - sr, _ = self._resolve() - self.assertEqual(sr.suite_vars, {}) + suite_resolution, _ = self._resolve() + self.assertEqual(suite_resolution.suite_vars, {}) def test_used_modules(self): - sr, _ = self._resolve() - calls = sr.groups[0].phase_calls['run'] + suite_resolution, _ = self._resolve() + calls = suite_resolution.groups[0].phase_calls['run'] mods = calls[0].used_modules # host_phys should appear (air_temperature, horizontal_dimension, etc.) self.assertIn('host_phys', mods) def test_control_args_no_module(self): - sr, _ = self._resolve() - calls = sr.groups[0].phase_calls['run'] + suite_resolution, _ = self._resolve() + calls = suite_resolution.groups[0].phase_calls['run'] ctrl = [a for a in calls[0].args if a.source == 'control'] for c in ctrl: self.assertIsNone(c.module_name) @@ -1326,8 +1326,8 @@ def _store(self): _parse(self._LOOP_SCHEME_SRC, 'loop_scheme.meta') ) - def _args_by_name(self, sr): - calls = list(iter_phase_calls(sr.groups[0].phase_calls['run'])) + def _args_by_name(self, suite_resolution): + calls = list(iter_phase_calls(suite_resolution.groups[0].phase_calls['run'])) return {a.scheme_local_name: a for a in calls[0].args} def test_counter_inside_subcycle_resolves_to_loop_local(self): @@ -1341,9 +1341,9 @@ def test_counter_inside_subcycle_resolves_to_loop_local(self): " \n" "\n" ) - sr = resolve_suite(self._build_suite_from_xml(xml), + suite_resolution = resolve_suite(self._build_suite_from_xml(xml), self._store(), _load_full_host_dict()) - args = self._args_by_name(sr) + args = self._args_by_name(suite_resolution) self.assertEqual(args['iter'].standard_name, 'ccpp_loop_counter') self.assertEqual(args['iter'].call_expr, 'ccpp_loop_counter') self.assertEqual(args['iter'].source, 'control') @@ -1359,9 +1359,9 @@ def test_extent_integer_literal(self): " \n" "\n" ) - sr = resolve_suite(self._build_suite_from_xml(xml), + suite_resolution = resolve_suite(self._build_suite_from_xml(xml), self._store(), _load_full_host_dict()) - args = self._args_by_name(sr) + args = self._args_by_name(suite_resolution) # ``loop=3`` is a literal — extent resolves to the same literal. self.assertEqual(args['niter'].call_expr, '3') @@ -1385,9 +1385,9 @@ def test_extent_std_name_resolves_to_host_local(self): " \n" "\n" ) - sr = resolve_suite(self._build_suite_from_xml(xml), + suite_resolution = resolve_suite(self._build_suite_from_xml(xml), self._store(), hd) - args = self._args_by_name(sr) + args = self._args_by_name(suite_resolution) self.assertEqual(args['niter'].call_expr, 'n_sub') def test_outside_subcycle_raises_clear_error(self): @@ -1567,11 +1567,11 @@ def test_init_call_attached(self): ' \n' '\n' ) - sr = self._resolve(suite_xml, self._SCHEME_META) - self.assertIsNotNone(sr.suite_init_call) - self.assertEqual(sr.suite_init_call.scheme_name, 'init_final_test') - self.assertEqual(sr.suite_init_call.phase, 'init') - self.assertIsNone(sr.suite_final_call) + suite_resolution = self._resolve(suite_xml, self._SCHEME_META) + self.assertIsNotNone(suite_resolution.suite_init_call) + self.assertEqual(suite_resolution.suite_init_call.scheme_name, 'init_final_test') + self.assertEqual(suite_resolution.suite_init_call.phase, 'init') + self.assertIsNone(suite_resolution.suite_final_call) def test_final_call_attached(self): suite_xml = ( @@ -1581,11 +1581,11 @@ def test_final_call_attached(self): ' init_final_test\n' '\n' ) - sr = self._resolve(suite_xml, self._SCHEME_META) - self.assertIsNone(sr.suite_init_call) - self.assertIsNotNone(sr.suite_final_call) - self.assertEqual(sr.suite_final_call.scheme_name, 'init_final_test') - self.assertEqual(sr.suite_final_call.phase, 'final') + suite_resolution = self._resolve(suite_xml, self._SCHEME_META) + self.assertIsNone(suite_resolution.suite_init_call) + self.assertIsNotNone(suite_resolution.suite_final_call) + self.assertEqual(suite_resolution.suite_final_call.scheme_name, 'init_final_test') + self.assertEqual(suite_resolution.suite_final_call.phase, 'final') def test_both_attached(self): suite_xml = ( @@ -1596,9 +1596,9 @@ def test_both_attached(self): ' init_final_test\n' '\n' ) - sr = self._resolve(suite_xml, self._SCHEME_META) - self.assertIsNotNone(sr.suite_init_call) - self.assertIsNotNone(sr.suite_final_call) + suite_resolution = self._resolve(suite_xml, self._SCHEME_META) + self.assertIsNotNone(suite_resolution.suite_init_call) + self.assertIsNotNone(suite_resolution.suite_final_call) def test_init_scheme_without_init_phase_raises(self): """If the named scheme has no ``init`` phase in its metadata, @@ -1679,8 +1679,8 @@ def test_resolve_does_not_raise(self): self.assertIsNotNone(self._resolve()) def test_run_phase_preserves_both_calls(self): - sr = self._resolve() - run_calls = list(iter_phase_calls(sr.groups[0].phase_calls['run'])) + suite_resolution = self._resolve() + run_calls = list(iter_phase_calls(suite_resolution.groups[0].phase_calls['run'])) self.assertEqual(len(run_calls), 2) self.assertEqual( [c.scheme_name for c in run_calls], @@ -1688,14 +1688,14 @@ def test_run_phase_preserves_both_calls(self): ) def test_init_phase_dedupes(self): - sr = self._resolve() - init_calls = list(iter_phase_calls(sr.groups[0].phase_calls['init'])) + suite_resolution = self._resolve() + init_calls = list(iter_phase_calls(suite_resolution.groups[0].phase_calls['init'])) self.assertEqual(len(init_calls), 1) self.assertEqual(init_calls[0].scheme_name, 'temp_calc_adjust') def test_final_phase_dedupes(self): - sr = self._resolve() - final_calls = list(iter_phase_calls(sr.groups[0].phase_calls['final'])) + suite_resolution = self._resolve() + final_calls = list(iter_phase_calls(suite_resolution.groups[0].phase_calls['final'])) self.assertEqual(len(final_calls), 1) @@ -1779,10 +1779,10 @@ def _fake_rg(self, args): # ``iter_phase_calls`` does ``isinstance(item, ResolvedCall)`` so a # MagicMock won't pass; build a real ResolvedCall (the only field # ``_collect_kinds_used`` reads is ``args``). - rc = ResolvedCall(scheme_name='s', phase='run', args=args) - rg = MagicMock() - rg.phase_calls = {'run': [rc]} - return rg + resolved_call = ResolvedCall(scheme_name='s', phase='run', args=args) + resolved_group = MagicMock() + resolved_group.phase_calls = {'run': [resolved_call]} + return resolved_group def test_keeps_kind_symbols(self): args = [self._fake_arg(kind_scheme='kind_phys'), @@ -1927,9 +1927,9 @@ def _resolve_and_generate(self): hd = _load_full_host_dict() store = _load_scheme_store() suite = _parse_suite('suite_test_simple.xml') - sr = resolve_suite(suite, store, hd) - rg = sr.groups[0] - lines = _generate_group_cap('test_simple', 'physics', rg, hd) + suite_resolution = resolve_suite(suite, store, hd) + resolved_group = suite_resolution.groups[0] + lines = _generate_group_cap('test_simple', 'physics', resolved_group, hd) return lines def test_module_header(self): @@ -1998,10 +1998,10 @@ def test_write_group_cap(self): hd = _load_full_host_dict() store = _load_scheme_store() suite = _parse_suite('suite_test_simple.xml') - sr = resolve_suite(suite, store, hd) - rg = sr.groups[0] + suite_resolution = resolve_suite(suite, store, hd) + resolved_group = suite_resolution.groups[0] with tempfile.TemporaryDirectory() as tmpdir: - path = write_group_cap('test_simple', 'physics', rg, hd, tmpdir) + path = write_group_cap('test_simple', 'physics', resolved_group, hd, tmpdir) self.assertTrue(os.path.isfile(path)) self.assertEqual(os.path.basename(path), 'ccpp_test_simple_physics_cap.F90') with open(path) as fh: @@ -2023,34 +2023,34 @@ def _resolve_subcycle(self): return resolve_suite(suite, store, hd) def test_run_phase_has_subcycle(self): - sr = self._resolve_subcycle() - rg = sr.groups[0] - run_items = rg.phase_calls['run'] + suite_resolution = self._resolve_subcycle() + resolved_group = suite_resolution.groups[0] + run_items = resolved_group.phase_calls['run'] self.assertEqual(len(run_items), 1) self.assertIsInstance(run_items[0], ResolvedSubcycle) def test_subcycle_loop_count(self): - sr = self._resolve_subcycle() - sub = sr.groups[0].phase_calls['run'][0] + suite_resolution = self._resolve_subcycle() + sub = suite_resolution.groups[0].phase_calls['run'][0] self.assertEqual(sub.loop, '3') def test_subcycle_contains_scheme(self): - sr = self._resolve_subcycle() - sub = sr.groups[0].phase_calls['run'][0] + suite_resolution = self._resolve_subcycle() + sub = suite_resolution.groups[0].phase_calls['run'][0] self.assertEqual(len(sub.calls), 1) self.assertEqual(sub.calls[0].scheme_name, 'temp_calc_adjust') def test_init_phase_is_flat(self): """Init phase flattens subcycles — no ResolvedSubcycle in init.""" - sr = self._resolve_subcycle() - rg = sr.groups[0] - for item in rg.phase_calls.get('init', []): + suite_resolution = self._resolve_subcycle() + resolved_group = suite_resolution.groups[0] + for item in resolved_group.phase_calls.get('init', []): self.assertNotIsInstance(item, ResolvedSubcycle) def test_iter_phase_calls_flattens(self): - sr = self._resolve_subcycle() - rg = sr.groups[0] - all_calls = list(iter_phase_calls(rg.phase_calls['run'])) + suite_resolution = self._resolve_subcycle() + resolved_group = suite_resolution.groups[0] + all_calls = list(iter_phase_calls(resolved_group.phase_calls['run'])) self.assertEqual(len(all_calls), 1) self.assertEqual(all_calls[0].scheme_name, 'temp_calc_adjust') @@ -2091,8 +2091,8 @@ def test_two_deep_nesting_preserved(self): ' \n' '\n' ) - sr = self._resolve_nested(suite_xml) - run = sr.groups[0].phase_calls['run'] + suite_resolution = self._resolve_nested(suite_xml) + run = suite_resolution.groups[0].phase_calls['run'] # Outer subcycle. self.assertEqual(len(run), 1) outer = run[0] @@ -2123,8 +2123,8 @@ def test_three_deep_nesting_preserved(self): ' \n' '\n' ) - sr = self._resolve_nested(suite_xml) - run = sr.groups[0].phase_calls['run'] + suite_resolution = self._resolve_nested(suite_xml) + run = suite_resolution.groups[0].phase_calls['run'] outer = run[0] mid = outer.calls[0] inner = mid.calls[0] @@ -2149,8 +2149,8 @@ def test_iter_phase_calls_recurses_through_nesting(self): ' \n' '\n' ) - sr = self._resolve_nested(suite_xml) - run = sr.groups[0].phase_calls['run'] + suite_resolution = self._resolve_nested(suite_xml) + run = suite_resolution.groups[0].phase_calls['run'] calls = list(iter_phase_calls(run)) # One scheme call, reachable through two subcycle wrappers. self.assertEqual(len(calls), 1) @@ -2320,9 +2320,9 @@ def setUp(self): hd = _load_full_host_dict() store = _load_scheme_store() suite = _parse_suite('suite_test_subcycle.xml') - sr = resolve_suite(suite, store, hd) - rg = sr.groups[0] - self.lines = _generate_group_cap('test_subcycle', 'physics', rg, hd) + suite_resolution = resolve_suite(suite, store, hd) + resolved_group = suite_resolution.groups[0] + self.lines = _generate_group_cap('test_subcycle', 'physics', resolved_group, hd) self.text = '\n'.join(self.lines) def test_do_loop_present(self): @@ -2355,9 +2355,9 @@ def setUp(self): hd = _load_full_host_dict() store = _load_scheme_store() suite = _parse_suite('suite_test_simple.xml') - sr = resolve_suite(suite, store, hd) - rg = sr.groups[0] - self.lines = _generate_group_cap('test_simple', 'physics', rg, hd) + suite_resolution = resolve_suite(suite, store, hd) + resolved_group = suite_resolution.groups[0] + self.lines = _generate_group_cap('test_simple', 'physics', resolved_group, hd) self.text = '\n'.join(self.lines) def test_state_constants_declared(self): @@ -2448,9 +2448,9 @@ def setUp(self): hd = _load_full_host_dict() store = _load_scheme_store() suite = _parse_suite('suite_test_simple.xml') - sr = resolve_suite(suite, store, hd) + suite_resolution = resolve_suite(suite, store, hd) # Pass host_dict so number_of_instances flows through. - lines = _generate_suite_cap('test_simple', sr, store, hd) + lines = _generate_suite_cap('test_simple', suite_resolution, store, hd) self.text = '\n'.join(lines) def test_init_calls_state_alloc_with_ninstances(self): @@ -2486,8 +2486,8 @@ def setUp(self): hd = {k: v for k, v in hd_full.items() if k != 'number_of_instances'} store = _load_scheme_store() suite = _parse_suite('suite_test_simple.xml') - sr = resolve_suite(suite, store, hd) - lines = _generate_suite_cap('test_simple', sr, store, hd) + suite_resolution = resolve_suite(suite, store, hd) + lines = _generate_suite_cap('test_simple', suite_resolution, store, hd) self.text = '\n'.join(lines) def test_init_calls_state_alloc_with_literal_1(self): @@ -2520,41 +2520,41 @@ def setUp(self): self.hd = _load_full_host_dict() self.store = _load_register_dim_scheme_store() self.suite = _parse_suite('suite_register_dim.xml') - self.sr = resolve_suite(self.suite, self.store, self.hd) + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) def test_dim_inter_promoted_to_suite_var(self): # The register-phase intent=out arg becomes a suite-owned variable. self.assertIn( - 'dimension_for_interstitial_variable', self.sr.suite_vars, + 'dimension_for_interstitial_variable', self.suite_resolution.suite_vars, ) - sv = self.sr.suite_vars['dimension_for_interstitial_variable'] - self.assertEqual(sv.type_, 'integer') - self.assertEqual(sv.dimensions, []) - self.assertEqual(sv.source_phase, 'register') + suite_var = self.suite_resolution.suite_vars['dimension_for_interstitial_variable'] + self.assertEqual(suite_var.type_, 'integer') + self.assertEqual(suite_var.dimensions, []) + self.assertEqual(suite_var.source_phase, 'register') def test_interstitial_var_promoted_to_suite_var(self): # The run-phase intent=out array also becomes a suite var, dimensioned # by the register-set scalar. self.assertIn( - 'output_only_interstitial_variable', self.sr.suite_vars, + 'output_only_interstitial_variable', self.suite_resolution.suite_vars, ) - sv = self.sr.suite_vars['output_only_interstitial_variable'] - self.assertEqual(sv.dimensions, ['dimension_for_interstitial_variable']) + suite_var = self.suite_resolution.suite_vars['output_only_interstitial_variable'] + self.assertEqual(suite_var.dimensions, ['dimension_for_interstitial_variable']) def test_register_phase_call_resolved(self): # Group's register phase has a ResolvedCall for the producer scheme. - rg = self.sr.groups[0] - register_calls = list(iter_phase_calls(rg.phase_calls.get('register', []))) + resolved_group = self.suite_resolution.groups[0] + register_calls = list(iter_phase_calls(resolved_group.phase_calls.get('register', []))) self.assertEqual(len(register_calls), 1) self.assertEqual(register_calls[0].scheme_name, 'register_dim_producer') def test_run_phase_dim_resolves_via_suite_var(self): # The run-phase consumer call's interstitial_var arg's call_expr # must reference ccpp_suite_data(...)%dim_inter as the upper bound. - rg = self.sr.groups[0] - run_calls = list(iter_phase_calls(rg.phase_calls.get('run', []))) - consumer = next(rc for rc in run_calls - if rc.scheme_name == 'register_dim_consumer') + resolved_group = self.suite_resolution.groups[0] + run_calls = list(iter_phase_calls(resolved_group.phase_calls.get('run', []))) + consumer = next(resolved_call for resolved_call in run_calls + if resolved_call.scheme_name == 'register_dim_consumer') inter_arg = next(a for a in consumer.args if a.standard_name == 'output_only_interstitial_variable') self.assertIn('1:ccpp_suite_data', inter_arg.subscript) @@ -2569,9 +2569,9 @@ def setUp(self): self.hd = _load_full_host_dict() self.store = _load_register_dim_scheme_store() self.suite = _parse_suite('suite_register_dim.xml') - self.sr = resolve_suite(self.suite, self.store, self.hd) + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) self.text = '\n'.join( - _generate_suite_cap('reg_dim', self.sr, self.store, self.hd) + _generate_suite_cap('reg_dim', self.suite_resolution, self.store, self.hd) ) def test_register_subroutine_emits_scheme_call(self): @@ -2634,23 +2634,23 @@ def setUp(self): self.hd = _load_constituent_host_dict() self.store = _load_constituent_scheme_store() self.suite = _parse_suite('suite_register_constituents.xml') - self.sr = resolve_suite(self.suite, self.store, self.hd) + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) def test_constituent_register_calls_recorded(self): self.assertEqual( - self.sr.constituent_register_calls, + self.suite_resolution.constituent_register_calls, [('register_constituents', 'dyn_const')], ) def test_constituent_arg_not_promoted_to_suite_var(self): # The constituent array is per-scheme transient — never a SuiteVar. self.assertNotIn( - 'dynamic_constituents_for_register_test', self.sr.suite_vars, + 'dynamic_constituents_for_register_test', self.suite_resolution.suite_vars, ) def test_constituent_arg_marked(self): - rg = self.sr.groups[0] - register_call = list(iter_phase_calls(rg.phase_calls['register']))[0] + resolved_group = self.suite_resolution.groups[0] + register_call = list(iter_phase_calls(resolved_group.phase_calls['register']))[0] const_arg = next(a for a in register_call.args if a.is_constituent_arg) self.assertEqual(const_arg.scheme_local_name, 'dyn_const') self.assertEqual(const_arg.call_expr, 'scheme_consts') @@ -2665,11 +2665,11 @@ def test_missing_host_object_does_not_raise(self): hd = _load_full_host_dict() store = _load_constituent_scheme_store() suite = _parse_suite('suite_register_constituents.xml') - sr = resolve_suite(suite, store, hd) + suite_resolution = resolve_suite(suite, store, hd) # The register-phase scheme is still recorded for the suite cap to # populate the per-suite dynamic-constituent buffer. self.assertEqual( - sr.constituent_register_calls, + suite_resolution.constituent_register_calls, [('register_constituents', 'dyn_const')], ) @@ -2687,9 +2687,9 @@ def setUp(self): self.hd = _load_constituent_host_dict() self.store = _load_constituent_scheme_store() self.suite = _parse_suite('suite_register_constituents.xml') - self.sr = resolve_suite(self.suite, self.store, self.hd) + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) self.text = '\n'.join( - _generate_suite_cap('reg_consts', self.sr, self.store, self.hd) + _generate_suite_cap('reg_consts', self.suite_resolution, self.store, self.hd) ) def test_uses_constituent_prop_type(self): @@ -2789,18 +2789,18 @@ def setUp(self): self.hd = _load_constituent_host_dict() self.store = _load_constituent_consumer_store() self.suite = _parse_suite('suite_consume_constituent.xml') - self.sr = resolve_suite(self.suite, self.store, self.hd) - run_calls = list(iter_phase_calls(self.sr.groups[0].phase_calls['run'])) + self.suite_resolution = resolve_suite(self.suite, self.store, self.hd) + run_calls = list(iter_phase_calls(self.suite_resolution.groups[0].phase_calls['run'])) self.run_args = {a.scheme_local_name: a for a in run_calls[0].args} def test_uses_constituents_flag_set(self): - self.assertTrue(self.sr.uses_constituents) + self.assertTrue(self.suite_resolution.uses_constituents) def test_constituent_index_names_enumerated(self): # Both the base read and the tendency write reference the same # base std name. self.assertEqual( - self.sr.constituent_index_names, + self.suite_resolution.constituent_index_names, ['cloud_liquid_water_mixing_ratio'], ) @@ -2905,14 +2905,14 @@ def test_ccpp_constituents_dim_lands_on_dedicated_field(self): # resolver routes through capgen-ng's auto-provisioning path. from generator.suite_resolver import _resolve_constituent_arg hd = _load_full_host_dict() - sv = self._scheme_var( + suite_var = self._scheme_var( 'consts', 'ccpp_constituents', '(horizontal_dimension, vertical_layer_dimension, ' 'number_of_ccpp_constituents)', intent='in', ) arg = _resolve_constituent_arg( - sv, 'run', hd, {}, 'consts_user', 'mysuite', + suite_var, 'run', hd, {}, 'consts_user', 'mysuite', ) self.assertIsNotNone(arg) self.assertEqual(arg.source, 'constituent') @@ -2932,14 +2932,14 @@ def test_no_const_dim_when_arg_does_not_reference_it(self): # intent=in). from generator.suite_resolver import _resolve_constituent_arg hd = _load_full_host_dict() - sv = self._scheme_var( + suite_var = self._scheme_var( 'cldliq', 'cloud_liquid_water_mixing_ratio', '(horizontal_dimension, vertical_layer_dimension)', intent='in', ) - sv.set_attr('advected', 'True', _ctx()) + suite_var.set_attr('advected', 'True', _ctx()) arg = _resolve_constituent_arg( - sv, 'run', hd, {}, 'cldliq_user', 'mysuite', + suite_var, 'run', hd, {}, 'cldliq_user', 'mysuite', ) self.assertIsNotNone(arg) self.assertEqual(arg.used_const_dim_std_names, set()) @@ -3110,12 +3110,12 @@ def _scheme_var(self, local, std_name, intent='in'): def test_host_index_of_resolves_to_host_local_name(self): hd = build_flat_host_dict(_parse(self._HOST_SRC), [], []) - sv = self._scheme_var( + suite_var = self._scheme_var( 'ntcw', 'index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array', intent='in', ) - arg = _resolve_one_arg(sv, 'run', hd, {}, 'gfs_mp_generic_pre', set()) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'gfs_mp_generic_pre', set()) # Host metadata wins: source=host, short local name, NO leakage of # the long std_name into ccpp_host_constituents. self.assertEqual(arg.source, 'host') @@ -3128,11 +3128,11 @@ def test_unclaimed_index_of_still_routes_to_constituents(self): ``index_of_`` names the host does NOT declare — required for capgen-ng-owned constituent flows (cf. the advection e2e test).""" hd = build_flat_host_dict(_parse(self._HOST_SRC), [], []) - sv = self._scheme_var( + suite_var = self._scheme_var( 'idx_other', 'index_of_some_other_constituent_not_in_host', intent='in', ) - arg = _resolve_one_arg(sv, 'run', hd, {}, 'some_scheme', set()) + arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'some_scheme', set()) self.assertEqual(arg.source, 'constituent') self.assertEqual(arg.call_expr, 'index_of_some_other_constituent_not_in_host') @@ -3343,21 +3343,21 @@ def _mock_arg(used_dim_std_names): def _mock_rg(self, arg): from unittest.mock import MagicMock from generator.suite_resolver import ResolvedCall - rg = MagicMock() - rg.phase_calls = { + resolved_group = MagicMock() + resolved_group.phase_calls = { 'run': [ResolvedCall( scheme_name='dummy', phase='run', args=[arg], scheme_module='dummy_mod', )], } - return rg + return resolved_group def test_collect_dim_uses_walks_to_root_for_ddt_dim(self): from generator.suite_resolver import _collect_dim_uses hd = self._host_dict() arg = self._mock_arg({'vertical_layer_dimension'}) - rg = self._mock_rg(arg) - dim_uses = _collect_dim_uses(rg, hd, suite_vars={}) + resolved_group = self._mock_rg(arg) + dim_uses = _collect_dim_uses(resolved_group, hd, suite_vars={}) # USE clause must pull the ROOT (``physics``), not the leaf # (``levs``, which is not a module symbol). self.assertIn('scm_host_mod', dim_uses) @@ -3370,8 +3370,8 @@ def test_collect_dim_uses_plain_var_unchanged(self): from generator.suite_resolver import _collect_dim_uses hd = self._host_dict() arg = self._mock_arg({'horizontal_dimension'}) - rg = self._mock_rg(arg) - dim_uses = _collect_dim_uses(rg, hd, suite_vars={}) + resolved_group = self._mock_rg(arg) + dim_uses = _collect_dim_uses(resolved_group, hd, suite_vars={}) self.assertEqual(dim_uses.get('scm_host_mod'), {'ncols'}) def test_collect_dim_uses_two_ddt_dims_dedupe_root(self): @@ -3382,8 +3382,8 @@ def test_collect_dim_uses_two_ddt_dims_dedupe_root(self): hd = self._host_dict() arg = self._mock_arg({'vertical_layer_dimension', 'horizontal_dimension_total'}) - rg = self._mock_rg(arg) - dim_uses = _collect_dim_uses(rg, hd, suite_vars={}) + resolved_group = self._mock_rg(arg) + dim_uses = _collect_dim_uses(resolved_group, hd, suite_vars={}) self.assertEqual(dim_uses.get('scm_host_mod'), {'physics'}) # ---- End-to-end: resolve_suite + group cap output ------------------ @@ -3485,10 +3485,10 @@ def test_end_to_end_ddt_dim_in_group_cap(self): logger = logging.getLogger('ddt_dim_e2e') suite = parse_suite_xml(xml_path, tmpdir, logger, skip_validation=True) - sr = resolve_suite(suite, store, hd) + suite_resolution = resolve_suite(suite, store, hd) # Inspect the resolved scheme arg's subscript. - run_call = list(iter_phase_calls(sr.groups[0].phase_calls['run']))[0] + run_call = list(iter_phase_calls(suite_resolution.groups[0].phase_calls['run']))[0] temp_arg = [a for a in run_call.args if a.scheme_local_name == 'temp'][0] self.assertIn('physics%Model%levs', temp_arg.subscript) @@ -3498,9 +3498,9 @@ def test_end_to_end_ddt_dim_in_group_cap(self): # Now emit the group cap and inspect the USE clause. from generator.group_cap import _generate_group_cap group_lines = _generate_group_cap( - suite_name=sr.suite_name, - group_name=sr.groups[0].group_name, - rg=sr.groups[0], host_dict=hd, + suite_name=suite_resolution.suite_name, + group_name=suite_resolution.groups[0].group_name, + resolved_group=suite_resolution.groups[0], host_dict=hd, ) group_text = '\n'.join(group_lines) # Pull the host-module USE line. Must import ``physics``, @@ -3529,9 +3529,9 @@ def test_end_to_end_ddt_dim_in_group_cap(self): ######################################################################## def load_tests(loader, tests, ignore): - import generator.suite_resolver as sr + import generator.suite_resolver as suite_resolution import generator.group_cap as gc - tests.addTests(doctest.DocTestSuite(sr)) + tests.addTests(doctest.DocTestSuite(suite_resolution)) tests.addTests(doctest.DocTestSuite(gc)) return tests diff --git a/unit-tests/test_suite_xml.py b/unit-tests/test_suite_xml.py index a20660bf..8fe3647b 100644 --- a/unit-tests/test_suite_xml.py +++ b/unit-tests/test_suite_xml.py @@ -151,25 +151,25 @@ class TestSuiteSubcycle(unittest.TestCase): """Tests for :class:`SuiteSubcycle`.""" def test_literal_integer_loop(self): - sc = SuiteSubcycle(loop='2', items=[]) - self.assertEqual(sc.loop, '2') - self.assertTrue(sc.is_literal_count) + subcycle = SuiteSubcycle(loop='2', items=[]) + self.assertEqual(subcycle.loop, '2') + self.assertTrue(subcycle.is_literal_count) def test_stdname_loop(self): - sc = SuiteSubcycle(loop='num_subcycles_for_ag', items=[]) - self.assertFalse(sc.is_literal_count) + subcycle = SuiteSubcycle(loop='num_subcycles_for_ag', items=[]) + self.assertFalse(subcycle.is_literal_count) def test_none_loop_is_literal(self): - sc = SuiteSubcycle(loop=None, items=[]) - self.assertIsNone(sc.loop) - self.assertTrue(sc.is_literal_count) + subcycle = SuiteSubcycle(loop=None, items=[]) + self.assertIsNone(subcycle.loop) + self.assertTrue(subcycle.is_literal_count) def test_scheme_names_from_items(self): - sc = SuiteSubcycle(loop='2', items=[ + subcycle = SuiteSubcycle(loop='2', items=[ SuiteScheme('scheme_a'), SuiteScheme('scheme_b'), ]) - self.assertEqual(sc.scheme_names(), ['scheme_a', 'scheme_b']) + self.assertEqual(subcycle.scheme_names(), ['scheme_a', 'scheme_b']) def test_nested_subcycle_scheme_names(self): inner = SuiteSubcycle(loop='3', items=[SuiteScheme('inner_sch')]) @@ -181,13 +181,13 @@ class TestSuiteSubcol(unittest.TestCase): """Tests for :class:`SuiteSubcol`.""" def test_creation(self): - sc = SuiteSubcol('gen_routine', 'avg_routine', [SuiteScheme('sch')]) - self.assertEqual(sc.gen_routine, 'gen_routine') - self.assertEqual(sc.avg_routine, 'avg_routine') + subcycle = SuiteSubcol('gen_routine', 'avg_routine', [SuiteScheme('sch')]) + self.assertEqual(subcycle.gen_routine, 'gen_routine') + self.assertEqual(subcycle.avg_routine, 'avg_routine') def test_scheme_names(self): - sc = SuiteSubcol('g', 'a', [SuiteScheme('s1'), SuiteScheme('s2')]) - self.assertEqual(sc.scheme_names(), ['s1', 's2']) + subcycle = SuiteSubcol('g', 'a', [SuiteScheme('s1'), SuiteScheme('s2')]) + self.assertEqual(subcycle.scheme_names(), ['s1', 's2']) class TestSuiteGroup(unittest.TestCase): @@ -274,10 +274,10 @@ def test_subcycle_with_loop(self): el = ET.fromstring(xml) items = _parse_group_items(el) self.assertEqual(len(items), 1) - sc = items[0] - self.assertIsInstance(sc, SuiteSubcycle) - self.assertEqual(sc.loop, '3') - self.assertEqual(len(sc.items), 1) + subcycle = items[0] + self.assertIsInstance(subcycle, SuiteSubcycle) + self.assertEqual(subcycle.loop, '3') + self.assertEqual(len(subcycle.items), 1) def test_subcol(self): xml = ('' @@ -431,10 +431,10 @@ def test_subcycle_in_group(self): ''' suite = self._parse(xml) grp = suite.groups[0] - sc = grp.items[0] - self.assertIsInstance(sc, SuiteSubcycle) - self.assertEqual(sc.loop, 'num_subcycles_for_scheme6') - self.assertFalse(sc.is_literal_count) + subcycle = grp.items[0] + self.assertIsInstance(subcycle, SuiteSubcycle) + self.assertEqual(subcycle.loop, 'num_subcycles_for_scheme6') + self.assertFalse(subcycle.is_literal_count) ######################################################################## @@ -522,9 +522,9 @@ def test_v2_suite_02_subcycle_preserved(self): suite = self._parse('suite_good_v2_test02.xml') grp = suite.get_group('main_group') # First item is the subcycle from the original suite - sc = grp.items[0] - self.assertIsInstance(sc, SuiteSubcycle) - self.assertEqual(sc.loop, 'num_subcycles_for_scheme6') + subcycle = grp.items[0] + self.assertIsInstance(subcycle, SuiteSubcycle) + self.assertEqual(subcycle.loop, 'num_subcycles_for_scheme6') def test_v2_suite_03_expand_multiple_nested_suites(self): """Expand two nested suites at group level + full suite at suite level.""" diff --git a/unit-tests/test_validator.py b/unit-tests/test_validator.py index ce1e0cb5..465f5525 100644 --- a/unit-tests/test_validator.py +++ b/unit-tests/test_validator.py @@ -161,6 +161,75 @@ def test_rrtmg_style_signature_round_trip(self): .format(result[0]), ) + def test_sfc_ocean_style_decorated_trailing_ampersand(self): + """Fixed-form continuation where the trailing ``&`` is followed by + a stray ``,`` and an inline comment. In strict F77, columns + past 72 are ignored, so ``&, ! --- inputs`` past col-71 is + invisible to the compiler. The parser must not glue the ``,`` + into the joined arg list — otherwise a phantom ``&`` token + appears between args. Triggers the decoration-repair branch + (next line's column-6 ``&`` confirms continuation).""" + src_lines = [ + ' subroutine sfc_ocean_run &\n', + ' & ( im, hvap, cp, &\n', + ' & wind, &, ! --- inputs\n', + ' & errmsg, errflg )\n', + ] + result = _join_continuation(src_lines, filename='sfc_ocean.F') + self.assertEqual(len(result), 1) + # Phantom ``&`` must not appear in the joined logical line. + self.assertNotIn('&', result[0]) + # All real args must still be present. + for tok in ('im', 'hvap', 'cp', 'wind', 'errmsg', 'errflg'): + self.assertIn(tok, result[0]) + # The stray comma from the decoration must NOT survive past + # ``wind`` (no double-comma). + self.assertNotIn(',,', result[0].replace(' ', '')) + + def test_decoration_repair_preserves_real_tokens_past_amp(self): + """When tokens past ``&`` look like real identifiers, the line + is returned untouched so the parser surfaces a real error + instead of silently dropping code.""" + # A pathological line: ``&`` mid-line with an identifier after. + # Next line is col-6 ``&`` so we enter the look-ahead branch. + src_lines = [ + ' foo = a & extra_token\n', + ' & + b\n', + ] + result = _join_continuation(src_lines, filename='/tmp/path.f') + # The line is left untouched (no repair); ``extra_token`` stays. + self.assertEqual(len(result), 1) + self.assertIn('extra_token', result[0]) + + def test_decoration_repair_emits_warning(self): + """The decoration-repair branch must emit a single + ``logger.warning`` naming the file:line so users see that their + source has decoration past the statement end.""" + import logging as _logging + src_lines = [ + ' subroutine foo( &\n', + ' & a, b, &, ! decoration\n', + ' & c )\n', + ' end subroutine foo\n', + ] + # Capture warnings from the validator module's logger. + records = [] + + class _Capture(_logging.Handler): + def emit(self, record): + records.append(record) + + cap = _Capture(level=_logging.WARNING) + val_mod._LOGGER.addHandler(cap) + try: + _join_continuation(src_lines, filename='/some/path/foo.F') + finally: + val_mod._LOGGER.removeHandler(cap) + self.assertEqual(len(records), 1, "expected exactly one warning") + msg = records[0].getMessage() + self.assertIn('/some/path/foo.F', msg) + self.assertIn(':2:', msg) # decoration is on line 2 of src_lines + class TestParseSubroutines(unittest.TestCase): From 687f5853373630bffc001e6a76bf7910b092b925 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 May 2026 14:57:56 -0600 Subject: [PATCH 15/74] Minor updates for GFS v16 suites in SCM --- capgen-ng/generator/group_cap.py | 13 +-- capgen-ng/generator/suite_types.py | 89 +++++++++++++++++--- doc/briefing.md | 16 ++-- doc/constituents_overhaul.md | 54 +++++++++++++ doc/migration.md | 4 +- doc/redesign_prompt.md | 12 +-- unit-tests/test_suite_types.py | 126 +++++++++++++++++++++++++++++ 7 files changed, 286 insertions(+), 28 deletions(-) diff --git a/capgen-ng/generator/group_cap.py b/capgen-ng/generator/group_cap.py index 185c5962..12213276 100644 --- a/capgen-ng/generator/group_cap.py +++ b/capgen-ng/generator/group_cap.py @@ -27,7 +27,7 @@ from metadata.parse_tools import FORTRAN_CONDITIONAL_REGEX, open_if_changed from metadata.variable_resolver import HostVarEntry -from generator.suite_types import _ptr_type_name, _ptr_type_for_arg +from generator.suite_types import _ptr_type_name_for_arg from generator.suite_resolver import ( ResolvedArg, ResolvedCall, @@ -858,8 +858,7 @@ def _generate_phase_subroutine( ) if arg.ptr_name and arg.ptr_name not in seen_ptr_names: seen_ptr_names.add(arg.ptr_name) - type_, kind, rank = _ptr_type_for_arg(arg) - ptr_tname = _ptr_type_name(type_, kind, rank) + ptr_tname = _ptr_type_name_for_arg(arg, resolved_call.scheme_name) local_decls.append( '{}type({}) :: {}'.format(_INDENT * 2, ptr_tname, arg.ptr_name) ) @@ -1065,13 +1064,17 @@ def _generate_group_cap( uses = _collect_group_uses(resolved_group, host_dict) # Add USE for types module when optional pointer args are present. + # Build the wrapper name via the per-arg helper so any + # unsupported shape (e.g. character(len=*)) raises a CCPPError + # naming the offending scheme + argument. ptr_type_names: Set[str] = set() for items in resolved_group.phase_calls.values(): for resolved_call in iter_phase_calls(items): for arg in resolved_call.args: if arg.ptr_name: - type_, kind, rank = _ptr_type_for_arg(arg) - ptr_type_names.add(_ptr_type_name(type_, kind, rank)) + ptr_type_names.add( + _ptr_type_name_for_arg(arg, resolved_call.scheme_name) + ) if ptr_type_names: types_mod = 'ccpp_{}_types'.format(suite_name) uses[types_mod] = ptr_type_names diff --git a/capgen-ng/generator/suite_types.py b/capgen-ng/generator/suite_types.py index 7f46247e..5baef2d1 100644 --- a/capgen-ng/generator/suite_types.py +++ b/capgen-ng/generator/suite_types.py @@ -56,7 +56,12 @@ def _split_external(type_: str) -> Tuple[str, str]: # Type-name helpers ######################################################################## -def _ptr_type_name(type_: str, kind: str, rank: int) -> str: +def _ptr_type_name( + type_: str, + kind: str, + rank: int, + context: str = '', +) -> str: """Return the Fortran derived-type name for a pointer wrapper. Parameters @@ -70,6 +75,12 @@ def _ptr_type_name(type_: str, kind: str, rank: int) -> str: Kind parameter (e.g. ``'kind_phys'``), or ``''`` if none. rank : int Number of array dimensions (0 = scalar). + context : str, optional + Free-form prefix passed through to :func:`_sanitize_len_suffix` + and prepended to any error message raised from there. Callers + identify the offending scheme + argument here so the user can + locate the metadata block to fix; see + :func:`_ptr_type_name_for_arg`. Returns ------- @@ -111,7 +122,7 @@ def _ptr_type_name(type_: str, kind: str, rank: int) -> str: # ``character_len:_rank1_ptr_type`` that the compiler # rejects. len_spec = kind[len('len='):].strip() - parts.append('len' + _sanitize_len_suffix(len_spec)) + parts.append('len' + _sanitize_len_suffix(len_spec, context=context)) else: parts.append(kind) parts.append('rank{}'.format(rank)) @@ -119,7 +130,7 @@ def _ptr_type_name(type_: str, kind: str, rank: int) -> str: return '_'.join(parts) -def _sanitize_len_suffix(len_spec: str) -> str: +def _sanitize_len_suffix(len_spec: str, context: str = '') -> str: """Return a Fortran-identifier-safe suffix for a ``character(len=…)`` spec. Pointer-wrapper type names embed the length specifier, e.g. @@ -136,31 +147,36 @@ def _sanitize_len_suffix(len_spec: str) -> str: Anything that doesn't fit those forms raises ``CCPPError`` rather than silently producing an illegal Fortran identifier. + + *context* — when non-empty, prefixed to every error message as + ``": ..."``. Callers identify the offending scheme + + argument so the user can locate the metadata block to fix. """ + prefix = '{}: '.format(context) if context else '' spec = len_spec.strip() if not spec: raise CCPPError( - "Empty character length specifier 'len=' in pointer-wrapper " + "{}Empty character length specifier 'len=' in pointer-wrapper " "type name construction; expected an integer literal, " - "a parameter name, or ':' for deferred length." + "a parameter name, or ':' for deferred length.".format(prefix) ) if spec == ':': return '_deferred' if spec == '*': raise CCPPError( - "character(len=*) cannot appear as a DDT component, so " + "{}character(len=*) cannot appear as a DDT component, so " "capgen-ng cannot generate a pointer-wrapper type for it. " "Use a concrete length, a parameter constant, or 'len=:' " "(deferred length, paired with allocatable / pointer) " - "in the metadata instead." + "in the metadata instead.".format(prefix) ) # Plain integer literal (digits) or Fortran identifier — accept verbatim. if spec.isdigit() or _IDENT_RE.match(spec): return spec raise CCPPError( - "Cannot derive a Fortran-identifier-safe pointer-wrapper type " + "{}Cannot derive a Fortran-identifier-safe pointer-wrapper type " "name from 'len={}'. Expected an integer literal, a parameter " - "identifier, or ':' (deferred length).".format(spec) + "identifier, or ':' (deferred length).".format(prefix, spec) ) @@ -187,17 +203,59 @@ def _ptr_type_for_arg(arg) -> Tuple[str, str, int]: variable directly — same type and kind. For Case 4 (optional + transform), the pointer targets the transformation temporary, which carries the scheme's kind. + + Narrow override for ``character(len=*)``: the resolver explicitly + accepts scheme ``kind=len=*`` paired with host ``kind=len=N`` (or + ``len=:``) as compatible and emits no kind transform — so we are + always in Case 2 here, and the pointer points at host data with a + concrete length. The DDT-component rules forbid ``len=*``, but the + host's concrete kind is usable verbatim, so fall through to the + host kind in that one situation. All other types keep the + scheme-wins precedence — Case 4 transform temps need the scheme + kind, and ``character`` is the only intrinsic where ``len=*`` + survives the resolver as an unconverted compatibility. """ if arg.host_entry is not None: type_ = arg.host_entry.type else: type_ = arg.suite_var.type_ - kind = arg.kind_scheme or (arg.host_entry.kind if arg.host_entry else - arg.suite_var.kind) + host_kind = (arg.host_entry.kind if arg.host_entry + else arg.suite_var.kind) + if type_ == 'character' and arg.kind_scheme == 'len=*': + kind = host_kind + else: + kind = arg.kind_scheme or host_kind rank = _ptr_rank(arg) return type_, kind, rank +def _ptr_type_name_for_arg(arg, scheme_name: str) -> str: + """Build the pointer-wrapper type name for *arg* with rich error context. + + Resolves ``(type_, kind, rank)`` via :func:`_ptr_type_for_arg`, then + delegates to :func:`_ptr_type_name` passing a *context* string of + the form ``"scheme '', optional argument [] + (standard_name=, intent=)"``. Any :class:`CCPPError` from + the name-construction path then carries enough information for the + user to find the offending metadata block without grepping. + + Use this at every site that calls :func:`_ptr_type_name` on a + resolved scheme argument. Bare :func:`_ptr_type_name` calls are + only appropriate when no per-argument context is available (e.g. + iterating the already-validated combo set in + :func:`_generate_suite_types`). + """ + type_, kind, rank = _ptr_type_for_arg(arg) + context = ( + "scheme '{}', optional argument [{}] " + "(standard_name={}, intent={})".format( + scheme_name, arg.scheme_local_name, + arg.standard_name, arg.intent, + ) + ) + return _ptr_type_name(type_, kind, rank, context=context) + + ######################################################################## # Collection helpers ######################################################################## @@ -207,6 +265,14 @@ def _collect_ptr_type_combos( ) -> Set[Tuple[str, str, int]]: """Collect unique (type, kind, rank) tuples needed by optional args. + Validates each combo eagerly via :func:`_ptr_type_name_for_arg` so + that any unsupported shape (notably ``character(len=*)``) raises a + :class:`CCPPError` carrying the offending scheme + argument name + here, rather than later from the bare-tuple loop in + :func:`_generate_suite_types` (which would have nothing useful to + say to the user). The constructed name itself is discarded — only + the validation matters at this point. + Parameters ---------- suite_res : SuiteResolution @@ -221,6 +287,7 @@ def _collect_ptr_type_combos( for resolved_call in iter_phase_calls(items): for arg in resolved_call.args: if arg.ptr_name: + _ptr_type_name_for_arg(arg, resolved_call.scheme_name) combos.add(_ptr_type_for_arg(arg)) return combos diff --git a/doc/briefing.md b/doc/briefing.md index 237ef706..76f5b26d 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -329,13 +329,19 @@ don't rebuild downstream objects unless something actually moved. ## 10. Where things stand right now -- **Unit tests**: 1208 passing on `main`. +- **Unit tests**: 1220 passing on `main`. - **End-to-end tests passing**: `advection`, `unit_conv`, `nested_suite`, `variable_transform`, `instances`, `ddt`. -- **CCPP-SCM**: actively driving development this week — every build - / runtime failure surfaced this week landed as a fix in capgen-ng - (rather than being patched around in the host). Most of the - `phys_ps` group now builds end-to-end via `--legacy-mode`. +- **CCPP-SCM**: actively driving development — every build / runtime + failure surfaced this week landed as a fix in capgen-ng (rather than + being patched around in the host). Most of the `phys_ps` group now + builds end-to-end via `--legacy-mode`. +- **`--no-host-introspection`** (new, 2026-05-14): stubs the bodies of + the five suite-introspection routines in `ccpp_static_api.F90`, + shrinking the file from ~33k lines to ~800 for the 10-suite SCM + build (the introspection case-blocks were making `-O3` compilation + effectively hang). Signatures stay so existing host callers still + link; stubbed bodies return `errflg = 1` with a clear `errmsg`. - **CAM-SIMA**: not yet reconnected; pending the constituents overhaul decision. - **UFS Weather Model / NEPTUNE**: not yet attempted; SCM is the diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index 04c6b8c2..717d548c 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -494,6 +494,60 @@ scheme-registering schemes don't rely on this; documented in `instance_number` through `ccpp_constituent_index` (interface change) or maintaining a per-instance pointer table. +### 4.12 Capgen-ng: drop `diagnostic_name_fixed`, keep only `diagnostic_name` (OPEN — proposed simplification) + +Today the metadata layer carries two mutually-exclusive scheme-arg +attributes: + +- `diagnostic_name = X` — emits `diagnostic_name="X"` in `datatable.xml`; + defaults to `local_name` when absent. +- `diagnostic_name_fixed = Y` — emits `diagnostic_name_fixed="Y"` in + `datatable.xml`; the `diagnostic_name` slot stays empty (no + auto-default to `local_name`). + +The behavioural difference is purely *which attribute name* host +tooling sees in `datatable.xml` — both attributes carry the same kind +of value (a Fortran-identifier-shaped string), and both are passed +through unmodified. `_fixed` is a signal to the host "use verbatim, do +not decorate or transform"; but `diagnostic_name = X` already means +exactly that — the cap code never decorates the value, and any host +tooling that wants to decorate would have to opt in by parsing a +separate attribute (or by syntactic convention on the value itself). + +**Proposal:** Remove `diagnostic_name_fixed` from the metadata layer +and the parser. Keep `diagnostic_name` with the existing defaulting +rule (explicit → use it; absent → fall back to `local_name`). Hosts +that today rely on the `_fixed` semantic ("don't auto-default to +`local_name`") get the same outcome by simply *setting* +`diagnostic_name` to the desired exact value. + +Touchpoints to retire: + +- `metadata/parse_tools/parse_checkers.py::check_diagnostic_fixed` and + the mutual-exclusion block at the top of `check_diagnostic_id`. +- `metadata_table.py::MetaVar._KNOWN_ATTRS` entry and the + `@property diagnostic_name` fallback that returns `''` when + `_diagnostic_name_fixed` is set. +- `generator/datatable.py:267-269` emission of the + `diagnostic_name_fixed` XML attribute. +- Existing unit-test coverage for `diagnostic_name_fixed` becomes + obsolete and is removed (not migrated). + +**Why it's worth doing as part of the overhaul:** the attribute has no +unique semantics that `diagnostic_name` can't express, and dropping it +shrinks the metadata-layer surface area at the same time the +`set_diagnostic_name(value)` framework setter (§4.4 / §4.10) is being +added on the framework side. Hosts that want runtime override get +`set_diagnostic_name`; hosts that want metadata-declared values get +`diagnostic_name`. There is no third use case that needs `_fixed`. + +**Risk:** non-CCPP-ng metadata in the wild may carry +`diagnostic_name_fixed`. Mitigation: a one-line legacy-mode rewrite +(`metadata/legacy_compat.py`) translates the deprecated attribute to +`diagnostic_name` at parse time with a loud warning, identical in +spirit to the existing `horizontal_loop_extent → horizontal_dimension` +shim. Remove the rewrite once known consumers are migrated. + --- ## 5. Property classification (Class A vs Class B) diff --git a/doc/migration.md b/doc/migration.md index d222c6f3..99d431a4 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -6,7 +6,7 @@ Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to **capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and `doc/redesign_analysis.md` (analysis of the old systems). -*Last revised: 2026-05-13 (late evening).* Current unit-test suite: 1208 passing. +*Last revised: 2026-05-14.* Current unit-test suite: 1220 passing. **Repository layout** (post-2026-05-13 cleanup): tooling lives under `capgen-ng/` (top-level of this repo). Unit tests live at the top @@ -518,7 +518,7 @@ Always generated: - `ccpp___cap.F90` — per-group phase implementations. - `ccpp__data.F90` — suite-owned interstitial DDT + module-level array. - `ccpp__types.F90` — pointer-wrapper types for optional args. -- `ccpp_.meta` — inspection artifact; matches the generated cap. +- `ccpp__data.meta` — inspection artifact; pairs with `ccpp__data.F90` (`.meta` ↔ `.F90` filename convention). - `datatable.xml` — build-system + host-introspection metadata. When any scheme registers constituents: diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index 4b33d5f2..3229fc0f 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -916,9 +916,11 @@ sites in the group cap, not in the suite cap. ### 13.3 Metadata -The generator also writes a `type = suite` metadata table (`ccpp_.meta`) as a -byproduct. This file is for inspection and debugging — it is not consumed by the -generator on subsequent runs. +The generator also writes a `type = suite` metadata table (`ccpp__data.meta`) +as a byproduct. The `_data` suffix matches the companion Fortran file +`ccpp__data.F90`, satisfying the `.meta` ↔ `.F90` filename pairing. This +file is for inspection and debugging — it is not consumed by the generator on +subsequent runs. --- @@ -993,7 +995,7 @@ All files are written to `--output-root`. | `ccpp___cap.F90` | Group cap — scheme call sites, state array, optionals, transformations. USEs `ccpp_kinds` for any kind referenced in transformation temporaries. | | `ccpp__data.F90` | Suite data module — framework-owned interstitial DDT. USEs `ccpp_kinds` for any kind referenced in suite-var declarations. | | `ccpp__types.F90` | Shared cap types — optional pointer wrapper types, transformation locals. USEs `ccpp_kinds` for any kind referenced in pointer wrappers. | -| `ccpp_.meta` | Generated `type = suite` metadata table (output-only, for inspection) | +| `ccpp__data.meta` | Generated `type = suite` metadata table — pairs with `ccpp__data.F90` (output-only, for inspection) | | `datatable.xml` | Generator database for `ccpp_datafile.py` queries. `ccpp_kinds.F90` and `ccpp_static_api.F90` appear in `...` (matches original capgen). | `ccpp_kinds.F90` is a dependency of all generated Fortran files that reference any kind parameter (group cap, suite types, suite data). The static API and suite cap have no kind references and do not USE it. @@ -1254,7 +1256,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` ### Test status -- **Unit tests**: 1208 passing (`python -m pytest unit-tests/`). +- **Unit tests**: 1220 passing (`python -m pytest unit-tests/`). - **End-to-end tests**: `advection`, `unit_conv`, `nested_suite`, `variable_transform`, `instances`, `ddt` covered. SCM running against ccpp-physics is the active driver right now — most of the diff --git a/unit-tests/test_suite_types.py b/unit-tests/test_suite_types.py index 0b9be141..819491ba 100644 --- a/unit-tests/test_suite_types.py +++ b/unit-tests/test_suite_types.py @@ -7,11 +7,14 @@ import unittest +from types import SimpleNamespace + from generator.suite_types import ( _collect_ddt_uses, _fortran_type_str_simple, _generate_suite_types, _ptr_type_name, + _ptr_type_name_for_arg, ) from metadata.parse_tools import CCPPError @@ -125,6 +128,129 @@ def test_character_len_expression_rejected(self): with self.assertRaisesRegex(CCPPError, 'Fortran-identifier-safe'): _ptr_type_name('character', 'len=N+1', 1) + def test_error_context_prefixed_when_provided(self): + """When ``context`` is supplied, the prefix lands at the very + start of the error message so the user can immediately tell + which arg/scheme is the offender.""" + import re as _re + prefix = "scheme 'GFS_rrtmgp_pre', optional argument [foo]" + with self.assertRaisesRegex(CCPPError, _re.escape(prefix)): + _ptr_type_name('character', 'len=*', 1, context=prefix) + + def test_error_no_context_unchanged(self): + """No context → no prefix; existing message wording unchanged.""" + with self.assertRaisesRegex(CCPPError, '^character\\(len=\\*\\)'): + _ptr_type_name('character', 'len=*', 1) + + +class TestPtrTypeNameForArg(unittest.TestCase): + """``_ptr_type_name_for_arg`` builds a rich-context wrapper around + ``_ptr_type_name`` so the user can locate the offending metadata + block without grepping.""" + + def _make_arg(self, type_, host_kind, dimensions, local='foo', + std='some_standard_name', intent='in', + scheme_kind=''): + # Minimal duck-typed ResolvedArg. _ptr_type_for_arg reads + # type_/host_kind from host_entry; scheme_kind from + # arg.kind_scheme; rank from host_entry.dimensions. The + # context-string builder reads standard_name / + # scheme_local_name / intent. + host_entry = SimpleNamespace( + type=type_, kind=host_kind, dimensions=dimensions, + ) + return SimpleNamespace( + host_entry=host_entry, + suite_var=None, + kind_scheme=scheme_kind, + standard_name=std, + scheme_local_name=local, + intent=intent, + ) + + def test_clean_case_returns_name(self): + """Happy path: concrete-length character with matching scheme + kind → wrapper name built without raising.""" + arg = self._make_arg('character', 'len=10', ['ncols'], + scheme_kind='len=10') + self.assertEqual( + _ptr_type_name_for_arg(arg, 'my_scheme'), + 'character_len10_rank1_ptr_type', + ) + + def test_scheme_lenstar_host_concrete_uses_host_kind(self): + """The SCM case driving the narrow fix: scheme metadata + declares ``kind=len=*`` (legal as an assumed-length dummy) and + the host declares ``kind=len=128``. The resolver treats this + pair as compatible with no kind transform, so the pointer + wrapper must use the host's concrete length — not the + scheme's ``len=*`` (which would be illegal as a DDT + component). Without the override this raises CCPPError; with + the override it returns the concrete wrapper name.""" + arg = self._make_arg( + 'character', 'len=128', + ['number_of_active_gases_used_by_RRTMGP'], + local='active_gases_array', + std='list_of_active_gases_used_by_RRTMGP', + intent='in', + scheme_kind='len=*', + ) + self.assertEqual( + _ptr_type_name_for_arg(arg, 'GFS_rrtmgp_pre'), + 'character_len128_rank1_ptr_type', + ) + + def test_scheme_lenstar_host_deferred_uses_host_kind(self): + """Same override, but with the host declaring ``kind=len=:`` + (deferred length). The wrapper takes the host's ``len=:`` and + the name builder maps it to ``_deferred`` (existing + deferred-length rule).""" + arg = self._make_arg( + 'character', 'len=:', ['ncols'], + scheme_kind='len=*', + ) + self.assertEqual( + _ptr_type_name_for_arg(arg, 'my_scheme'), + 'character_len_deferred_rank1_ptr_type', + ) + + def test_real_kind_transform_unchanged(self): + """Regression: the narrow override applies only to + ``character`` + scheme-``len=*``. Real/integer args with a + kind transform must still take the scheme's kind so the Case-4 + transform-temp wrapping keeps working.""" + arg = self._make_arg( + 'real', 'kind_dbl_prec', ['ncols'], + scheme_kind='kind_phys', + ) + self.assertEqual( + _ptr_type_name_for_arg(arg, 'my_scheme'), + 'real_kind_phys_rank1_ptr_type', + ) + + def test_host_actually_lenstar_still_errors(self): + """Edge case: if the *host* metadata itself declares + ``kind=len=*`` (which would normally be rejected upstream), + the wrapper builder still has nothing it can use — the error + fires and names the offending scheme + arg. This guards the + error-enrichment path itself.""" + arg = self._make_arg( + 'character', 'len=*', + ['number_of_active_gases_used_by_RRTMGP'], + local='active_gases_array', + std='list_of_active_gases_used_by_RRTMGP', + intent='in', + scheme_kind='', + ) + with self.assertRaises(CCPPError) as cm: + _ptr_type_name_for_arg(arg, 'GFS_rrtmgp_pre') + msg = str(cm.exception) + self.assertIn("scheme 'GFS_rrtmgp_pre'", msg) + self.assertIn('[active_gases_array]', msg) + self.assertIn('list_of_active_gases_used_by_RRTMGP', msg) + self.assertIn('intent=in', msg) + self.assertIn('cannot appear as a DDT component', msg) + class TestCollectDdtUses(unittest.TestCase): From e25e253afcc5152fc9c27ba6e0595bdf2f59b72c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 14 May 2026 19:50:47 -0600 Subject: [PATCH 16/74] Minor updates from last CCPP-SCM suites --- capgen-ng/metadata/variable_resolver.py | 28 ++++++++++++-- doc/briefing.md | 2 +- doc/migration.md | 2 +- doc/redesign_prompt.md | 2 +- unit-tests/test_variable_resolver.py | 51 +++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 6 deletions(-) diff --git a/capgen-ng/metadata/variable_resolver.py b/capgen-ng/metadata/variable_resolver.py index 37f0272f..cae8aec0 100644 --- a/capgen-ng/metadata/variable_resolver.py +++ b/capgen-ng/metadata/variable_resolver.py @@ -777,6 +777,11 @@ def __init__(self) -> None: # falls back to the scheme name (the common case where the .meta # file shares its base name with the Fortran module). self._modules: Dict[str, str] = {} + # _source_paths[scheme_name][phase] = .meta file that first + # registered this (scheme, phase) pair. Consulted only on the + # duplicate-phase error path so the message can name both the + # original registration site and the duplicate. + self._source_paths: Dict[str, Dict[str, str]] = {} @classmethod def build_from(cls, scheme_tables: List[MetadataTable]) -> 'SchemeStore': @@ -801,6 +806,7 @@ def build_from(cls, scheme_tables: List[MetadataTable]) -> 'SchemeStore': name = tbl.table_name if name not in store._data: store._data[name] = {} + store._source_paths[name] = {} # Resolve module: explicit ``module_name`` from the table # properties overrides the implicit "module name equals scheme # name" convention. See doc/scheme metadata format. @@ -810,13 +816,29 @@ def build_from(cls, scheme_tables: List[MetadataTable]) -> 'SchemeStore': if sec.phase is None: continue if sec.phase in store._data[name]: + first_path = store._source_paths[name].get(sec.phase, '') + dup_path = tbl.file_path or '' + # Same-path duplicate is the common case (a .meta + # file listed twice in the host's --scheme-files + # input, often a stray CMake list entry); call it + # out so the user knows to look in the build glue + # rather than in the metadata content. + if first_path == dup_path: + hint = (' (both paths are identical — likely a ' + 'duplicate entry in the --scheme-files ' + 'list passed to capgen-ng)') + else: + hint = '' raise CCPPError( - "Duplicate phase '{}' for scheme '{}'; " - "check that the same scheme metadata is not loaded twice".format( - sec.phase, name + "Duplicate phase '{}' for scheme '{}': " + "first registered from '{}', then again from " + "'{}'.{} Check that the same scheme metadata " + "is not loaded twice.".format( + sec.phase, name, first_path, dup_path, hint, ) ) store._data[name][sec.phase] = list(sec.variables) + store._source_paths[name][sec.phase] = tbl.file_path or '' return store def scheme_names(self) -> List[str]: diff --git a/doc/briefing.md b/doc/briefing.md index 76f5b26d..5a05e070 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -329,7 +329,7 @@ don't rebuild downstream objects unless something actually moved. ## 10. Where things stand right now -- **Unit tests**: 1220 passing on `main`. +- **Unit tests**: 1229 passing on `main`. - **End-to-end tests passing**: `advection`, `unit_conv`, `nested_suite`, `variable_transform`, `instances`, `ddt`. - **CCPP-SCM**: actively driving development — every build / runtime diff --git a/doc/migration.md b/doc/migration.md index 99d431a4..f90fe668 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -6,7 +6,7 @@ Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to **capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and `doc/redesign_analysis.md` (analysis of the old systems). -*Last revised: 2026-05-14.* Current unit-test suite: 1220 passing. +*Last revised: 2026-05-14 (end-of-day).* Current unit-test suite: 1229 passing. **Repository layout** (post-2026-05-13 cleanup): tooling lives under `capgen-ng/` (top-level of this repo). Unit tests live at the top diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index 3229fc0f..906c7838 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -1256,7 +1256,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` ### Test status -- **Unit tests**: 1220 passing (`python -m pytest unit-tests/`). +- **Unit tests**: 1229 passing (`python -m pytest unit-tests/`). - **End-to-end tests**: `advection`, `unit_conv`, `nested_suite`, `variable_transform`, `instances`, `ddt` covered. SCM running against ccpp-physics is the active driver right now — most of the diff --git a/unit-tests/test_variable_resolver.py b/unit-tests/test_variable_resolver.py index 16daf6e6..a06de790 100644 --- a/unit-tests/test_variable_resolver.py +++ b/unit-tests/test_variable_resolver.py @@ -1175,6 +1175,57 @@ def test_duplicate_phase_raises(self): self.assertIn('run', str(cm.exception)) self.assertIn('my_scheme', str(cm.exception)) + def test_duplicate_phase_names_both_source_files(self): + """When two distinct ``.meta`` files declare the same + (scheme, phase) pair, the error must name both file paths so + the user can locate the conflict instead of grepping the + ``--scheme-files`` list.""" + tables_a = _parse_lines( + _SIMPLE_SCHEME_SRC.splitlines(keepends=True), + '/projA/my_scheme.meta', + ) + tables_b = _parse_lines( + _SIMPLE_SCHEME_SRC.splitlines(keepends=True), + '/projB/my_scheme.meta', + ) + with self.assertRaises(CCPPError) as cm: + SchemeStore.build_from(tables_a + tables_b) + msg = str(cm.exception) + self.assertIn('my_scheme', msg) + # Both paths appear, in order: original then duplicate. + idx_a = msg.find('/projA/my_scheme.meta') + idx_b = msg.find('/projB/my_scheme.meta') + self.assertGreaterEqual(idx_a, 0, 'original path not in error: ' + msg) + self.assertGreaterEqual(idx_b, 0, 'duplicate path not in error: ' + msg) + self.assertLess(idx_a, idx_b, + 'expected original (A) before duplicate (B)') + # Different paths → no CMake-list hint. + self.assertNotIn('--scheme-files', msg) + + def test_duplicate_phase_same_path_hints_at_cmake_list(self): + """The motivating SCM case: a single ``.meta`` path listed + twice in the host's ``--scheme-files`` argument (typically a + stray CMake list entry). Both reported paths are byte-equal, + and the message appends an explicit ``--scheme-files`` hint so + the user knows to look in the build glue, not in the + metadata.""" + path = '/host/ccpp/physics/GWD/ugwpv1_gsldrag.meta' + tables_first = _parse_lines( + _SIMPLE_SCHEME_SRC.splitlines(keepends=True), path, + ) + tables_again = _parse_lines( + _SIMPLE_SCHEME_SRC.splitlines(keepends=True), path, + ) + with self.assertRaises(CCPPError) as cm: + SchemeStore.build_from(tables_first + tables_again) + msg = str(cm.exception) + # The single path appears (at least once; same string both + # places, so a single substring search suffices). + self.assertIn(path, msg) + # CMake-list duplication hint fires when the paths are equal. + self.assertIn('--scheme-files', msg) + self.assertIn('identical', msg) + def test_build_from_scheme_files(self): """Integration: build SchemeStore from the multipart scheme sample file.""" from metadata.metadata_table import parse_metadata_file From f29e7e74d1e7e0de7ad9452b4ad0e46abaf51e24 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 May 2026 06:51:49 -0600 Subject: [PATCH 17/74] Add --trace flag to capgen call --- capgen-ng/ccpp_capgen_ng.py | 22 +++ capgen-ng/generator/group_cap.py | 30 +++- capgen-ng/generator/static_api.py | 59 +++++-- capgen-ng/generator/suite_cap.py | 45 ++++- capgen-ng/generator/trace.py | 190 ++++++++++++++++++++++ end-to-end-tests.sh | 1 + end-to-end-tests/advection/CMakeLists.txt | 6 +- end-to-end-tests/cmake/ccpp_capgen.cmake | 36 ++-- unit-tests/test_static_api.py | 91 +++++++++-- unit-tests/test_suite_cap.py | 55 +++++++ unit-tests/test_trace.py | 179 ++++++++++++++++++++ 11 files changed, 660 insertions(+), 54 deletions(-) create mode 100644 capgen-ng/generator/trace.py create mode 100644 unit-tests/test_trace.py diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index 7a947b7e..582df41c 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -264,6 +264,23 @@ def _build_arg_parser() -> argparse.ArgumentParser: "the introspection case-blocks." ), ) + parser.add_argument( + '--trace', + action='store_true', + help=( + "Set the default value of the per-module ``trace`` " + "parameter to ``.true.`` in every generated cap. The " + "gated ``if (trace) write(error_unit,*) ...`` lines are " + "ALWAYS emitted (one per cap subroutine that has " + "intent(in)/inout control dummies) so that strict " + "unused-variable warnings -- such as Intel oneAPI's -- " + "are silenced even when tracing is off. This flag only " + "flips the compile-time default; a developer can also " + "hand-edit ``logical, parameter :: trace`` in any " + "generated cap to ``.true.`` to enable tracing for that " + "module and rebuild." + ), + ) return parser @@ -796,6 +813,7 @@ def capgen( kind_types: Dict[str, Tuple[str, str]], logger: Optional[logging.Logger] = None, no_host_introspection: bool = False, + trace: bool = False, ) -> None: """Programmatic entry point for the cap generator. @@ -924,6 +942,7 @@ def capgen( write_group_cap( suite.name, resolved_group.group_name, resolved_group, host_dict, output_root, logger=log, + trace=trace, ) # Suite data module @@ -947,6 +966,7 @@ def capgen( write_suite_cap( suite.name, suite_res, scheme_store, output_root, host_dict, logger=log, + trace=trace, ) # ---- static API (one file for all suites) ------------------------------ @@ -954,6 +974,7 @@ def capgen( suite_names, suite_resolutions, output_root, host_dict, scheme_store, logger=log, no_host_introspection=no_host_introspection, + trace=trace, ) # ---- host-wide constituent module (only when any suite touches @@ -1136,6 +1157,7 @@ def main(argv: Optional[List[str]] = None) -> int: output_root=args.output_root, kind_types=kind_types, no_host_introspection=args.no_host_introspection, + trace=args.trace, ) except CCPPError as exc: _LOGGER.error("%s", exc) diff --git a/capgen-ng/generator/group_cap.py b/capgen-ng/generator/group_cap.py index 12213276..dc87d567 100644 --- a/capgen-ng/generator/group_cap.py +++ b/capgen-ng/generator/group_cap.py @@ -37,6 +37,11 @@ iter_phase_calls, iter_phase_subcycles, ) +from generator.trace import ( + emit_module_gate, + emit_trace_block, + ensure_error_unit_use, +) _INDENT = ' ' _CONT = ' &' @@ -876,6 +881,15 @@ def _generate_phase_subroutine( errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') sub_label = '{}_{}_{}'.format(suite_name, group_name, phase) + # ---- trace block (always emitted; gated by the module ``trace`` + # parameter so the I/O is dead-code-eliminated when trace=.false.). + # Placed before errmsg/errflg init so the write references no + # intent(out) dummy and fires even when a state guard then bails. + trace_lines = emit_trace_block(sub_name, ctrl_entries, call_indent) + if trace_lines: + lines.extend(trace_lines) + lines.append('') + # ---- initialize error reporting vars ------------------------------- if errflg_local and errmsg_local: lines.append("{}{} = ''".format(call_indent, errmsg_local)) @@ -1031,6 +1045,7 @@ def _generate_group_cap( group_name: str, resolved_group: ResolvedGroup, host_dict, + trace: bool = False, ) -> List[str]: """Generate the full group cap module source lines. @@ -1087,6 +1102,8 @@ def _generate_group_cap( use_lines = _use_statements(uses) use_lines.extend(_scheme_use_statements(resolved_group)) + # Trace block writes to error_unit; ensure the USE is present. + ensure_error_unit_use(use_lines, _INDENT) lines.extend(use_lines) if use_lines: lines.append('') @@ -1113,6 +1130,14 @@ def _generate_group_cap( lines.append('{}integer, private, parameter :: CCPP_GROUP_IN_TIMESTEP = 2'.format(_INDENT)) lines.append('{}integer, private, allocatable :: ccpp_group_state(:)'.format(_INDENT)) + # ---- trace gate ------------------------------------------------------- + # Module-level compile-time toggle; flip to .true. (or pass --trace at + # generation time) to enable the per-subroutine trace writes. When + # .false., the gated writes are dead-code-eliminated by the compiler + # but the control dummies remain syntactically referenced, which + # silences strict unused-dummy warnings (Intel oneAPI in particular). + lines.extend(emit_module_gate(trace, _INDENT)) + lines.append('') lines.append('contains') lines.append('') @@ -1162,6 +1187,7 @@ def write_group_cap( host_dict, output_root: str, logger: Optional[logging.Logger] = None, + trace: bool = False, ) -> str: """Write the group cap Fortran module to *output_root*. @@ -1185,7 +1211,9 @@ def write_group_cap( filename = 'ccpp_{}_{}_cap.F90'.format(suite_name, group_name) out_path = os.path.join(output_root, filename) - lines = _generate_group_cap(suite_name, group_name, resolved_group, host_dict) + lines = _generate_group_cap( + suite_name, group_name, resolved_group, host_dict, trace=trace, + ) with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return out_path diff --git a/capgen-ng/generator/static_api.py b/capgen-ng/generator/static_api.py index 003c0595..d8137679 100644 --- a/capgen-ng/generator/static_api.py +++ b/capgen-ng/generator/static_api.py @@ -53,6 +53,11 @@ iter_phase_subcycles, ) from metadata.variable_resolver import HostVarEntry +from generator.trace import ( + emit_module_gate, + emit_trace_block, + ensure_error_unit_use, +) from generator.suite_cap import ( _all_suite_scheme_names, _schemes_with_register, @@ -380,6 +385,13 @@ def _register_subroutine(suite_names: List[str], host_dict=None) -> List[str]: lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + trace_entries = [suite_name_entry] if suite_name_entry else [] + trace_lines = emit_trace_block( + 'ccpp_register', trace_entries, i2, instance_local=inst_local, + ) + if trace_lines: + lines.append('') + lines.extend(trace_lines) lines += [ '', "{}{} = ''".format(i2, errmsg_local), @@ -439,6 +451,13 @@ def _init_subroutine(suite_names: List[str], host_dict=None) -> List[str]: lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + trace_entries = [suite_name_entry] if suite_name_entry else [] + trace_lines = emit_trace_block( + 'ccpp_init', trace_entries, i2, instance_local=inst_local, + ) + if trace_lines: + lines.append('') + lines.extend(trace_lines) lines += [ '', "{}{} = ''".format(i2, errmsg_local), @@ -497,6 +516,13 @@ def _final_subroutine(suite_names: List[str], host_dict=None) -> List[str]: lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + trace_entries = [suite_name_entry] if suite_name_entry else [] + trace_lines = emit_trace_block( + 'ccpp_final', trace_entries, i2, instance_local=inst_local, + ) + if trace_lines: + lines.append('') + lines.extend(trace_lines) lines += [ '', "{}{} = ''".format(i2, errmsg_local), @@ -577,6 +603,13 @@ def _physics_subroutine( '{}{}{}{} :: {}'.format(i2, t, intent, dim, entry.local_name) ) + # Trace block: every intent(in)/inout control dummy is referenced so + # strict compilers don't flag any of them as unused. + trace_lines = emit_trace_block(sub_name, ctrl_entries, i2) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + lines.append('') # Initialize error reporting vars before any work. @@ -910,6 +943,7 @@ def _generate_static_api( host_dict=None, scheme_store: Optional[SchemeStore] = None, no_host_introspection: bool = False, + trace: bool = False, ) -> List[str]: """Generate the full ``ccpp_static_api.F90`` module source lines. @@ -945,17 +979,12 @@ def _generate_static_api( lines.append('module ccpp_static_api') lines.append('') - # Pull in ``error_unit`` for the stubbed ``ccpp_physics_suite_list`` - # body (which has no errflg/errmsg arg, so error_unit is the only - # available channel). Only emitted when --no-host-introspection is - # on, to keep the module imports minimal in the normal case. - if no_host_introspection: - lines.append( - '{}use iso_fortran_env, only: error_unit'.format(_INDENT) - ) - - # USE each suite cap module. ``_register`` is now mandatory and - # always emitted in the suite cap, so always import it here too. + # Collect USE lines into a list so the trace helper can guarantee + # ``error_unit`` is present. The trace block writes to error_unit + # and is emitted in every cap subroutine (gated by the module + # ``trace`` parameter), so the USE is now unconditional. Replaces + # an earlier --no-host-introspection-only emission. + use_lines: List[str] = [] for sname in suite_names: suite_cap_mod = 'ccpp_{}_cap'.format(sname) suite_subs = [] @@ -965,7 +994,9 @@ def _generate_static_api( suite_subs.append('{}_physics_{}'.format(sname, phase)) suite_subs.append('{}_final'.format(sname)) syms = ', '.join(suite_subs) - lines.append('{}use {}, only: {}'.format(_INDENT, suite_cap_mod, syms)) + use_lines.append('{}use {}, only: {}'.format(_INDENT, suite_cap_mod, syms)) + ensure_error_unit_use(use_lines, _INDENT) + lines.extend(use_lines) # Re-export the host-facing constituent API + the constituent object so # host code can do ``use ccpp_static_api, only: ...`` for *everything* @@ -1019,6 +1050,8 @@ def _generate_static_api( for sub in pub_subs: lines.append('{}public :: {}'.format(_INDENT, sub)) + lines.append('') + lines.extend(emit_module_gate(trace, _INDENT)) lines.append('') lines.append('contains') @@ -1068,6 +1101,7 @@ def write_static_api( scheme_store: Optional[SchemeStore] = None, logger: Optional[logging.Logger] = None, no_host_introspection: bool = False, + trace: bool = False, ) -> str: """Write ``ccpp_static_api.F90`` to *output_root*. @@ -1109,6 +1143,7 @@ def write_static_api( lines = _generate_static_api( suite_names, suite_resolutions, host_dict, scheme_store, no_host_introspection=no_host_introspection, + trace=trace, ) with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 2fdc133c..21c944ba 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -31,6 +31,11 @@ SuiteResolution, iter_phase_calls, ) +from generator.trace import ( + emit_module_gate, + emit_trace_block, + ensure_error_unit_use, +) from generator.group_cap import ( _ctrl_args_for_phase, _ctrl_intent_for, @@ -337,6 +342,13 @@ def _register_lines( ) lines.append('{}integer :: num_consts, i'.format(i2)) + # Trace block: dummies referenced inside the gated write so strict + # compilers don't flag instance_number as unused when the gate is off. + trace_lines = emit_trace_block(sub_name, [], i2, instance_local=inst_local) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + lines += [ '', "{}{} = ''".format(i2, errmsg_local), @@ -497,6 +509,12 @@ def _init_lines( lines += [ '{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local), '{}integer, intent(out) :: {}'.format(i2, errflg_local), + ] + trace_lines = emit_trace_block(sub_name, [], i2, instance_local=inst_local) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + lines += [ '', "{}{} = ''".format(i2, errmsg_local), '{}{} = 0'.format(i2, errflg_local), @@ -635,6 +653,12 @@ def _final_lines( lines += [ '{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local), '{}integer, intent(out) :: {}'.format(i2, errflg_local), + ] + trace_lines = emit_trace_block(sub_name, [], i2, instance_local=inst_local) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + lines += [ '', "{}{} = ''".format(i2, errmsg_local), '{}{} = 0'.format(i2, errflg_local), @@ -774,6 +798,13 @@ def _physics_dispatch_lines( '{}{}{}{} :: {}'.format(i2, t, intent, dim, entry.local_name) ) + # Trace block: references every intent(in)/inout control dummy so that + # strict compilers don't flag any of them as unused. + trace_lines = emit_trace_block(sub_name, ctrl_entries, i2) + if trace_lines: + lines.append('') + lines.extend(trace_lines) + lines.append('') # Initialize error reporting vars before any work, then guard on the @@ -949,6 +980,7 @@ def _generate_suite_cap( suite_res: SuiteResolution, scheme_store: SchemeStore, host_dict=None, + trace: bool = False, ) -> List[str]: """Generate the full ``ccpp__cap.F90`` module source lines. @@ -979,6 +1011,7 @@ def _generate_suite_cap( lines.append('') # USE statements: one per group cap (all phase + state subroutines). + use_lines: List[str] = [] for resolved_group in suite_res.groups: group_cap_mod = 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'cap') syms_list = [ @@ -987,10 +1020,14 @@ def _generate_suite_cap( ] syms_list.append('ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'state_alloc')) syms_list.append('ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'state_dealloc')) - lines.append('{}use {}, only: {}'.format( + use_lines.append('{}use {}, only: {}'.format( _INDENT, group_cap_mod, ', '.join(syms_list) )) + # Trace block writes to error_unit; ensure the USE is present. + ensure_error_unit_use(use_lines, _INDENT) + lines.extend(use_lines) + lines.append('') lines.append('{}implicit none'.format(_INDENT)) lines.append('{}private'.format(_INDENT)) @@ -1016,6 +1053,7 @@ def _generate_suite_cap( lines.append('{}integer, private, parameter :: CCPP_SUITE_REGISTERED = 1'.format(_INDENT)) lines.append('{}integer, private, parameter :: CCPP_SUITE_FRAMEWORK_INITIALIZED = 2'.format(_INDENT)) lines.append('{}integer, private, allocatable :: ccpp_suite_state(:)'.format(_INDENT)) + lines.extend(emit_module_gate(trace, _INDENT)) lines.append('') lines.append('contains') @@ -1046,6 +1084,7 @@ def write_suite_cap( output_root: str, host_dict=None, logger: Optional[logging.Logger] = None, + trace: bool = False, ) -> str: """Write ``ccpp__cap.F90`` to *output_root*. @@ -1068,7 +1107,9 @@ def write_suite_cap( filename = 'ccpp_{}_cap.F90'.format(suite_name) out_path = os.path.join(output_root, filename) - lines = _generate_suite_cap(suite_name, suite_res, scheme_store, host_dict) + lines = _generate_suite_cap( + suite_name, suite_res, scheme_store, host_dict, trace=trace, + ) with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') return out_path diff --git a/capgen-ng/generator/trace.py b/capgen-ng/generator/trace.py new file mode 100644 index 00000000..4fa3b1a6 --- /dev/null +++ b/capgen-ng/generator/trace.py @@ -0,0 +1,190 @@ +"""Trace-emission helpers shared by all cap generators. + +A generated cap module carries a compile-time gate:: + + use, intrinsic :: iso_fortran_env, only: error_unit + logical, parameter :: trace = .false. + +and every cap subroutine that has at least one ``intent(in)``/ +``intent(inout)`` control dummy emits a guarded write as the very first +line of its body:: + + if (trace) write(error_unit, *) & + 'CCPP TRACE :', & + ' =', , & + ' =', trim(), & + ... + +Two effects: + +1. With ``trace = .false.`` (the default) the compiler eliminates the + write at the dead branch, but every control dummy is *syntactically* + referenced inside the dead block. This silences "unused dummy + argument" warnings on strict compilers (Intel oneAPI in particular) + without any runtime cost. +2. Flipping the parameter to ``.true.`` -- either via the ``--trace`` CLI + flag at generation time, or by hand-editing one generated file -- + turns the cap into a self-describing trace, useful for diagnosing + call-chain problems in a host integration. + +The helpers below are pure (no I/O, no side effects on the caller) and +return lists of Fortran source lines with no trailing newlines. +""" + +from typing import Iterable, List, Optional + + +def emit_module_gate(trace_default: bool, indent: str) -> List[str]: + """Return the module-scope ``trace`` parameter declaration lines. + + The caller is responsible for ensuring ``use, intrinsic :: iso_fortran_env, + only: error_unit`` is present in the module USE list (see + :func:`ensure_error_unit_use`). This helper only emits the + ``logical, parameter`` line so it can be placed alongside other + module-scope parameters. + + Parameters + ---------- + trace_default : bool + ``True`` -> emit ``logical, parameter :: trace = .true.``; + ``False`` -> ``.false.``. + indent : str + Leading whitespace for each emitted line. + + Returns + ------- + list of str + One line, no trailing newline. + + Examples + -------- + >>> emit_module_gate(False, ' ') + [' logical, parameter :: trace = .false.'] + >>> emit_module_gate(True, ' ') + [' logical, parameter :: trace = .true.'] + """ + value = '.true.' if trace_default else '.false.' + return ['{}logical, parameter :: trace = {}'.format(indent, value)] + + +def ensure_error_unit_use(use_lines: List[str], indent: str) -> List[str]: + """Insert an ``iso_fortran_env`` USE for ``error_unit`` if not present. + + The trace block writes to ``error_unit``, which must be visible inside + each cap module. Callers that already build a USE list pass it in; + this helper appends the required line iff no existing line references + ``error_unit`` (matched as a whole word). + + Parameters + ---------- + use_lines : list of str + Existing module-level USE lines (modified in place and returned). + indent : str + Leading whitespace for the emitted line. + + Returns + ------- + list of str + The same list, possibly with one extra line appended. + + Examples + -------- + >>> ensure_error_unit_use([], ' ') + [' use, intrinsic :: iso_fortran_env, only: error_unit'] + >>> ensure_error_unit_use([' use foo, only: bar'], ' ') + [' use foo, only: bar', ' use, intrinsic :: iso_fortran_env, only: error_unit'] + + Idempotent when ``error_unit`` is already there: + + >>> ensure_error_unit_use( + ... [' use, intrinsic :: iso_fortran_env, only: error_unit'], ' ' + ... ) + [' use, intrinsic :: iso_fortran_env, only: error_unit'] + """ + for line in use_lines: + # Whole-word match so we don't false-positive on a substring. + tokens = line.replace(',', ' ').replace(':', ' ').split() + if 'error_unit' in tokens: + return use_lines + use_lines.append( + '{}use, intrinsic :: iso_fortran_env, only: error_unit'.format(indent) + ) + return use_lines + + +def emit_trace_block( + sub_name: str, + ctrl_entries: Iterable, + indent: str, + instance_local: Optional[str] = None, + extra_in_names: Optional[Iterable[str]] = None, +) -> List[str]: + """Return the gated trace ``write`` lines for one cap subroutine. + + The trace lists every control dummy whose intent is ``in`` or + ``inout`` (i.e. the dummies that a strict compiler would otherwise + flag as unused). ``intent(out)`` dummies (``ccpp_error_code`` / + ``ccpp_error_message``) are excluded so the write can sit at the + very first body line, before any initialisation, with no risk of + reading uninitialised storage. + + Parameters + ---------- + sub_name : str + Fully-qualified Fortran name of the subroutine being traced; + emitted verbatim into the trace string for grep-ability. + ctrl_entries : iterable of HostVarEntry + Control-variable dummies in the subroutine signature (any order; + the helper preserves it). + indent : str + Leading whitespace for the ``if`` and continuation lines. + instance_local : str, optional + Local name of ``instance_number`` when it is not already in + *ctrl_entries* (some lifecycle routines pass it separately). + extra_in_names : iterable of str, optional + Extra dummy local names to include unconditionally (treated as + ``intent(in)`` integers). Used for routines whose signatures + carry non-control intent(in) dummies that the compiler may also + flag as unused (e.g. ``number_of_instances`` in state_alloc). + + Returns + ------- + list of str + Lines forming a single ``if (trace) write(error_unit, *) ...`` + continuation block. Empty when there is nothing to print. + """ + from generator.group_cap import _ctrl_intent_for # avoid import cycle at module load + + # Build the (local_name, is_character) list in signature order. + items: List = [] + seen = set() + for entry in ctrl_entries: + if _ctrl_intent_for(entry.standard_name) == 'out': + continue + if entry.local_name in seen: + continue + seen.add(entry.local_name) + is_char = entry.type.strip().lower() == 'character' + items.append((entry.local_name, is_char)) + if instance_local and instance_local not in seen: + seen.add(instance_local) + items.append((instance_local, False)) + if extra_in_names: + for name in extra_in_names: + if name not in seen: + seen.add(name) + items.append((name, False)) + + if not items: + return [] + + lines: List[str] = [] + lines.append('{}if (trace) write(error_unit, *) &'.format(indent)) + lines.append("{} 'CCPP TRACE {}:', &".format(indent, sub_name)) + for i, (local_name, is_char) in enumerate(items): + expr = 'trim({})'.format(local_name) if is_char else local_name + sep = ', &' if i < len(items) - 1 else '' + lines.append( + "{} ' {}=', {}{}".format(indent, local_name, expr, sep) + ) + return lines diff --git a/end-to-end-tests.sh b/end-to-end-tests.sh index c930887f..c9bc074e 100755 --- a/end-to-end-tests.sh +++ b/end-to-end-tests.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +mkdir -p build rm -fr build/* cd build cmake ../end-to-end-tests diff --git a/end-to-end-tests/advection/CMakeLists.txt b/end-to-end-tests/advection/CMakeLists.txt index dbc86bae..0a3132ab 100644 --- a/end-to-end-tests/advection/CMakeLists.txt +++ b/end-to-end-tests/advection/CMakeLists.txt @@ -27,8 +27,12 @@ ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} METADATA_FILES ${HOST_METADATA_FILES}) +# Enable trace output in auto-generated caps +set(CCPP_TRACE ON) + # Run ccpp_capgen -ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} +ccpp_capgen(TRACE ${CCPP_TRACE} + VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} SCHEMEFILES ${SCHEME_METADATA_FILES} SUITES ${SUITE_FILES} diff --git a/end-to-end-tests/cmake/ccpp_capgen.cmake b/end-to-end-tests/cmake/ccpp_capgen.cmake index a7a1e719..9942101f 100644 --- a/end-to-end-tests/cmake/ccpp_capgen.cmake +++ b/end-to-end-tests/cmake/ccpp_capgen.cmake @@ -55,15 +55,15 @@ endfunction() # CMake wrapper for ccpp_capgen_ng.py # -# CAPGEN_EXPECT_THROW_ERROR - ON/OFF (Default: OFF) - Scans ccpp_capgen.py log for error string and errors if not found. -# HOST_NAME - String name of host -# OUTPUT_ROOT - String path to put generated caps -# VERBOSITY - Number of --verbose flags to pass to capgen -# HOSTFILES - CMake list of host metadata filenames -# SCHEMEFILES - CMake list of scheme metadata files -# SUITES - CMake list of suite xml files +# TRACE - ON/OFF (Default: OFF) - Add --trace flag to capgen call +# HOST_NAME - String name of host +# OUTPUT_ROOT - String path to put generated caps +# VERBOSITY - Number of --verbose flags to pass to capgen +# HOSTFILES - CMake list of host metadata filenames +# SCHEMEFILES - CMake list of scheme metadata files +# SUITES - CMake list of suite xml files function(ccpp_capgen) - set(optionalArgs CAPGEN_EXPECT_THROW_ERROR) + set(optionalArgs CAPGEN_EXPECT_THROW_ERROR TRACE) set(oneValueArgs HOST_NAME OUTPUT_ROOT VERBOSITY KIND_SPECS) set(multi_value_keywords HOSTFILES SCHEMEFILES SUITES) @@ -124,6 +124,10 @@ function(ccpp_capgen) list(APPEND CCPP_CAPGEN_CMD_LIST ${KIND_SPEC_PARAMS}) endif() + if(arg_TRACE) + list(APPEND CCPP_CAPGEN_CMD_LIST "--trace") + endif() + message(STATUS "Running ccpp_capgen.py from ${CMAKE_CURRENT_SOURCE_DIR}") # Unset CAPGEN_OUT to prevent incorrect output on subsequent ccpp_capgen(...) calls @@ -137,22 +141,6 @@ function(ccpp_capgen) message(STATUS "ccpp-capgen stdout: ${CAPGEN_OUT}") - if(arg_CAPGEN_EXPECT_THROW_ERROR) - # Determine if the process succeeded but had an expected string in the process log. - string(FIND "${CAPGEN_OUT}" "Variables of type ccpp_constituent_properties_t only allowed in register phase" ERROR_INDEX) - - if (ERROR_INDEX GREATER -1) - message(STATUS "Capgen build produces expected error message.") - else() - message(FATAL_ERROR "CCPP cap generation did not generate expected error. Expected 'Variables of type constituent_properties_t only allowed in register phase.") - endif() - else() - if(RES EQUAL 0) - message(STATUS "ccpp-capgen completed successfully") - else() - message(FATAL_ERROR "CCPP cap generation FAILED: result = ${RES}") - endif() - endif() endfunction() diff --git a/unit-tests/test_static_api.py b/unit-tests/test_static_api.py index 22ae49e1..a57533f8 100644 --- a/unit-tests/test_static_api.py +++ b/unit-tests/test_static_api.py @@ -1200,19 +1200,20 @@ def test_suite_host_data_stub(self): self.assertIn('allocate(variable_list(0))', text) self.assertNotIn('select case (trim(suite_name))', text) - def test_module_imports_error_unit_only_when_stubbed(self): - # With stub on: iso_fortran_env appears for error_unit. - text_on = '\n'.join(_generate_static_api( - ['test_simple'], [self.suite_resolution], self.hd, - no_host_introspection=True, - )) - self.assertIn('use iso_fortran_env, only: error_unit', text_on) - # With stub off: no such import. - text_off = '\n'.join(_generate_static_api( - ['test_simple'], [self.suite_resolution], self.hd, - no_host_introspection=False, - )) - self.assertNotIn('use iso_fortran_env', text_off) + def test_module_imports_error_unit_unconditionally(self): + # error_unit is always imported because every cap subroutine + # emits a gated ``if (trace) write(error_unit, *) ...`` line. + # Stub-on and stub-off both include the same USE. + for stub in (True, False): + text = '\n'.join(_generate_static_api( + ['test_simple'], [self.suite_resolution], self.hd, + no_host_introspection=stub, + )) + self.assertIn( + 'use, intrinsic :: iso_fortran_env, only: error_unit', + text, + msg='no_host_introspection={}'.format(stub), + ) def test_public_declarations_unchanged_when_stubbed(self): # All five introspection routines remain public — callers must @@ -1256,7 +1257,69 @@ def test_write_static_api_passes_flag_through(self): text = fh.read() self.assertIn('ccpp_physics_suite_variables: ' + self._DISABLED_MSG, text) - self.assertIn('use iso_fortran_env, only: error_unit', text) + self.assertIn( + 'use, intrinsic :: iso_fortran_env, only: error_unit', text, + ) + + +class TestTraceEmission(unittest.TestCase): + """The generated static API always carries a module-level ``trace`` + parameter (default .false.) and a gated ``write(error_unit,*)`` in + every cap subroutine that has at least one intent(in) control dummy. + ``--trace`` flips the parameter default to .true. + """ + + def setUp(self): + self.hd = _load_full_host_dict() + self.suite_resolution = _resolve() + + def test_module_gate_default_off(self): + text = '\n'.join(_generate_static_api( + ['test_simple'], [self.suite_resolution], self.hd, + )) + self.assertIn('logical, parameter :: trace = .false.', text) + self.assertNotIn('logical, parameter :: trace = .true.', text) + + def test_module_gate_default_on(self): + text = '\n'.join(_generate_static_api( + ['test_simple'], [self.suite_resolution], self.hd, + trace=True, + )) + self.assertIn('logical, parameter :: trace = .true.', text) + self.assertNotIn('logical, parameter :: trace = .false.', text) + + def test_trace_block_present_in_physics_phases(self): + text = '\n'.join(_generate_static_api( + ['test_simple'], [self.suite_resolution], self.hd, + )) + # Every ccpp_physics_ dispatch has a gated write. + for phase in ('init', 'timestep_init', 'run', + 'timestep_final', 'final'): + self.assertIn( + "'CCPP TRACE ccpp_physics_{}:'".format(phase), + text, + msg='trace string missing for phase {}'.format(phase), + ) + + def test_trace_block_present_in_lifecycle_routines(self): + text = '\n'.join(_generate_static_api( + ['test_simple'], [self.suite_resolution], self.hd, + )) + for sub in ('ccpp_register', 'ccpp_init', 'ccpp_final'): + self.assertIn( + "'CCPP TRACE {}:'".format(sub), text, + msg='trace string missing for {}'.format(sub), + ) + + def test_write_static_api_threads_trace_flag(self): + with tempfile.TemporaryDirectory() as tmpdir: + out_path = write_static_api( + ['test_simple'], [self.suite_resolution], tmpdir, self.hd, + trace=True, + ) + with open(out_path) as fh: + text = fh.read() + self.assertIn('logical, parameter :: trace = .true.', text) def load_tests(loader, tests, ignore): diff --git a/unit-tests/test_suite_cap.py b/unit-tests/test_suite_cap.py index a40e9fc7..7d6a96c1 100644 --- a/unit-tests/test_suite_cap.py +++ b/unit-tests/test_suite_cap.py @@ -403,6 +403,61 @@ def test_no_constituent_prop_ptr_type_import(self): self.assertNotIn('ccpp_constituent_prop_ptr_t', self.text) +class TestTraceEmission(unittest.TestCase): + """The generated suite cap always carries a module-level ``trace`` + parameter (default .false.) and a gated ``write(error_unit,*)`` in + every physics-dispatch subroutine; ``trace=True`` flips the default. + """ + + def setUp(self): + self.suite_resolution, self.store = _resolve() + self.hd = _load_full_host_dict() + + def test_module_gate_default_off(self): + text = '\n'.join(_generate_suite_cap( + 'test_simple', self.suite_resolution, self.store, self.hd, + )) + self.assertIn('logical, parameter :: trace = .false.', text) + + def test_module_gate_default_on(self): + text = '\n'.join(_generate_suite_cap( + 'test_simple', self.suite_resolution, self.store, self.hd, + trace=True, + )) + self.assertIn('logical, parameter :: trace = .true.', text) + self.assertNotIn('logical, parameter :: trace = .false.', text) + + def test_error_unit_use_unconditional(self): + text = '\n'.join(_generate_suite_cap( + 'test_simple', self.suite_resolution, self.store, self.hd, + )) + self.assertIn( + 'use, intrinsic :: iso_fortran_env, only: error_unit', text, + ) + + def test_trace_block_present_in_physics_phases(self): + text = '\n'.join(_generate_suite_cap( + 'test_simple', self.suite_resolution, self.store, self.hd, + )) + for phase in ('init', 'timestep_init', 'run', + 'timestep_final', 'final'): + self.assertIn( + "'CCPP TRACE test_simple_physics_{}:'".format(phase), + text, + msg='trace string missing for phase {}'.format(phase), + ) + + def test_write_suite_cap_threads_trace_flag(self): + with tempfile.TemporaryDirectory() as tmpdir: + out_path = write_suite_cap( + 'test_simple', self.suite_resolution, self.store, tmpdir, + self.hd, trace=True, + ) + with open(out_path) as fh: + text = fh.read() + self.assertIn('logical, parameter :: trace = .true.', text) + + def load_tests(loader, tests, ignore): import generator.suite_cap as subcycle tests.addTests(doctest.DocTestSuite(subcycle)) diff --git a/unit-tests/test_trace.py b/unit-tests/test_trace.py new file mode 100644 index 00000000..28023317 --- /dev/null +++ b/unit-tests/test_trace.py @@ -0,0 +1,179 @@ +"""Unit tests for generator.trace.""" + +import doctest +import unittest + +from generator.trace import ( + emit_module_gate, + emit_trace_block, + ensure_error_unit_use, +) + + +class _FakeEntry: + """Minimal HostVarEntry stand-in for the trace helper.""" + + def __init__(self, standard_name, local_name, type_): + self.standard_name = standard_name + self.local_name = local_name + self.type = type_ + + +class TestEmitModuleGate(unittest.TestCase): + + def test_default_off(self): + self.assertEqual( + emit_module_gate(False, ' '), + [' logical, parameter :: trace = .false.'], + ) + + def test_default_on(self): + self.assertEqual( + emit_module_gate(True, ' '), + [' logical, parameter :: trace = .true.'], + ) + + def test_indent_preserved(self): + self.assertEqual( + emit_module_gate(False, ' '), + [' logical, parameter :: trace = .false.'], + ) + + +class TestEnsureErrorUnitUse(unittest.TestCase): + + def test_appends_when_absent(self): + out = ensure_error_unit_use([], ' ') + self.assertEqual( + out, + [' use, intrinsic :: iso_fortran_env, only: error_unit'], + ) + + def test_idempotent(self): + existing = [' use, intrinsic :: iso_fortran_env, only: error_unit'] + out = ensure_error_unit_use(list(existing), ' ') + self.assertEqual(out, existing) + + def test_appends_after_other_uses(self): + out = ensure_error_unit_use([' use foo, only: bar'], ' ') + self.assertEqual(out, [ + ' use foo, only: bar', + ' use, intrinsic :: iso_fortran_env, only: error_unit', + ]) + + def test_no_substring_false_positive(self): + # ``my_error_unit_proxy`` must not be matched as ``error_unit``. + out = ensure_error_unit_use([' use foo, only: my_error_unit_proxy'], ' ') + self.assertEqual(out, [ + ' use foo, only: my_error_unit_proxy', + ' use, intrinsic :: iso_fortran_env, only: error_unit', + ]) + + +class TestEmitTraceBlock(unittest.TestCase): + + def _ctrl_in(self): + return [ + _FakeEntry('horizontal_loop_begin', 'lb', 'integer'), + _FakeEntry('horizontal_loop_end', 'ub', 'integer'), + _FakeEntry('thread_number', 'thread_num', 'integer'), + ] + + def _ctrl_out(self): + return [ + _FakeEntry('ccpp_error_code', 'errflg', 'integer'), + _FakeEntry('ccpp_error_message', 'errmsg', 'character'), + ] + + def test_emits_for_intent_in_dummies(self): + out = emit_trace_block('my_sub', self._ctrl_in(), ' ') + self.assertEqual(out, [ + ' if (trace) write(error_unit, *) &', + " 'CCPP TRACE my_sub:', &", + " ' lb=', lb, &", + " ' ub=', ub, &", + " ' thread_num=', thread_num", + ]) + + def test_filters_intent_out(self): + # An intent(out) entry should not appear in the trace. + out = emit_trace_block( + 'my_sub', self._ctrl_in() + self._ctrl_out(), ' ', + ) + joined = '\n'.join(out) + self.assertNotIn('errflg', joined) + self.assertNotIn('errmsg', joined) + self.assertIn('lb', joined) + self.assertIn('thread_num', joined) + + def test_character_wrapped_in_trim(self): + entries = [_FakeEntry('suite_name', 'suite_name', 'character')] + out = emit_trace_block('my_sub', entries, ' ') + self.assertEqual(out, [ + ' if (trace) write(error_unit, *) &', + " 'CCPP TRACE my_sub:', &", + " ' suite_name=', trim(suite_name)", + ]) + + def test_empty_when_only_intent_out(self): + out = emit_trace_block('my_sub', self._ctrl_out(), ' ') + self.assertEqual(out, []) + + def test_empty_when_no_entries(self): + out = emit_trace_block('my_sub', [], ' ') + self.assertEqual(out, []) + + def test_instance_local_appended(self): + out = emit_trace_block( + 'my_sub', self._ctrl_in(), ' ', instance_local='inst_num', + ) + joined = '\n'.join(out) + self.assertIn("' inst_num=', inst_num", joined) + + def test_instance_local_not_duplicated_when_already_in_entries(self): + entries = self._ctrl_in() + [ + _FakeEntry('instance_number', 'inst_num', 'integer'), + ] + out = emit_trace_block( + 'my_sub', entries, ' ', instance_local='inst_num', + ) + joined = '\n'.join(out) + # Should appear exactly once. + self.assertEqual(joined.count(" inst_num=', inst_num"), 1) + + def test_signature_order_preserved(self): + entries = [ + _FakeEntry('a', 'first', 'integer'), + _FakeEntry('b', 'second', 'integer'), + _FakeEntry('c', 'third', 'integer'), + ] + out = emit_trace_block('my_sub', entries, ' ') + first = out.index(" ' first=', first, &") + second = out.index(" ' second=', second, &") + third = out.index(" ' third=', third") + self.assertLess(first, second) + self.assertLess(second, third) + + def test_continuation_only_after_last_var_omitted(self): + entries = [ + _FakeEntry('a', 'one', 'integer'), + _FakeEntry('b', 'two', 'integer'), + ] + out = emit_trace_block('my_sub', entries, ' ') + # Every line but the last must end with ``&``. + for line in out[:-1]: + self.assertTrue( + line.rstrip().endswith('&'), + msg='line missing continuation: {!r}'.format(line), + ) + self.assertFalse(out[-1].rstrip().endswith('&')) + + +def load_tests(loader, tests, ignore): + import generator.trace as t + tests.addTests(doctest.DocTestSuite(t)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) From a35632aafed7f1dd5891890d45060393a61aeb5e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 May 2026 08:10:01 -0600 Subject: [PATCH 18/74] Shorten Fortran names of auto-generated subroutines/functions --- capgen-ng/generator/group_cap.py | 33 ++++++++++++----- capgen-ng/generator/suite_cap.py | 27 +++++++------- capgen-ng/generator/suite_data.py | 11 +++--- end-to-end-tests/cmake/ccpp_capgen.cmake | 2 +- unit-tests/test_integration.py | 46 ++++++++++++------------ unit-tests/test_suite_cap.py | 14 ++++---- unit-tests/test_suite_resolver.py | 36 +++++++++---------- 7 files changed, 96 insertions(+), 73 deletions(-) diff --git a/capgen-ng/generator/group_cap.py b/capgen-ng/generator/group_cap.py index dc87d567..a9e30e74 100644 --- a/capgen-ng/generator/group_cap.py +++ b/capgen-ng/generator/group_cap.py @@ -799,7 +799,12 @@ def _generate_phase_subroutine( Returns a list of Fortran source lines (no trailing newlines). """ - sub_name = 'ccpp_{}_{}_{}'.format(suite_name, group_name, phase) + # Short Fortran symbol; module name already namespaces ``_`` + # as ``ccpp___cap_mp__`` at link time, keeping + # the mangled global name under Intel's ~90-char threshold. The long + # form ``__`` is kept in ``sub_label`` below for + # trace strings and error messages (string literals have no length cap). + sub_name = '{}_{}'.format(group_name, phase) lines: List[str] = [] # ---- subroutine declaration ------------------------------------------ @@ -879,13 +884,15 @@ def _generate_phase_subroutine( inst_idx = _instance_idx(host_dict) errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') + # Long form used as the trace-message label and in runtime error + # strings so grep against logs still finds the suite + group context. sub_label = '{}_{}_{}'.format(suite_name, group_name, phase) # ---- trace block (always emitted; gated by the module ``trace`` # parameter so the I/O is dead-code-eliminated when trace=.false.). # Placed before errmsg/errflg init so the write references no # intent(out) dummy and fires even when a state guard then bails. - trace_lines = emit_trace_block(sub_name, ctrl_entries, call_indent) + trace_lines = emit_trace_block(sub_label, ctrl_entries, call_indent) if trace_lines: lines.extend(trace_lines) lines.append('') @@ -998,7 +1005,9 @@ def _generate_state_alloc(suite_name: str, group_name: str) -> List[str]: avoid clobbering peer-instance state slots. Matches the ``_suite_state_alloc`` pattern. """ - sub_name = 'ccpp_{}_{}_{}'.format(suite_name, group_name, 'state_alloc') + # Short Fortran symbol; the module ``ccpp___cap`` + # already namespaces this routine at link time. + sub_name = '{}_state_alloc'.format(group_name) i1 = _INDENT i2 = _INDENT * 2 lines = [ @@ -1021,8 +1030,13 @@ def _generate_state_alloc(suite_name: str, group_name: str) -> List[str]: def _generate_state_dealloc(suite_name: str, group_name: str) -> List[str]: - """Generate the ``ccpp___state_dealloc`` subroutine.""" - sub_name = 'ccpp_{}_{}_{}'.format(suite_name, group_name, 'state_dealloc') + """Generate the ``_state_dealloc`` subroutine (Fortran symbol). + + The module name ``ccpp___cap`` already namespaces this + routine, so the short Fortran name keeps the mangled global symbol + under Intel's ~90-char limit. + """ + sub_name = '{}_state_dealloc'.format(group_name) i1 = _INDENT i2 = _INDENT * 2 return [ @@ -1062,8 +1076,11 @@ def _generate_group_cap( list of str (without trailing newlines) """ mod_name = 'ccpp_{}_{}_{}'.format(suite_name, group_name, 'cap') - alloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, group_name, 'state_alloc') - dealloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, group_name, 'state_dealloc') + # Short Fortran symbols for the state-management subroutines; the + # module name carries ``ccpp___`` and keeps the mangled + # global symbol (``_mp_``) under Intel's ~90-char limit. + alloc_sub = '{}_state_alloc'.format(group_name) + dealloc_sub = '{}_state_dealloc'.format(group_name) lines: List[str] = [] # ---- module header -------------------------------------------------- @@ -1116,7 +1133,7 @@ def _generate_group_cap( # transitioning through every phase, even when a group has no scheme # routine for a particular phase. for phase in _GROUP_PHASE_ORDER: - sub_name = 'ccpp_{}_{}_{}'.format(suite_name, group_name, phase) + sub_name = '{}_{}'.format(group_name, phase) lines.append('{}public :: {}'.format(_INDENT, sub_name)) # Public state management subroutines. diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 21c944ba..05b68161 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -495,7 +495,7 @@ def _init_lines( ) if suite_res.suite_vars: data_mod = 'ccpp_{}_data'.format(suite_name) - init_fields = 'ccpp_{}_suite_data_init_fields'.format(suite_name) + init_fields = 'suite_data_init_fields' extra_uses.setdefault(data_mod, set()).add(init_fields) if suite_res.suite_init_call is not None: _add_call_uses(extra_uses, suite_res.suite_init_call) @@ -548,7 +548,7 @@ def _init_lines( # Group state allocators (idempotent). for resolved_group in suite_res.groups: - alloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'state_alloc') + alloc_sub = '{}_state_alloc'.format(resolved_group.group_name) lines.append('{}call {}({}, {}, {})'.format( i2, alloc_sub, ninstances_arg, errmsg_local, errflg_local )) @@ -556,7 +556,7 @@ def _init_lines( # Allocate inner suite-data allocatable fields for this instance. if suite_res.suite_vars: - init_fields = 'ccpp_{}_suite_data_init_fields'.format(suite_name) + init_fields = 'suite_data_init_fields' lines.append('{}call {}({}, {}, {})'.format( i2, init_fields, inst_idx, errmsg_local, errflg_local )) @@ -629,7 +629,7 @@ def _final_lines( final_uses: Dict[str, Set[str]] = {} if suite_res.suite_vars: data_mod = 'ccpp_{}_data'.format(suite_name) - final_fields = 'ccpp_{}_suite_data_final_fields'.format(suite_name) + final_fields = 'suite_data_final_fields' final_uses.setdefault(data_mod, set()).add(final_fields) # If we registered constituents, the per-suite buffer (owned by @@ -679,7 +679,7 @@ def _final_lines( # Deallocate inner suite-data fields if this instance was past REGISTERED. if suite_res.suite_vars: - final_fields = 'ccpp_{}_suite_data_final_fields'.format(suite_name) + final_fields = 'suite_data_final_fields' lines.append( '{}if (ccpp_suite_state({}) == CCPP_SUITE_FRAMEWORK_INITIALIZED) then'.format( i2, inst_idx @@ -707,7 +707,7 @@ def _final_lines( '{}if (all(ccpp_suite_state == CCPP_SUITE_UNREGISTERED)) then'.format(i2) ) for resolved_group in suite_res.groups: - dealloc_sub = 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'state_dealloc') + dealloc_sub = '{}_state_dealloc'.format(resolved_group.group_name) lines.append('{} call {}({}, {})'.format( i2, dealloc_sub, errmsg_local, errflg_local )) @@ -841,7 +841,7 @@ def _physics_dispatch_lines( def _emit_group_call(resolved_group, indent): # Group phase subroutines are always emitted (so the per-group state # machine transitions through every phase), so we always dispatch. - cap_sub = 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, phase) + cap_sub = '{}_{}'.format(resolved_group.group_name, phase) if group_ctrl_local: lines.append('{}call {}( &'.format(indent, cap_sub)) for idx, lname in enumerate(group_ctrl_local): @@ -900,7 +900,7 @@ def _suite_state_alloc_lines( any suite-owned scalar dimensions. """ sub_name = '{}_suite_state_alloc'.format(suite_name) - data_alloc = 'ccpp_{}_suite_data_alloc'.format(suite_name) + data_alloc = 'suite_data_alloc' data_mod = 'ccpp_{}_data'.format(suite_name) i1 = _INDENT i2 = _INDENT * 2 @@ -940,7 +940,7 @@ def _suite_state_dealloc_lines( ) -> List[str]: """Generate the ``_suite_state_dealloc`` subroutine.""" sub_name = '{}_suite_state_dealloc'.format(suite_name) - data_dealloc = 'ccpp_{}_suite_data_dealloc'.format(suite_name) + data_dealloc = 'suite_data_dealloc' data_mod = 'ccpp_{}_data'.format(suite_name) i1 = _INDENT i2 = _INDENT * 2 @@ -1011,15 +1011,18 @@ def _generate_suite_cap( lines.append('') # USE statements: one per group cap (all phase + state subroutines). + # Group cap subroutine names are short (``_`` etc.) so + # the mangled global ``_mp_`` stays under Intel's ~90-char + # limit even for long suite/group name combinations. use_lines: List[str] = [] for resolved_group in suite_res.groups: group_cap_mod = 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'cap') syms_list = [ - 'ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, p) + '{}_{}'.format(resolved_group.group_name, p) for p in _PHYSICS_PHASES ] - syms_list.append('ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'state_alloc')) - syms_list.append('ccpp_{}_{}_{}'.format(suite_name, resolved_group.group_name, 'state_dealloc')) + syms_list.append('{}_state_alloc'.format(resolved_group.group_name)) + syms_list.append('{}_state_dealloc'.format(resolved_group.group_name)) use_lines.append('{}use {}, only: {}'.format( _INDENT, group_cap_mod, ', '.join(syms_list) )) diff --git a/capgen-ng/generator/suite_data.py b/capgen-ng/generator/suite_data.py index e0e2a0ec..31187555 100644 --- a/capgen-ng/generator/suite_data.py +++ b/capgen-ng/generator/suite_data.py @@ -171,10 +171,13 @@ def _generate_suite_data( """ mod_name = 'ccpp_{}_data'.format(suite_name) type_name = 'ccpp_{}_data_t'.format(suite_name) - alloc_sub = 'ccpp_{}_suite_data_alloc'.format(suite_name) - dealloc_sub = 'ccpp_{}_suite_data_dealloc'.format(suite_name) - init_fields_sub = 'ccpp_{}_suite_data_init_fields'.format(suite_name) - final_fields_sub = 'ccpp_{}_suite_data_final_fields'.format(suite_name) + # Short Fortran symbols; the module ``ccpp__data`` already + # namespaces these routines at link time, keeping the mangled global + # symbol ``_mp_`` well under Intel's ~90-char limit. + alloc_sub = 'suite_data_alloc' + dealloc_sub = 'suite_data_dealloc' + init_fields_sub = 'suite_data_init_fields' + final_fields_sub = 'suite_data_final_fields' i1 = _INDENT i2 = _INDENT * 2 lines: List[str] = [] diff --git a/end-to-end-tests/cmake/ccpp_capgen.cmake b/end-to-end-tests/cmake/ccpp_capgen.cmake index 9942101f..dc5b9918 100644 --- a/end-to-end-tests/cmake/ccpp_capgen.cmake +++ b/end-to-end-tests/cmake/ccpp_capgen.cmake @@ -63,7 +63,7 @@ endfunction() # SCHEMEFILES - CMake list of scheme metadata files # SUITES - CMake list of suite xml files function(ccpp_capgen) - set(optionalArgs CAPGEN_EXPECT_THROW_ERROR TRACE) + set(optionalArgs TRACE) set(oneValueArgs HOST_NAME OUTPUT_ROOT VERBOSITY KIND_SPECS) set(multi_value_keywords HOSTFILES SCHEMEFILES SUITES) diff --git a/unit-tests/test_integration.py b/unit-tests/test_integration.py index 4642eab1..3c13d123 100644 --- a/unit-tests/test_integration.py +++ b/unit-tests/test_integration.py @@ -486,10 +486,10 @@ def test_init_guard(self): self.assertIn('ccpp_group_state', self.text) def test_state_alloc_subroutine(self): - self.assertIn('subroutine ccpp_test_simple_physics_state_alloc', self.text) + self.assertIn('subroutine physics_state_alloc', self.text) def test_state_dealloc_subroutine(self): - self.assertIn('subroutine ccpp_test_simple_physics_state_dealloc', self.text) + self.assertIn('subroutine physics_state_dealloc', self.text) def test_ends_with_newline(self): with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: @@ -725,7 +725,7 @@ def test_suite_init_passes_ninstances_to_group_state_alloc(self): with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: text = fh.read() self.assertIn( - 'call ccpp_test_simple_physics_state_alloc(ninstances, errmsg, errflg)', + 'call physics_state_alloc(ninstances, errmsg, errflg)', text, ) @@ -733,7 +733,7 @@ def test_state_alloc_takes_number_of_instances_arg(self): with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: text = fh.read() self.assertIn( - 'subroutine ccpp_test_simple_physics_state_alloc(number_of_instances, errmsg, errflg)', + 'subroutine physics_state_alloc(number_of_instances, errmsg, errflg)', text, ) @@ -767,7 +767,7 @@ def test_state_guard_uses_inst_num(self): def test_group_init_has_inst_num_arg(self): with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: text = fh.read() - init_sub = text.split('subroutine ccpp_test_simple_physics_init')[1] + init_sub = text.split('subroutine physics_init')[1] init_sub = init_sub.split('end subroutine')[0] self.assertIn('inst_num', init_sub) @@ -853,14 +853,14 @@ def test_group_state_alloc_passes_literal_one(self): with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: text = fh.read() self.assertIn( - 'call ccpp_test_simple_physics_state_alloc(1, errmsg, errflg)', + 'call physics_state_alloc(1, errmsg, errflg)', text, ) def test_group_cap_init_omits_inst_num(self): with open(os.path.join(self._tmpdir, 'ccpp_test_simple_physics_cap.F90')) as fh: text = fh.read() - init_sub = text.split('subroutine ccpp_test_simple_physics_init')[1] + init_sub = text.split('subroutine physics_init')[1] init_sub = init_sub.split('end subroutine')[0] self.assertNotIn('inst_num', init_sub) @@ -1596,14 +1596,14 @@ def test_inner_loop_present(self): def test_two_end_do_statements(self): """Each nesting level closes with an ``end do``.""" # Find the run-phase body to scope the check. - run = self.text.split('subroutine ccpp_nested_subcycle_suite_physics_run')[1] + run = self.text.split('subroutine physics_run')[1] run = run.split('end subroutine')[0] self.assertEqual(run.count('end do'), 2) def test_inner_loop_inside_outer(self): """The inner ``do`` line appears AFTER the outer ``do`` line and BEFORE the first ``end do``.""" - run = self.text.split('subroutine ccpp_nested_subcycle_suite_physics_run')[1] + run = self.text.split('subroutine physics_run')[1] run = run.split('end subroutine')[0] outer = run.index('do ccpp_loop_counter = 1, 3') inner = run.index('do ccpp_loop_counter_2 = 1, 2') @@ -1614,7 +1614,7 @@ def test_inner_loop_inside_outer(self): def test_scheme_call_inside_inner_loop(self): """The scheme call is nested inside the inner loop — not in between the loops.""" - run = self.text.split('subroutine ccpp_nested_subcycle_suite_physics_run')[1] + run = self.text.split('subroutine physics_run')[1] run = run.split('end subroutine')[0] inner = run.index('do ccpp_loop_counter_2 = 1, 2') call = run.index('call temp_calc_adjust_run') @@ -1962,7 +1962,7 @@ def test_phase_subroutine_order_canonical(self): '_state_alloc', '_state_dealloc', ] positions = [ - text.index('public :: ccpp_chunked_data_chunked_data_group{}'.format(s)) + text.index('public :: chunked_data_group{}'.format(s)) for s in canonical ] self.assertEqual(positions, sorted(positions)) @@ -2027,12 +2027,12 @@ def test_suite_data_has_allocatable_instance_array(self): self.assertIn('ccpp_suite_data(:)', text) def test_suite_data_alloc_subroutine_exists(self): - self.assertIn('ccpp_interstitial_suite_data_alloc', self._data()) + self.assertIn('suite_data_alloc', self._data()) def test_suite_data_init_fields_uses_host_dims(self): # init_fields now owns the inner allocations; it needs the host dims. text = self._data() - init_fields = text.split('subroutine ccpp_interstitial_suite_data_init_fields')[1].split('end subroutine')[0] + init_fields = text.split('subroutine suite_data_init_fields')[1].split('end subroutine')[0] self.assertIn('use host_phys', init_fields) self.assertIn('ncols', init_fields) self.assertIn('nlev', init_fields) @@ -2045,14 +2045,14 @@ def test_suite_data_init_fields_allocates_field_per_instance(self): # suite-owned dims (e.g. set during _register) can be picked up after # the register phase has run. text = self._data() - init_fields = text.split('subroutine ccpp_interstitial_suite_data_init_fields')[1].split('end subroutine')[0] + init_fields = text.split('subroutine suite_data_init_fields')[1].split('end subroutine')[0] self.assertIn('allocate(ccpp_suite_data(i)%diag_out(ncols, nlev))', init_fields) def test_suite_data_dealloc_subroutine_exists(self): - self.assertIn('ccpp_interstitial_suite_data_dealloc', self._data()) + self.assertIn('suite_data_dealloc', self._data()) def test_suite_data_final_fields_subroutine_exists(self): - self.assertIn('ccpp_interstitial_suite_data_final_fields', self._data()) + self.assertIn('suite_data_final_fields', self._data()) def test_suite_cap_has_suite_state_alloc(self): self.assertIn('interstitial_suite_state_alloc', self._suite_cap()) @@ -2067,13 +2067,13 @@ def test_suite_cap_register_calls_suite_state_alloc(self): def test_suite_state_alloc_calls_suite_data_alloc(self): text = self._suite_cap() alloc_body = text.split('subroutine interstitial_suite_state_alloc')[1].split('end subroutine')[0] - self.assertIn('ccpp_interstitial_suite_data_alloc', alloc_body) + self.assertIn('suite_data_alloc', alloc_body) def test_suite_init_calls_init_fields(self): # _init triggers per-instance inner allocations. text = self._suite_cap() init_body = text.split('subroutine interstitial_init')[1].split('end subroutine')[0] - self.assertIn('ccpp_interstitial_suite_data_init_fields', init_body) + self.assertIn('suite_data_init_fields', init_body) def test_suite_state_alloc_allocates_state_array(self): self.assertIn('allocate(ccpp_suite_state(number_of_instances))', self._suite_cap()) @@ -2094,7 +2094,7 @@ def test_group_cap_accesses_suite_data_with_instance(self): def test_group_run_has_inst_num_dummy_arg(self): """Gap 3: instance_number must be a dummy arg when suite vars are referenced.""" text = self._group_cap() - run_sub = text.split('subroutine ccpp_interstitial_diag_group_run')[1] + run_sub = text.split('subroutine diag_group_run')[1] run_sub = run_sub.split('end subroutine')[0] self.assertIn('inst_num', run_sub) self.assertIn('integer, intent(in)', run_sub) @@ -2105,7 +2105,7 @@ def test_suite_cap_passes_inst_num_to_group_run(self): physics_run = text.split('subroutine interstitial_physics_run')[1] physics_run = physics_run.split('end subroutine')[0] self.assertIn('inst_num', physics_run) - self.assertIn('call ccpp_interstitial_diag_group_run', physics_run) + self.assertIn('call diag_group_run', physics_run) def test_static_api_physics_run_has_inst_num(self): """Static API ccpp_physics_run must include inst_num when groups need it.""" @@ -2248,7 +2248,7 @@ def tearDown(self): def test_timestep_init_entry_guard(self): """timestep_init subroutine must have IN_TIMESTEP entry guard.""" ts_init = self.text.split( - 'subroutine ccpp_opt_arg_opt_arg_group_timestep_init' + 'subroutine opt_arg_group_timestep_init' )[1].split('end subroutine')[0] self.assertIn('CCPP_GROUP_IN_TIMESTEP', ts_init) self.assertIn('return', ts_init) @@ -2256,7 +2256,7 @@ def test_timestep_init_entry_guard(self): def test_timestep_init_sets_in_timestep(self): """timestep_init must set state to IN_TIMESTEP after scheme calls.""" ts_init = self.text.split( - 'subroutine ccpp_opt_arg_opt_arg_group_timestep_init' + 'subroutine opt_arg_group_timestep_init' )[1].split('end subroutine')[0] self.assertIn('ccpp_group_state', ts_init) self.assertIn('CCPP_GROUP_IN_TIMESTEP', ts_init) @@ -2264,7 +2264,7 @@ def test_timestep_init_sets_in_timestep(self): def test_timestep_final_resets_to_initialized(self): """timestep_final must reset state to INITIALIZED.""" ts_final = self.text.split( - 'subroutine ccpp_opt_arg_opt_arg_group_timestep_final' + 'subroutine opt_arg_group_timestep_final' )[1].split('end subroutine')[0] self.assertIn('CCPP_GROUP_INITIALIZED', ts_final) self.assertIn('ccpp_group_state', ts_final) diff --git a/unit-tests/test_suite_cap.py b/unit-tests/test_suite_cap.py index 7d6a96c1..9fa300c8 100644 --- a/unit-tests/test_suite_cap.py +++ b/unit-tests/test_suite_cap.py @@ -178,7 +178,7 @@ def test_final_subroutine(self): def test_init_calls_group_state_alloc(self): # No host_dict passed → single-instance → literal 1 for ninstances. self.assertIn( - 'call ccpp_test_simple_physics_state_alloc(1, errmsg, errflg)', + 'call physics_state_alloc(1, errmsg, errflg)', self.text, ) @@ -191,7 +191,7 @@ def test_register_calls_suite_state_alloc(self): ) def test_final_calls_state_dealloc(self): - self.assertIn('call ccpp_test_simple_physics_state_dealloc(errmsg, errflg)', self.text) + self.assertIn('call physics_state_dealloc(errmsg, errflg)', self.text) class TestPhysicsDispatch(unittest.TestCase): @@ -206,24 +206,24 @@ def test_run_dispatch_present(self): def test_run_dispatches_to_group_cap(self): # No group_name control var → unconditional call, no select case. - self.assertIn('call ccpp_test_simple_physics_run()', self.text) + self.assertIn('call physics_run()', self.text) def test_init_dispatches_to_group_cap(self): - self.assertIn('call ccpp_test_simple_physics_init()', self.text) + self.assertIn('call physics_init()', self.text) def test_final_dispatches_to_group_cap(self): - self.assertIn('call ccpp_test_simple_physics_final()', self.text) + self.assertIn('call physics_final()', self.text) def test_timestep_init_dispatches_to_group_cap(self): # Group phase subroutines are always emitted so the state machine # transitions through every phase, even when no scheme has a routine # for that phase — so the dispatch must always call into the group cap. self.assertIn('subroutine test_simple_physics_timestep_init()', self.text) - self.assertIn('call ccpp_test_simple_physics_timestep_init()', self.text) + self.assertIn('call physics_timestep_init()', self.text) def test_timestep_final_dispatches_to_group_cap(self): self.assertIn('subroutine test_simple_physics_timestep_final()', self.text) - self.assertIn('call ccpp_test_simple_physics_timestep_final()', self.text) + self.assertIn('call physics_timestep_final()', self.text) def test_no_select_case_without_group_name_ctrl(self): # No group_name control var in test setup → no select case dispatch. diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index b85732cc..308a7819 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -1956,7 +1956,7 @@ def test_implicit_none_private(self): def test_public_subroutines(self): lines = self._resolve_and_generate() text = '\n'.join(lines) - self.assertIn('public :: ccpp_test_simple_physics_run', text) + self.assertIn('public :: physics_run', text) def test_contains_block(self): lines = self._resolve_and_generate() @@ -1965,8 +1965,8 @@ def test_contains_block(self): def test_run_subroutine(self): lines = self._resolve_and_generate() text = '\n'.join(lines) - self.assertIn('subroutine ccpp_test_simple_physics_run', text) - self.assertIn('end subroutine ccpp_test_simple_physics_run', text) + self.assertIn('subroutine physics_run', text) + self.assertIn('end subroutine physics_run', text) def test_scheme_call_present(self): lines = self._resolve_and_generate() @@ -1990,7 +1990,7 @@ def test_errflg_check(self): def test_init_subroutine(self): lines = self._resolve_and_generate() text = '\n'.join(lines) - self.assertIn('subroutine ccpp_test_simple_physics_init', text) + self.assertIn('subroutine physics_init', text) self.assertIn('call temp_calc_adjust_init', text) def test_write_group_cap(self): @@ -2342,7 +2342,7 @@ def test_scheme_call_inside_loop(self): def test_init_not_in_do_loop(self): """Init phase is flat — no do loop.""" - self.assertNotIn('do ccpp_loop_counter', self.text.split('subroutine ccpp_test_subcycle_physics_init')[1].split('end subroutine')[0]) + self.assertNotIn('do ccpp_loop_counter', self.text.split('subroutine physics_init')[1].split('end subroutine')[0]) ######################################################################## @@ -2369,10 +2369,10 @@ def test_state_array_declared(self): self.assertIn('integer, private, allocatable :: ccpp_group_state(:)', self.text) def test_state_alloc_public(self): - self.assertIn('public :: ccpp_test_simple_physics_state_alloc', self.text) + self.assertIn('public :: physics_state_alloc', self.text) def test_state_dealloc_public(self): - self.assertIn('public :: ccpp_test_simple_physics_state_dealloc', self.text) + self.assertIn('public :: physics_state_dealloc', self.text) def test_init_idempotent_skip(self): # init returns silently when already INITIALIZED. @@ -2384,7 +2384,7 @@ def test_init_idempotent_skip(self): def test_init_errors_on_invalid_state(self): # init must error if the state is anything other than UNINITIALIZED # or INITIALIZED (idempotent skip). - init_sub = self.text.split('subroutine ccpp_test_simple_physics_init')[1] + init_sub = self.text.split('subroutine physics_init')[1] init_sub = init_sub.split('end subroutine')[0] self.assertIn( 'ccpp_group_state(inst_num) /= CCPP_GROUP_UNINITIALIZED', init_sub @@ -2399,7 +2399,7 @@ def test_final_resets_state(self): def test_run_guards_on_in_timestep(self): # run requires IN_TIMESTEP; otherwise sets errflg and returns. - run_sub = self.text.split('subroutine ccpp_test_simple_physics_run')[1] + run_sub = self.text.split('subroutine physics_run')[1] run_sub = run_sub.split('end subroutine')[0] self.assertIn( 'ccpp_group_state(inst_num) /= CCPP_GROUP_IN_TIMESTEP', run_sub @@ -2409,7 +2409,7 @@ def test_run_guards_on_in_timestep(self): def test_state_alloc_subroutine(self): # state_alloc always takes number_of_instances as explicit arg. self.assertIn( - 'subroutine ccpp_test_simple_physics_state_alloc(number_of_instances, errmsg, errflg)', + 'subroutine physics_state_alloc(number_of_instances, errmsg, errflg)', self.text, ) self.assertIn('allocate(ccpp_group_state(number_of_instances))', self.text) @@ -2421,17 +2421,17 @@ def test_ninstances_not_used_in_group_cap(self): self.assertNotIn('ninstances', preamble) def test_state_dealloc_subroutine(self): - self.assertIn('subroutine ccpp_test_simple_physics_state_dealloc(errmsg, errflg)', self.text) + self.assertIn('subroutine physics_state_dealloc(errmsg, errflg)', self.text) self.assertIn('if (allocated(ccpp_group_state)) deallocate(ccpp_group_state)', self.text) def test_inst_num_in_init_args(self): # inst_num (the local name for instance_number) must be a dummy arg of init. - init_sub = self.text.split('subroutine ccpp_test_simple_physics_init')[1] + init_sub = self.text.split('subroutine physics_init')[1] init_sub = init_sub.split('end subroutine')[0] self.assertIn('inst_num', init_sub) def test_inst_num_in_final_args(self): - final_sub = self.text.split('subroutine ccpp_test_simple_physics_final')[1] + final_sub = self.text.split('subroutine physics_final')[1] final_sub = final_sub.split('end subroutine')[0] self.assertIn('inst_num', final_sub) @@ -2456,7 +2456,7 @@ def setUp(self): def test_init_calls_state_alloc_with_ninstances(self): # host_full.meta has ninstances → number_of_instances. self.assertIn( - 'call ccpp_test_simple_physics_state_alloc(ninstances, errmsg, errflg)', self.text + 'call physics_state_alloc(ninstances, errmsg, errflg)', self.text ) def test_init_subroutine_has_ninstances_arg(self): @@ -2465,14 +2465,14 @@ def test_init_subroutine_has_ninstances_arg(self): def test_final_calls_state_dealloc(self): self.assertIn( - 'call ccpp_test_simple_physics_state_dealloc(errmsg, errflg)', self.text + 'call physics_state_dealloc(errmsg, errflg)', self.text ) def test_state_alloc_imported_in_suite_cap(self): - self.assertIn('ccpp_test_simple_physics_state_alloc', self.text.split('contains')[0]) + self.assertIn('physics_state_alloc', self.text.split('contains')[0]) def test_state_dealloc_imported_in_suite_cap(self): - self.assertIn('ccpp_test_simple_physics_state_dealloc', self.text.split('contains')[0]) + self.assertIn('physics_state_dealloc', self.text.split('contains')[0]) class TestSuiteCapStateCallsSingleInstance(unittest.TestCase): @@ -2492,7 +2492,7 @@ def setUp(self): def test_init_calls_state_alloc_with_literal_1(self): self.assertIn( - 'call ccpp_test_simple_physics_state_alloc(1, errmsg, errflg)', self.text + 'call physics_state_alloc(1, errmsg, errflg)', self.text ) def test_init_subroutine_has_no_ninstances_arg(self): From 205ec79eea3815f16e2b3c3a87f76a67ddff6aab Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 May 2026 09:43:18 -0600 Subject: [PATCH 19/74] Update capgen-ng: move number_of_instances from host table to control table --- capgen-ng/ccpp_capgen_ng.py | 63 ++++------------ capgen-ng/generator/host_constituents.py | 22 ++++-- capgen-ng/generator/static_api.py | 49 ++++++++++-- capgen-ng/generator/suite_cap.py | 70 +++++++++++++----- end-to-end-tests/cmake/ccpp_capgen.cmake | 12 ++- end-to-end-tests/instances/data.meta | 6 -- end-to-end-tests/instances/main.F90 | 31 +++++--- end-to-end-tests/instances/main.meta | 6 ++ .../sample_files/control_chunked_data.meta | 6 ++ unit-tests/sample_files/control_full.meta | 6 ++ .../sample_files/control_inst_only.meta | 74 +++++++++++++++++++ .../sample_files/control_ninst_only.meta | 74 +++++++++++++++++++ unit-tests/sample_files/control_opt_arg.meta | 6 ++ unit-tests/sample_files/control_simple.meta | 6 ++ .../sample_files/control_unit_conv.meta | 6 ++ .../sample_files/host_chunked_data.meta | 6 -- unit-tests/sample_files/host_full.meta | 6 -- unit-tests/sample_files/host_opt_arg.meta | 6 -- unit-tests/sample_files/host_simple.meta | 6 -- unit-tests/sample_files/host_unit_conv.meta | 6 -- .../sample_files/host_with_constituents.meta | 6 -- .../sample_files/host_with_dependencies.meta | 6 -- unit-tests/test_control_validation.py | 14 ++-- unit-tests/test_host_constituents.py | 5 +- unit-tests/test_integration.py | 15 ++-- unit-tests/test_static_api.py | 33 ++++----- unit-tests/test_variable_resolver.py | 7 +- 27 files changed, 371 insertions(+), 182 deletions(-) create mode 100644 unit-tests/sample_files/control_inst_only.meta create mode 100644 unit-tests/sample_files/control_ninst_only.meta diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index 582df41c..32754908 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -721,73 +721,38 @@ def _check_control_var(std_name, expected_type, description, required: bool) -> ) ) - def _check_host_module_var(std_name, expected_type, description) -> None: - """Validate a variable that must live in a ``type=host`` table. - - Used for symbols the generator emits via ``use , only: - `` rather than as call-arg control vars. - """ - entry = host_dict.get(std_name) - if entry is None: - return - - if entry.is_control: - errors.append( - "Variable '{}' must be declared in a type=host table for " - "host '{}' (it is USE'd from the host module), but it was " - "found in a type=control table.\n" - " Move it to a [ccpp-table-properties] / type=host " - "block.".format(std_name, host_name) - ) - return - - if entry.type.lower() != expected_type.lower(): - errors.append( - "Host variable '{}' in host '{}' has Fortran type '{}' but " - "'{}' is required.".format( - std_name, host_name, entry.type, expected_type - ) - ) - - if entry.dimensions: - errors.append( - "Host variable '{}' in host '{}' must be a scalar (rank-0) " - "but has dimensions {}.".format( - std_name, host_name, entry.dimensions - ) - ) - for std_name, expected_type, description in _REQUIRED_CTRL_VARS: _check_control_var(std_name, expected_type, description, required=True) - # Paired optional: instance_number lives in type=control (call-arg); - # number_of_instances lives in type=host (USE'd by the suite cap for - # state-array sizing). Either both declared or neither. + # Paired optional: both ``instance_number`` (the per-call index) and + # ``number_of_instances`` (the bound, used at register time to size + # the per-instance state arrays) live in ``type=control``. Symmetric + # with the (thread_number, number_of_threads) pair. Either both + # declared or neither. _check_control_var( 'instance_number', 'integer', 'current model instance index', required=False, ) - _check_host_module_var( + _check_control_var( 'number_of_instances', 'integer', - 'total number of model instances', + 'total number of model instances', required=False, ) inst_present = host_dict.get('instance_number') is not None ninst_present = host_dict.get('number_of_instances') is not None if inst_present ^ ninst_present: - present, missing, present_table, missing_table = ( - ('instance_number', 'number_of_instances', 'control', 'host') + present, missing = ( + ('instance_number', 'number_of_instances') if inst_present - else ('number_of_instances', 'instance_number', 'host', 'control') + else ('number_of_instances', 'instance_number') ) errors.append( - "Host '{}' declares '{}' (in a type={} table) but is missing " - "the paired variable '{}' (which must be declared in a " - "type={} table).\n" + "Host '{}' declares '{}' (in a type=control table) but is " + "missing the paired variable '{}' (which must also be in a " + "type=control table).\n" " Declare both for a multi-instance API, or neither for a " "single-instance API.".format( - host_name, present, present_table, - missing, missing_table, + host_name, present, missing, ) ) diff --git a/capgen-ng/generator/host_constituents.py b/capgen-ng/generator/host_constituents.py index c0c5302d..7c9a3e64 100644 --- a/capgen-ng/generator/host_constituents.py +++ b/capgen-ng/generator/host_constituents.py @@ -3,7 +3,8 @@ """Generate ``ccpp_host_constituents.F90`` — the host-wide constituent module. In capgen-ng's per-instance design, the constituent state is sized to -``number_of_instances`` (declared by the host's ``type=host`` table). +``number_of_instances`` (declared by the host's ``type=control`` table, +paired with ``instance_number``). This module owns: * ``ccpp_model_constituents_obj(:)`` — one DDT instance per host @@ -97,11 +98,15 @@ def _host_lookup(host_dict, std_name: str) -> Tuple[Optional[str], Optional[str] def _instance_signature( base_args: List[str], inst_local: Optional[str], + ninst_local: Optional[str] = None, ) -> List[str]: - """Return signature args with *inst_local* inserted before err args.""" + """Return signature args with *inst_local* and (optionally) + *ninst_local* inserted before err args.""" sig = list(base_args) if inst_local: sig.append(inst_local) + if ninst_local: + sig.append(ninst_local) sig += ['errflg', 'errmsg'] return sig @@ -116,17 +121,20 @@ def _register_constituents_lines( i3 = _INDENT * 3 register_suites = _suites_with_register_consts(suite_results) inst_local, _ = _host_lookup(host_dict, 'instance_number') - ninst_local, ninst_mod = _host_lookup(host_dict, 'number_of_instances') + # ``number_of_instances`` is now a paired control variable; it arrives + # as a dummy argument rather than via ``use ``. Its module + # is ``None`` (control vars carry no module), so ninst_mod is ignored. + ninst_local, _ = _host_lookup(host_dict, 'number_of_instances') inst_idx = inst_local if inst_local else '1' ninst_arg = ninst_local if ninst_local else '1' - sig = _instance_signature(['host_constituents'], inst_local) + sig = _instance_signature( + ['host_constituents'], inst_local, ninst_local=ninst_local, + ) lines: List[str] = [''] lines.append('{}subroutine ccpp_register_constituents({})'.format( i1, ', '.join(sig), )) - if ninst_local and ninst_mod: - lines.append('{}use {}, only: {}'.format(i2, ninst_mod, ninst_local)) lines.append('') lines.append( '{}type({}), target, intent(in) :: host_constituents(:)'.format( @@ -135,6 +143,8 @@ def _register_constituents_lines( ) if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninst_local)) lines.append('{}integer, intent(out) :: errflg'.format(i2)) lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) lines.append('') diff --git a/capgen-ng/generator/static_api.py b/capgen-ng/generator/static_api.py index d8137679..3f64edb6 100644 --- a/capgen-ng/generator/static_api.py +++ b/capgen-ng/generator/static_api.py @@ -350,7 +350,9 @@ def _register_subroutine(suite_names: List[str], host_dict=None) -> List[str]: """Generate ``ccpp_register`` (mandatory entry point). Always emitted with the minimal lifecycle signature - ``(suite_name, ccpp_error_code, ccpp_error_message, instance_number)``. + ``(suite_name, ccpp_error_code, ccpp_error_message, + [instance_number, number_of_instances])``. The instance pair is + forwarded to ``_register`` only when the host declares it. The body dispatches to ``_register`` for every known suite. Each suite's register routine is responsible for allocating its state array and DDT instance array, calling its register-phase scheme entrypoints, @@ -362,6 +364,8 @@ def _register_subroutine(suite_names: List[str], host_dict=None) -> List[str]: i3 = _INDENT * 3 inst_local = _instance_local(host_dict) + ninst_entry = host_dict.get('number_of_instances') if host_dict else None + ninst_local = ninst_entry.local_name if ninst_entry else None errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' @@ -375,6 +379,9 @@ def _register_subroutine(suite_names: List[str], host_dict=None) -> List[str]: if inst_local: sig_args.append(inst_local) suite_call_args.append(inst_local) + if ninst_local: + sig_args.append(ninst_local) + suite_call_args.append(ninst_local) suite_call_args += [errmsg_local, errflg_local] lines: List[str] = [''] @@ -385,9 +392,13 @@ def _register_subroutine(suite_names: List[str], host_dict=None) -> List[str]: lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninst_local)) trace_entries = [suite_name_entry] if suite_name_entry else [] + extra_in = [ninst_local] if ninst_local else None trace_lines = emit_trace_block( - 'ccpp_register', trace_entries, i2, instance_local=inst_local, + 'ccpp_register', trace_entries, i2, + instance_local=inst_local, extra_in_names=extra_in, ) if trace_lines: lines.append('') @@ -420,14 +431,17 @@ def _register_subroutine(suite_names: List[str], host_dict=None) -> List[str]: def _init_subroutine(suite_names: List[str], host_dict=None) -> List[str]: """Generate ``ccpp_init`` (minimal lifecycle signature). - Signature: ``(suite_name, ccpp_error_code, ccpp_error_message, instance_number)``. - Forwards ``instance_number`` (when host-declared) to ``_init``. + Signature: ``(suite_name, ccpp_error_code, ccpp_error_message, + [instance_number, number_of_instances])``. The instance pair is + forwarded to ``_init`` only when the host declares it. """ i1 = _INDENT i2 = _INDENT * 2 i3 = _INDENT * 3 inst_local = _instance_local(host_dict) + ninst_entry = host_dict.get('number_of_instances') if host_dict else None + ninst_local = ninst_entry.local_name if ninst_entry else None errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' @@ -441,6 +455,9 @@ def _init_subroutine(suite_names: List[str], host_dict=None) -> List[str]: if inst_local: sig_args.append(inst_local) suite_call_args.append(inst_local) + if ninst_local: + sig_args.append(ninst_local) + suite_call_args.append(ninst_local) suite_call_args += [errmsg_local, errflg_local] lines: List[str] = [''] @@ -451,9 +468,13 @@ def _init_subroutine(suite_names: List[str], host_dict=None) -> List[str]: lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninst_local)) trace_entries = [suite_name_entry] if suite_name_entry else [] + extra_in = [ninst_local] if ninst_local else None trace_lines = emit_trace_block( - 'ccpp_init', trace_entries, i2, instance_local=inst_local, + 'ccpp_init', trace_entries, i2, + instance_local=inst_local, extra_in_names=extra_in, ) if trace_lines: lines.append('') @@ -484,15 +505,20 @@ def _init_subroutine(suite_names: List[str], host_dict=None) -> List[str]: def _final_subroutine(suite_names: List[str], host_dict=None) -> List[str]: - """Generate ``ccpp_final`` (minimal lifecycle signature). + """Generate ``ccpp_final`` (lifecycle signature). - Signature: ``(suite_name, ccpp_error_code, ccpp_error_message, instance_number)``. + Signature: ``(suite_name, ccpp_error_code, ccpp_error_message, + [instance_number, number_of_instances])``. ``number_of_instances`` + is carried for API symmetry with ``ccpp_register`` / ``ccpp_init`` + even though the framework does not need it at final time. """ i1 = _INDENT i2 = _INDENT * 2 i3 = _INDENT * 3 inst_local = _instance_local(host_dict) + ninst_entry = host_dict.get('number_of_instances') if host_dict else None + ninst_local = ninst_entry.local_name if ninst_entry else None errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' @@ -506,6 +532,9 @@ def _final_subroutine(suite_names: List[str], host_dict=None) -> List[str]: if inst_local: sig_args.append(inst_local) suite_call_args.append(inst_local) + if ninst_local: + sig_args.append(ninst_local) + suite_call_args.append(ninst_local) suite_call_args += [errmsg_local, errflg_local] lines: List[str] = [''] @@ -516,9 +545,13 @@ def _final_subroutine(suite_names: List[str], host_dict=None) -> List[str]: lines.append('{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local)) if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninst_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninst_local)) trace_entries = [suite_name_entry] if suite_name_entry else [] + extra_in = [ninst_local] if ninst_local else None trace_lines = emit_trace_block( - 'ccpp_final', trace_entries, i2, instance_local=inst_local, + 'ccpp_final', trace_entries, i2, + instance_local=inst_local, extra_in_names=extra_in, ) if trace_lines: lines.append('') diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 05b68161..9d384fd4 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -286,8 +286,8 @@ def _register_lines( register-phase scheme call across all groups in suite-XML order, and transitions the suite state for this instance to ``CCPP_SUITE_REGISTERED``. - Minimal signature: ``(instance_number, errmsg, errflg)`` (instance_number - is included only when the host declares it). + Minimal signature: ``(instance_number, number_of_instances, errmsg, + errflg)`` (the instance pair is included only when the host declares it). """ sub_name = '{}_register'.format(suite_name) i1 = _INDENT @@ -296,6 +296,12 @@ def _register_lines( inst_local = _instance_local(host_dict) inst_idx = _instance_idx(host_dict) + # ``number_of_instances`` is now a paired control variable (see + # ccpp_capgen_ng._PAIRED_OPTIONAL_CTRL_VARS). When present it enters + # the suite-cap signature as a dummy alongside ``instance_number``; + # the framework consumes it at register-time to size the per-instance + # state arrays. When absent we fall back to the literal ``1`` + # (single-instance API). ninstances_entry = host_dict.get('number_of_instances') if host_dict else None ninstances_local = ninstances_entry.local_name if ninstances_entry else None ninstances_arg = ninstances_local if ninstances_local else '1' @@ -306,6 +312,8 @@ def _register_lines( sig_args: List[str] = [] if inst_local: sig_args.append(inst_local) + if ninstances_local: + sig_args.append(ninstances_local) sig_args += [errmsg_local, errflg_local] lines: List[str] = [] @@ -313,12 +321,9 @@ def _register_lines( lines.append('{}subroutine {}({})'.format(i1, sub_name, ', '.join(sig_args))) # USE statements: scheme modules + host/suite-data modules referenced by - # register-phase scheme args. + # register-phase scheme args. (``number_of_instances`` used to be USE'd + # from the host module here; now it arrives as a dummy argument.) reg_uses = _register_uses(suite_res, suite_name, host_dict) - if ninstances_local and ninstances_entry is not None and ninstances_entry.module_name: - reg_uses.setdefault(ninstances_entry.module_name, set()).add( - ninstances_local - ) for mod in sorted(reg_uses): syms = ', '.join(sorted(reg_uses[mod])) lines.append('{}use {}, only: {}'.format(i2, mod, syms)) @@ -326,6 +331,8 @@ def _register_lines( lines.append('') if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninstances_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninstances_local)) lines += [ '{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local), '{}integer, intent(out) :: {}'.format(i2, errflg_local), @@ -343,8 +350,12 @@ def _register_lines( lines.append('{}integer :: num_consts, i'.format(i2)) # Trace block: dummies referenced inside the gated write so strict - # compilers don't flag instance_number as unused when the gate is off. - trace_lines = emit_trace_block(sub_name, [], i2, instance_local=inst_local) + # compilers don't flag intent(in) args as unused when the gate is off. + extra_in = [ninstances_local] if ninstances_local else None + trace_lines = emit_trace_block( + sub_name, [], i2, + instance_local=inst_local, extra_in_names=extra_in, + ) if trace_lines: lines.append('') lines.extend(trace_lines) @@ -460,12 +471,15 @@ def _init_lines( been written during the register phase. 4. Sets ``ccpp_suite_state(instance_number) = CCPP_SUITE_FRAMEWORK_INITIALIZED``. - Minimal signature: ``(instance_number, errmsg, errflg)``. + Minimal signature: ``(instance_number, number_of_instances, errmsg, + errflg)`` -- the instance pair is included only when the host declares it. """ sub_name = '{}_init'.format(suite_name) i1 = _INDENT i2 = _INDENT * 2 + # ``number_of_instances`` is now a paired control variable; it arrives + # as a dummy argument rather than via ``use ``. ninstances_entry = host_dict.get('number_of_instances') if host_dict else None ninstances_local = ninstances_entry.local_name if ninstances_entry else None ninstances_arg = ninstances_local if ninstances_local else '1' @@ -479,20 +493,18 @@ def _init_lines( sig_args: List[str] = [] if inst_local: sig_args.append(inst_local) + if ninstances_local: + sig_args.append(ninstances_local) sig_args += [errmsg_local, errflg_local] lines: List[str] = [''] lines.append('{}subroutine {}({})'.format(i1, sub_name, ', '.join(sig_args))) - # USE: number_of_instances (from host module) for group state alloc; - # suite_data init_fields routine when this suite owns any vars; + # USE: suite_data init_fields routine when this suite owns any vars; # constituent object (from host module) for pointer binding; # suite-level scheme module + per-arg host modules. + # (``number_of_instances`` used to be USE'd here; now it's a dummy arg.) extra_uses: Dict[str, Set[str]] = {} - if ninstances_local and ninstances_entry is not None and ninstances_entry.module_name: - extra_uses.setdefault(ninstances_entry.module_name, set()).add( - ninstances_local - ) if suite_res.suite_vars: data_mod = 'ccpp_{}_data'.format(suite_name) init_fields = 'suite_data_init_fields' @@ -506,11 +518,17 @@ def _init_lines( lines.append('') if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninstances_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninstances_local)) lines += [ '{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local), '{}integer, intent(out) :: {}'.format(i2, errflg_local), ] - trace_lines = emit_trace_block(sub_name, [], i2, instance_local=inst_local) + extra_in = [ninstances_local] if ninstances_local else None + trace_lines = emit_trace_block( + sub_name, [], i2, + instance_local=inst_local, extra_in_names=extra_in, + ) if trace_lines: lines.append('') lines.extend(trace_lines) @@ -606,7 +624,11 @@ def _final_lines( flip, calls each group ``state_dealloc`` and the suite ``state_dealloc`` (which also tears down the suite_data DDT array). - Minimal signature: ``(instance_number, errmsg, errflg)``. + Signature: ``(instance_number, number_of_instances, errmsg, errflg)`` + when the host declares the multi-instance pair, else + ``(errmsg, errflg)``. ``number_of_instances`` is carried for API + symmetry with ``_register`` / ``_init``; the framework + does not consume it at final time. """ sub_name = '{}_final'.format(suite_name) i1 = _INDENT @@ -614,6 +636,8 @@ def _final_lines( inst_local = _instance_local(host_dict) inst_idx = _instance_idx(host_dict) + ninstances_entry = host_dict.get('number_of_instances') if host_dict else None + ninstances_local = ninstances_entry.local_name if ninstances_entry else None errflg_local = _ctrl_local(host_dict, 'ccpp_error_code') or 'errflg' errmsg_local = _ctrl_local(host_dict, 'ccpp_error_message') or 'errmsg' @@ -621,6 +645,8 @@ def _final_lines( sig_args: List[str] = [] if inst_local: sig_args.append(inst_local) + if ninstances_local: + sig_args.append(ninstances_local) sig_args += [errmsg_local, errflg_local] lines: List[str] = [''] @@ -650,11 +676,17 @@ def _final_lines( lines.append('') if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) + if ninstances_local: + lines.append('{}integer, intent(in) :: {}'.format(i2, ninstances_local)) lines += [ '{}character(len=*), intent(out) :: {}'.format(i2, errmsg_local), '{}integer, intent(out) :: {}'.format(i2, errflg_local), ] - trace_lines = emit_trace_block(sub_name, [], i2, instance_local=inst_local) + extra_in = [ninstances_local] if ninstances_local else None + trace_lines = emit_trace_block( + sub_name, [], i2, + instance_local=inst_local, extra_in_names=extra_in, + ) if trace_lines: lines.append('') lines.extend(trace_lines) diff --git a/end-to-end-tests/cmake/ccpp_capgen.cmake b/end-to-end-tests/cmake/ccpp_capgen.cmake index dc5b9918..0b179bc6 100644 --- a/end-to-end-tests/cmake/ccpp_capgen.cmake +++ b/end-to-end-tests/cmake/ccpp_capgen.cmake @@ -141,6 +141,10 @@ function(ccpp_capgen) message(STATUS "ccpp-capgen stdout: ${CAPGEN_OUT}") + if(NOT RES EQUAL 0) + message(FATAL_ERROR "CCPP cap generation FAILED: result = ${RES}") + endif() + endfunction() @@ -179,15 +183,15 @@ function(ccpp_datafile) OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_STRIP_TRAILING_WHITESPACE COMMAND_ECHO STDOUT) - #message(STATUS "CCPP_FILES = ${CCPP_FILES}") - if(RES EQUAL 0) - message(STATUS "CCPP files retrieved") - else() + + if(NOT RES EQUAL 0) message(FATAL_ERROR "CCPP file retrieval FAILED: result = ${RES}") endif() + if(CCPP_FILES) # Convert "," separated list from python back to ";" separated list for CMake string(REPLACE "," ";" CCPP_FILES ${CCPP_FILES}) endif() set(CCPP_FILES "${CCPP_FILES}" PARENT_SCOPE) + endfunction() diff --git a/end-to-end-tests/instances/data.meta b/end-to-end-tests/instances/data.meta index 7abae0e2..1fccc944 100644 --- a/end-to-end-tests/instances/data.meta +++ b/end-to-end-tests/instances/data.meta @@ -63,12 +63,6 @@ units = count dimensions = () type = integer -[ninstances] - standard_name = number_of_instances - long_name = number of instances for multi-instance test - units = count - dimensions = () - type = integer [instance_data] standard_name = instance_data long_name = instance data for multi-instance test diff --git a/end-to-end-tests/instances/main.F90 b/end-to-end-tests/instances/main.F90 index 20a49128..45643507 100644 --- a/end-to-end-tests/instances/main.F90 +++ b/end-to-end-tests/instances/main.F90 @@ -7,10 +7,6 @@ program test_unit_conv use data, only: ncols, & nspecies, ninstances - !use data, only: cdata, & - ! data_array, & - ! data_array2, & - ! opt_array_flag use data, only: instance_data use ccpp_static_api, only: ccpp_register, & @@ -55,7 +51,9 @@ program test_unit_conv !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! do ins=1,ninstances - call ccpp_register(suite_name=ccpp_suite, instance=ins, errmsg=errmsg, errflg=errflg) + call ccpp_register(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) if (errflg/=0) then write(error_unit, '(a)') "An error occurred in ccpp_register:" write(error_unit, '(a)') trim(errmsg) @@ -68,7 +66,9 @@ program test_unit_conv !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! do ins=1,ninstances - call ccpp_init(suite_name=ccpp_suite, instance=ins, errmsg=errmsg, errflg=errflg) + call ccpp_init(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) if (errflg/=0) then write(error_unit, '(a)') "An error occurred in ccpp_init:" write(error_unit, '(a)') trim(errmsg) @@ -83,7 +83,8 @@ program test_unit_conv do ins=1,ninstances call ccpp_physics_init( & - suite_name=ccpp_suite, group_name='all', instance=ins, & + suite_name=ccpp_suite, group_name='all', & + instance=ins, ninstances=ninstances, & thread_num=1, nthreads=1, nphys_threads=nphys_threads, & lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) if (errflg/=0) then @@ -100,7 +101,8 @@ program test_unit_conv do ins=1,ninstances call ccpp_physics_timestep_init( & - suite_name=ccpp_suite, group_name='all', instance=ins, & + suite_name=ccpp_suite, group_name='all', & + instance=ins, ninstances=ninstances, & thread_num=1, nthreads=1, nphys_threads=nphys_threads, & lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) if (errflg/=0) then @@ -116,7 +118,8 @@ program test_unit_conv do ins=1,ninstances call ccpp_physics_run( & - suite_name=ccpp_suite, group_name='all', instance=ins, & + suite_name=ccpp_suite, group_name='all', & + instance=ins, ninstances=ninstances, & thread_num=1, nthreads=1, nphys_threads=nphys_threads, & lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) if (errflg/=0) then @@ -133,7 +136,8 @@ program test_unit_conv do ins=1,ninstances call ccpp_physics_timestep_final( & - suite_name=ccpp_suite, group_name='all', instance=ins, & + suite_name=ccpp_suite, group_name='all', & + instance=ins, ninstances=ninstances, & thread_num=1, nthreads=1, nphys_threads=nphys_threads, & lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) if (errflg/=0) then @@ -150,7 +154,8 @@ program test_unit_conv do ins=1,ninstances call ccpp_physics_final( & - suite_name=ccpp_suite, group_name='all', instance=ins, & + suite_name=ccpp_suite, group_name='all', & + instance=ins, ninstances=ninstances, & thread_num=1, nthreads=1, nphys_threads=nphys_threads, & lb=1, ub=ncols, errmsg=errmsg, errflg=errflg) if (errflg/=0) then @@ -165,7 +170,9 @@ program test_unit_conv !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! do ins=1,ninstances - call ccpp_final(suite_name=ccpp_suite, instance=ins, errmsg=errmsg, errflg=errflg) + call ccpp_final(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) if (errflg/=0) then write(error_unit, '(a)') "An error occurred in ccpp_final:" write(error_unit, '(a)') trim(errmsg) diff --git a/end-to-end-tests/instances/main.meta b/end-to-end-tests/instances/main.meta index 8d774264..e1f64d13 100644 --- a/end-to-end-tests/instances/main.meta +++ b/end-to-end-tests/instances/main.meta @@ -69,3 +69,9 @@ units = index dimensions = () type = integer +[ninstances] + standard_name = number_of_instances + long_name = number of instances for multi-instance test + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_chunked_data.meta b/unit-tests/sample_files/control_chunked_data.meta index 04dfdf98..4873c759 100644 --- a/unit-tests/sample_files/control_chunked_data.meta +++ b/unit-tests/sample_files/control_chunked_data.meta @@ -77,3 +77,9 @@ units = 1 dimensions = () type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_full.meta b/unit-tests/sample_files/control_full.meta index 3f419372..1bc31c7b 100644 --- a/unit-tests/sample_files/control_full.meta +++ b/unit-tests/sample_files/control_full.meta @@ -71,3 +71,9 @@ units = 1 dimensions = () type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_inst_only.meta b/unit-tests/sample_files/control_inst_only.meta new file mode 100644 index 00000000..d796f960 --- /dev/null +++ b/unit-tests/sample_files/control_inst_only.meta @@ -0,0 +1,74 @@ +# Half-paired multi-instance host: control table declares instance_number +# but NOT its mandatory pair partner number_of_instances. Used to +# exercise the paired-validation error path (declaring one of the pair +# without the other must raise). +[ccpp-table-properties] + name = ccpp_control + type = control + +[ccpp-arg-table] + name = ccpp_control + type = control +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = total number of physics threads + units = 1 + dimensions = () + type = integer +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ inst_num ] + standard_name = instance_number + long_name = current model instance number + units = 1 + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_ninst_only.meta b/unit-tests/sample_files/control_ninst_only.meta new file mode 100644 index 00000000..69402958 --- /dev/null +++ b/unit-tests/sample_files/control_ninst_only.meta @@ -0,0 +1,74 @@ +# Half-paired multi-instance host: control table declares +# number_of_instances but NOT its mandatory pair partner instance_number. +# Used to exercise the paired-validation error path (declaring one of +# the pair without the other must raise). +[ccpp-table-properties] + name = ccpp_control + type = control + +[ccpp-arg-table] + name = ccpp_control + type = control +[ suite_name_var ] + standard_name = suite_name + long_name = name of the CCPP suite + units = none + dimensions = () + type = character + kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = 1 + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = total number of physics threads + units = 1 + dimensions = () + type = integer +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of threads + units = 1 + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_opt_arg.meta b/unit-tests/sample_files/control_opt_arg.meta index bbbf2e4b..27919026 100644 --- a/unit-tests/sample_files/control_opt_arg.meta +++ b/unit-tests/sample_files/control_opt_arg.meta @@ -71,3 +71,9 @@ units = 1 dimensions = () type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_simple.meta b/unit-tests/sample_files/control_simple.meta index 81a47798..7b5d2980 100644 --- a/unit-tests/sample_files/control_simple.meta +++ b/unit-tests/sample_files/control_simple.meta @@ -68,3 +68,9 @@ units = 1 dimensions = () type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/control_unit_conv.meta b/unit-tests/sample_files/control_unit_conv.meta index 2f9fd066..75edc085 100644 --- a/unit-tests/sample_files/control_unit_conv.meta +++ b/unit-tests/sample_files/control_unit_conv.meta @@ -64,3 +64,9 @@ units = 1 dimensions = () type = integer +[ ninstances ] + standard_name = number_of_instances + long_name = total number of model instances + units = count + dimensions = () + type = integer diff --git a/unit-tests/sample_files/host_chunked_data.meta b/unit-tests/sample_files/host_chunked_data.meta index 225ea68f..278602af 100644 --- a/unit-tests/sample_files/host_chunked_data.meta +++ b/unit-tests/sample_files/host_chunked_data.meta @@ -23,9 +23,3 @@ units = DDT dimensions = () type = chunked_data_type -[ ninstances ] - standard_name = number_of_instances - long_name = number of model instances - units = count - dimensions = () - type = integer diff --git a/unit-tests/sample_files/host_full.meta b/unit-tests/sample_files/host_full.meta index c4d7ce32..854db157 100644 --- a/unit-tests/sample_files/host_full.meta +++ b/unit-tests/sample_files/host_full.meta @@ -48,9 +48,3 @@ dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys -[ ninstances ] - standard_name = number_of_instances - long_name = number of model instances (ensemble members) - units = count - dimensions = () - type = integer diff --git a/unit-tests/sample_files/host_opt_arg.meta b/unit-tests/sample_files/host_opt_arg.meta index 5fb01b7d..0de58b65 100644 --- a/unit-tests/sample_files/host_opt_arg.meta +++ b/unit-tests/sample_files/host_opt_arg.meta @@ -44,9 +44,3 @@ units = flag dimensions = () type = logical -[ ninstances ] - standard_name = number_of_instances - long_name = number of model instances - units = count - dimensions = () - type = integer diff --git a/unit-tests/sample_files/host_simple.meta b/unit-tests/sample_files/host_simple.meta index 3527e210..baa5c035 100644 --- a/unit-tests/sample_files/host_simple.meta +++ b/unit-tests/sample_files/host_simple.meta @@ -17,9 +17,3 @@ units = count dimensions = () type = integer -[ ninstances ] - standard_name = number_of_instances - long_name = number of model instances - units = count - dimensions = () - type = integer diff --git a/unit-tests/sample_files/host_unit_conv.meta b/unit-tests/sample_files/host_unit_conv.meta index 3fd44700..de0c3b98 100644 --- a/unit-tests/sample_files/host_unit_conv.meta +++ b/unit-tests/sample_files/host_unit_conv.meta @@ -43,9 +43,3 @@ units = flag dimensions = () type = logical -[ ninstances ] - standard_name = number_of_instances - long_name = number of model instances - units = count - dimensions = () - type = integer diff --git a/unit-tests/sample_files/host_with_constituents.meta b/unit-tests/sample_files/host_with_constituents.meta index 75328806..4f9360f2 100644 --- a/unit-tests/sample_files/host_with_constituents.meta +++ b/unit-tests/sample_files/host_with_constituents.meta @@ -22,12 +22,6 @@ units = count dimensions = () type = integer -[ ninstances ] - standard_name = number_of_instances - long_name = number of model instances - units = count - dimensions = () - type = integer [ host_consts_obj ] standard_name = ccpp_model_constituents_object long_name = host-owned model constituent object diff --git a/unit-tests/sample_files/host_with_dependencies.meta b/unit-tests/sample_files/host_with_dependencies.meta index cf998279..54d7a99d 100644 --- a/unit-tests/sample_files/host_with_dependencies.meta +++ b/unit-tests/sample_files/host_with_dependencies.meta @@ -37,12 +37,6 @@ units = count dimensions = () type = integer -[ ninstances ] - standard_name = number_of_instances - long_name = number of model instances - units = count - dimensions = () - type = integer [ dt ] standard_name = time_step_for_physics long_name = physics time step diff --git a/unit-tests/test_control_validation.py b/unit-tests/test_control_validation.py index b81cb324..e5f3ba1d 100644 --- a/unit-tests/test_control_validation.py +++ b/unit-tests/test_control_validation.py @@ -219,15 +219,15 @@ def test_no_instance_pair_passes(self): class TestInstanceNumberPairing(unittest.TestCase): - """instance_number (control) and number_of_instances (host) are - paired-optional: declaring exactly one is an error. + """instance_number and number_of_instances both live in type=control + and are paired-optional: declaring exactly one is an error. """ def test_instance_alone_raises(self): - """control declares instance_number, host omits number_of_instances.""" + """control declares instance_number but not number_of_instances.""" host_dict = _build_host_dict( host_files=[_sf('host_no_instance.meta')], - control_files=[_sf('control_simple.meta')], + control_files=[_sf('control_inst_only.meta')], ) with self.assertRaises(CCPPError) as ctx: _validate_required_control_vars('test_host', host_dict) @@ -237,10 +237,10 @@ def test_instance_alone_raises(self): self.assertIn('paired', msg.lower()) def test_ninstances_alone_raises(self): - """host declares number_of_instances, control omits instance_number.""" + """control declares number_of_instances but not instance_number.""" host_dict = _build_host_dict( - host_files=[_sf('host_simple.meta')], - control_files=[_sf('control_no_instance.meta')], + host_files=[_sf('host_no_instance.meta')], + control_files=[_sf('control_ninst_only.meta')], ) with self.assertRaises(CCPPError) as ctx: _validate_required_control_vars('test_host', host_dict) diff --git a/unit-tests/test_host_constituents.py b/unit-tests/test_host_constituents.py index 16a5433f..828f7175 100644 --- a/unit-tests/test_host_constituents.py +++ b/unit-tests/test_host_constituents.py @@ -200,10 +200,11 @@ def setUp(self): self.text = _render_register() def test_takes_host_constituents_and_instance(self): - # instance_number is in the signature when the host declares it. + # instance_number AND number_of_instances are both in the signature + # when the host declares the multi-instance pair. self.assertIn( 'subroutine ccpp_register_constituents(host_constituents, ' - 'inst_num, errflg, errmsg)', + 'inst_num, ninstances, errflg, errmsg)', self.text, ) self.assertIn( diff --git a/unit-tests/test_integration.py b/unit-tests/test_integration.py index 3c13d123..85894712 100644 --- a/unit-tests/test_integration.py +++ b/unit-tests/test_integration.py @@ -695,11 +695,12 @@ def tearDown(self): shutil.rmtree(self._tmpdir) def test_ccpp_init_minimal_signature(self): - # New minimal lifecycle signature: drops ninstances entirely. + # Lifecycle signature for a multi-instance host carries the + # paired (inst_num, ninstances) control vars. with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: text = fh.read() self.assertIn( - 'subroutine ccpp_init(suite_name, errflg, errmsg, inst_num)', + 'subroutine ccpp_init(suite_name, errflg, errmsg, inst_num, ninstances)', text, ) @@ -707,7 +708,7 @@ def test_suite_init_minimal_signature(self): with open(os.path.join(self._tmpdir, 'ccpp_test_simple_cap.F90')) as fh: text = fh.read() self.assertIn( - 'subroutine test_simple_init(inst_num, errmsg, errflg)', + 'subroutine test_simple_init(inst_num, ninstances, errmsg, errflg)', text, ) @@ -882,8 +883,8 @@ def test_instance_alone_raises(self): capgen( host_name='test_host', host_files=[ - _sf('host_no_instance.meta'), # no number_of_instances - _sf('control_full.meta'), # has instance_number + _sf('host_no_instance.meta'), + _sf('control_inst_only.meta'), # has instance_number only ], scheme_files=[_sf('scheme_multipart.meta')], suite_files=[_suite_file('suite_test_simple.xml')], @@ -901,8 +902,8 @@ def test_ninstances_alone_raises(self): capgen( host_name='test_host', host_files=[ - _sf('host_full.meta'), # has number_of_instances - _sf('control_no_instance.meta'),# no instance_number + _sf('host_no_instance.meta'), + _sf('control_ninst_only.meta'), # has ninstances only ], scheme_files=[_sf('scheme_multipart.meta')], suite_files=[_suite_file('suite_test_simple.xml')], diff --git a/unit-tests/test_static_api.py b/unit-tests/test_static_api.py index a57533f8..48c78975 100644 --- a/unit-tests/test_static_api.py +++ b/unit-tests/test_static_api.py @@ -339,34 +339,38 @@ def setUp(self): lines = _generate_static_api(['test_simple'], [suite_resolution], hd) self.text = '\n'.join(lines) - def test_init_signature_has_instance_number(self): - # host_full.meta declares inst_num as instance_number; ninstances is - # NOT in the lifecycle signature any more. + def test_init_signature_has_instance_pair(self): + # host_full.meta declares the multi-instance pair (inst_num, + # ninstances). Both must appear in the lifecycle signature. self.assertIn( - 'subroutine ccpp_init(suite_name, errflg, errmsg, inst_num)', + 'subroutine ccpp_init(suite_name, errflg, errmsg, inst_num, ninstances)', self.text, ) - def test_init_signature_no_ninstances(self): + def test_init_signature_has_ninstances(self): init_block = self.text.split('subroutine ccpp_init')[1].split( 'end subroutine ccpp_init' )[0] - self.assertNotIn('ninstances', init_block) + self.assertIn('ninstances', init_block) - def test_init_passes_inst_to_suite(self): + def test_init_passes_inst_pair_to_suite(self): self.assertIn( - 'call test_simple_init(inst_num, errmsg, errflg)', self.text, + 'call test_simple_init(inst_num, ninstances, errmsg, errflg)', + self.text, ) - def test_register_signature_has_instance_number(self): + def test_register_signature_has_instance_pair(self): self.assertIn( - 'subroutine ccpp_register(suite_name, errflg, errmsg, inst_num)', + 'subroutine ccpp_register(suite_name, errflg, errmsg, inst_num, ninstances)', self.text, ) - def test_final_signature_has_instance_number(self): + def test_final_signature_has_instance_pair(self): + # Final carries (inst_num, ninstances) for API symmetry with + # register/init even though the framework doesn't read + # ninstances at final time. self.assertIn( - 'subroutine ccpp_final(suite_name, errflg, errmsg, inst_num)', + 'subroutine ccpp_final(suite_name, errflg, errmsg, inst_num, ninstances)', self.text, ) @@ -799,11 +803,6 @@ def setUp(self): units = count dimensions = () type = integer -[ ninstances ] - standard_name = number_of_instances - units = count - dimensions = () - type = integer [ flag_passive ] standard_name = flag_for_passive_check units = flag diff --git a/unit-tests/test_variable_resolver.py b/unit-tests/test_variable_resolver.py index a06de790..414fe0d2 100644 --- a/unit-tests/test_variable_resolver.py +++ b/unit-tests/test_variable_resolver.py @@ -558,12 +558,13 @@ def test_plain_host_vars(self): d = build_flat_host_dict(host_tables, [], []) self.assertIn('horizontal_dimension', d) self.assertIn('vertical_layer_dimension', d) - # number_of_instances is a host-table var (USE'd by the suite cap) - self.assertIn('number_of_instances', d) + # number_of_instances is now a control-table var (paired with + # instance_number), so it is NOT in a host-only dictionary. + self.assertNotIn('number_of_instances', d) # loop bounds and error vars live in the control table, not the host table self.assertNotIn('horizontal_loop_begin', d) self.assertNotIn('horizontal_loop_end', d) - self.assertEqual(len(d), 3) + self.assertEqual(len(d), 2) def test_plain_host_access_paths(self): host_tables = _parse_file('host_simple.meta') From 09095242f075e0e553e58dd7f7e081ddae07a17e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 May 2026 14:44:42 -0600 Subject: [PATCH 20/74] Bug fixes for NEPTUNE, rename ccpp_static_api --- capgen-ng/ccpp_capgen_ng.py | 31 ++- capgen-ng/generator/static_api.py | 31 ++- capgen-ng/generator/suite_resolver.py | 14 +- end-to-end-tests/advection/CMakeLists.txt | 3 +- end-to-end-tests/advection/test_host.F90 | 38 ++-- end-to-end-tests/capgen_ng/CMakeLists.txt | 4 +- end-to-end-tests/capgen_ng/test_host.F90 | 22 +- end-to-end-tests/chunked_data/CMakeLists.txt | 1 - end-to-end-tests/chunked_data/main.F90 | 2 +- end-to-end-tests/cmake/ccpp_capgen.cmake | 22 +- end-to-end-tests/ddthost/CMakeLists.txt | 2 - end-to-end-tests/ddthost/test_host.F90 | 22 +- end-to-end-tests/instances/CMakeLists.txt | 1 - end-to-end-tests/instances/main.F90 | 2 +- end-to-end-tests/nested_suite/CMakeLists.txt | 2 - end-to-end-tests/nested_suite/test_host.F90 | 22 +- end-to-end-tests/opt_arg/CMakeLists.txt | 5 +- end-to-end-tests/opt_arg/main.F90 | 2 +- end-to-end-tests/var_compat/CMakeLists.txt | 3 +- end-to-end-tests/var_compat/test_host.F90 | 22 +- unit-tests/test_ccpp_datafile.py | 3 +- unit-tests/test_control_validation.py | 13 -- unit-tests/test_datatable.py | 13 +- unit-tests/test_integration.py | 28 +-- unit-tests/test_static_api.py | 44 ++-- unit-tests/test_suite_resolver.py | 218 +++++++++++++++++++ 26 files changed, 402 insertions(+), 168 deletions(-) diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index 32754908..d0946f6d 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -183,7 +183,13 @@ def _build_arg_parser() -> argparse.ArgumentParser: '--host-name', required=True, metavar='NAME', - help='Host model identifier (used in generated subroutine names)', + help=( + 'Host model identifier. Drives the file and module name ' + 'of the generated static-API cap (``_ccpp_cap.F90`` / ' + '``module _ccpp_cap``) so multiple host integrations ' + 'can co-exist in one executable, and is written into ' + '``datatable.xml`` as the host var-dictionary name.' + ), ) parser.add_argument( '--host-files', @@ -669,7 +675,9 @@ def _validate_required_control_vars( Parameters ---------- host_name : str - Host model identifier, used in error messages. + Host model identifier, used in error messages so the developer + can tell which host the failure refers to when more than one + capgen invocation is in flight. host_dict : dict Flat host variable dictionary built by :func:`build_flat_host_dict`. @@ -691,15 +699,15 @@ def _check_control_var(std_name, expected_type, description, required: bool) -> "Required control variable '{}' not found in host '{}' " "type=control metadata.\n" " This variable {}. Add it to a " - "[ccpp-table-properties] / type=control block in your " + "[ccpp-table-properties] / type=control block in the " "host metadata files.".format(std_name, host_name, description) ) return if not entry.is_control: errors.append( - "Variable '{}' must be declared in a type=control table for " - "host '{}', but it was found in a type=host table.\n" + "Variable '{}' must be declared in a type=control table " + "for host '{}', but it was found in a type=host table.\n" " Move it to a [ccpp-table-properties] / type=control " "block.".format(std_name, host_name) ) @@ -751,9 +759,7 @@ def _check_control_var(std_name, expected_type, description, required: bool) -> "missing the paired variable '{}' (which must also be in a " "type=control table).\n" " Declare both for a multi-instance API, or neither for a " - "single-instance API.".format( - host_name, present, missing, - ) + "single-instance API.".format(host_name, present, missing) ) if errors: @@ -788,7 +794,9 @@ def capgen( Parameters ---------- host_name : str - Host model identifier. + Host model identifier. Drives the file and module name of the + generated static-API cap (``_ccpp_cap.F90`` / ``module + _ccpp_cap``) and is written into ``datatable.xml``. host_files : list of str Host metadata (``.meta``) file paths. scheme_files : list of str @@ -936,7 +944,8 @@ def capgen( # ---- static API (one file for all suites) ------------------------------ write_static_api( - suite_names, suite_resolutions, output_root, host_dict, scheme_store, + host_name, suite_names, suite_resolutions, output_root, + host_dict, scheme_store, logger=log, no_host_introspection=no_host_introspection, trace=trace, @@ -961,7 +970,7 @@ def capgen( # framework F90 dependencies so the host build picks them up. utility_paths.extend(_resolve_framework_f90_files()) host_file_paths = [ - os.path.join(abs_root, 'ccpp_static_api.F90'), + os.path.join(abs_root, '{}_ccpp_cap.F90'.format(host_name)), ] suite_file_paths = [] suite_meta_paths = [] diff --git a/capgen-ng/generator/static_api.py b/capgen-ng/generator/static_api.py index 3f64edb6..65aa3e7d 100644 --- a/capgen-ng/generator/static_api.py +++ b/capgen-ng/generator/static_api.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""Generate the static API module ``ccpp_static_api.F90``. +"""Generate the static API module ``_ccpp_cap.F90``. The static API module is generated once per build (not per suite) and provides the canonical public entry points that a host model calls: @@ -971,6 +971,7 @@ def _suite_io_subroutine( ######################################################################## def _generate_static_api( + host_name: str, suite_names: List[str], suite_resolutions: List[SuiteResolution], host_dict=None, @@ -978,10 +979,13 @@ def _generate_static_api( no_host_introspection: bool = False, trace: bool = False, ) -> List[str]: - """Generate the full ``ccpp_static_api.F90`` module source lines. + """Generate the full ``_ccpp_cap.F90`` module source lines. Parameters ---------- + host_name : str + Host identifier; drives the emitted module name + (``module _ccpp_cap``) and the comment header. suite_names : list of str Suite names in order. suite_resolutions : list of SuiteResolution @@ -1005,11 +1009,13 @@ def _generate_static_api( 'suite_names and suite_resolutions must have the same length' ) + mod_name = '{}_ccpp_cap'.format(host_name) + lines: List[str] = [] lines.append( - '! ccpp_static_api.F90 -- generated by ccpp_capgen_ng, do not edit' + '! {}.F90 -- generated by ccpp_capgen_ng, do not edit'.format(mod_name) ) - lines.append('module ccpp_static_api') + lines.append('module {}'.format(mod_name)) lines.append('') # Collect USE lines into a list so the trace helper can guarantee @@ -1032,7 +1038,7 @@ def _generate_static_api( lines.extend(use_lines) # Re-export the host-facing constituent API + the constituent object so - # host code can do ``use ccpp_static_api, only: ...`` for *everything* + # host code can do ``use _ccpp_cap, only: ...`` for *everything* # it needs from CCPP. Mirrors original capgen, which put all of these # on the generated host cap module. Only emitted when any suite uses # constituent state (the ccpp_host_constituents module is only emitted @@ -1118,7 +1124,7 @@ def _generate_static_api( )) lines.append('') - lines.append('end module ccpp_static_api') + lines.append('end module {}'.format(mod_name)) return lines @@ -1127,6 +1133,7 @@ def _generate_static_api( ######################################################################## def write_static_api( + host_name: str, suite_names: List[str], suite_resolutions: List[SuiteResolution], output_root: str, @@ -1136,10 +1143,14 @@ def write_static_api( no_host_introspection: bool = False, trace: bool = False, ) -> str: - """Write ``ccpp_static_api.F90`` to *output_root*. + """Write ``_ccpp_cap.F90`` to *output_root*. Parameters ---------- + host_name : str + Host identifier; drives both the file name + (``_ccpp_cap.F90``) and the emitted module name + (``module _ccpp_cap``). suite_names : list of str suite_resolutions : list of SuiteResolution Parallel to suite_names. @@ -1160,7 +1171,7 @@ def write_static_api( clear ``errmsg`` (or write to ``error_unit`` for ``ccpp_physics_suite_list``, which has no error channel). Signatures remain so existing callers still link. Use this to - shrink ``ccpp_static_api.F90`` from ~33k lines to ~800 for + shrink ``_ccpp_cap.F90`` from ~33k lines to ~800 for multi-suite builds where the introspection case-blocks make ``-O3`` compilation impractical. @@ -1170,11 +1181,11 @@ def write_static_api( Absolute path of the written file. """ os.makedirs(output_root, exist_ok=True) - filename = 'ccpp_static_api.F90' + filename = '{}_ccpp_cap.F90'.format(host_name) out_path = os.path.join(output_root, filename) lines = _generate_static_api( - suite_names, suite_resolutions, host_dict, scheme_store, + host_name, suite_names, suite_resolutions, host_dict, scheme_store, no_host_introspection=no_host_introspection, trace=trace, ) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 6a102969..3c268329 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -595,11 +595,19 @@ def _build_merged_subscript( # (e.g. ``q(:,:,index_of_)`` where index_of_X lives # on a DDT) resolve to the full DDT walk, not the bare # leaf name. Identical to ``local_name`` for plain - # module-level host vars. - parts.append(entry.access_path) + # module-level host vars. The DDT walk bakes registered + # scalar-index std-name placeholders (e.g. + # ``GFS_Control(instance_number)%ntqv``) into the + # access path; rewrite them to host local names here so + # the inner ``(instance_number)`` doesn't leak through + # to the emitted Fortran when this access path is itself + # used as a subscript token. + parts.append(_substitute_scalar_idx(entry.access_path, host_dict)) used.add(key) elif suite_vars and key in suite_vars: - parts.append(suite_vars[key].access_path) + parts.append( + _substitute_scalar_idx(suite_vars[key].access_path, host_dict) + ) used.add(key) else: raise CCPPError( diff --git a/end-to-end-tests/advection/CMakeLists.txt b/end-to-end-tests/advection/CMakeLists.txt index 0a3132ab..6f10b58b 100644 --- a/end-to-end-tests/advection/CMakeLists.txt +++ b/end-to-end-tests/advection/CMakeLists.txt @@ -8,8 +8,7 @@ set(SCHEME_FILES "cld_liq" "cld_ice" "apply_constituent_tendencies" "const_indices") set(HOST_FILES "test_host_data" "test_host_mod" "test_host") set(SUITE_FILES "cld_suite.xml") -set(HOST "test_host") - +set(HOST "test_host") # By default, generated caps go in ccpp subdir set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") diff --git a/end-to-end-tests/advection/test_host.F90 b/end-to-end-tests/advection/test_host.F90 index f02456e3..ed4cf557 100644 --- a/end-to-end-tests/advection/test_host.F90 +++ b/end-to-end-tests/advection/test_host.F90 @@ -49,8 +49,8 @@ subroutine check_errflg(subname, errflg, errmsg, errflg_final) end subroutine check_errflg logical function check_suite(test_suite) - use ccpp_static_api, only: ccpp_physics_suite_part_list - use ccpp_static_api, only: ccpp_physics_suite_variables + use test_host_ccpp_cap, only: ccpp_physics_suite_part_list + use test_host_ccpp_cap, only: ccpp_physics_suite_variables use test_utils, only: check_list ! Dummy argument @@ -148,23 +148,23 @@ subroutine test_host(retval, test_suites) std_name_array, & const_std_name use test_host_data, only: check_constituent_indices - use ccpp_static_api, only: ccpp_deallocate_dynamic_constituents - use ccpp_static_api, only: ccpp_register_constituents - use ccpp_static_api, only: ccpp_is_scheme_constituent - use ccpp_static_api, only: ccpp_initialize_constituents - use ccpp_static_api, only: ccpp_number_constituents - use ccpp_static_api, only: ccpp_constituents_array - use ccpp_static_api, only: ccpp_register - use ccpp_static_api, only: ccpp_init - use ccpp_static_api, only: ccpp_physics_init - use ccpp_static_api, only: ccpp_physics_timestep_init - use ccpp_static_api, only: ccpp_physics_run - use ccpp_static_api, only: ccpp_physics_timestep_final - use ccpp_static_api, only: ccpp_physics_final - use ccpp_static_api, only: ccpp_final - use ccpp_static_api, only: ccpp_physics_suite_list - use ccpp_static_api, only: ccpp_const_get_index - use ccpp_static_api, only: ccpp_model_const_properties + use test_host_ccpp_cap, only: ccpp_deallocate_dynamic_constituents + use test_host_ccpp_cap, only: ccpp_register_constituents + use test_host_ccpp_cap, only: ccpp_is_scheme_constituent + use test_host_ccpp_cap, only: ccpp_initialize_constituents + use test_host_ccpp_cap, only: ccpp_number_constituents + use test_host_ccpp_cap, only: ccpp_constituents_array + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final + use test_host_ccpp_cap, only: ccpp_physics_suite_list + use test_host_ccpp_cap, only: ccpp_const_get_index + use test_host_ccpp_cap, only: ccpp_model_const_properties use test_utils, only: check_list type(suite_info), intent(in) :: test_suites(:) diff --git a/end-to-end-tests/capgen_ng/CMakeLists.txt b/end-to-end-tests/capgen_ng/CMakeLists.txt index a1e8b8ae..74f57bbd 100644 --- a/end-to-end-tests/capgen_ng/CMakeLists.txt +++ b/end-to-end-tests/capgen_ng/CMakeLists.txt @@ -8,9 +8,8 @@ set(SCHEME_FILES "setup_coeffs" "temp_set" "temp_adjust" "temp_calc_adjust" "make_ddt" "environ_conditions") set(HOST_FILES "test_host_data" "test_host_mod" "test_host") set(SUITE_FILES "ddt_suite.xml" "temp_suite.xml") -set(KIND_TYPE "kind_phys=REAL64") set(HOST "test_host") - +set(KIND_TYPE "kind_phys=REAL64") # By default, generated caps go in ccpp subdir set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") @@ -44,7 +43,6 @@ ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} SCHEMEFILES ${SCHEME_METADATA_FILES} SUITES ${SUITE_FILES} HOST_NAME ${HOST} - KIND_TYPES ${KIND_TYPES} OUTPUT_ROOT "${OUTPUT_ROOT}") # Retrieve the list of Fortran files required for test host from datatable.xml; diff --git a/end-to-end-tests/capgen_ng/test_host.F90 b/end-to-end-tests/capgen_ng/test_host.F90 index 409338a3..43f01aa0 100644 --- a/end-to-end-tests/capgen_ng/test_host.F90 +++ b/end-to-end-tests/capgen_ng/test_host.F90 @@ -25,8 +25,8 @@ module test_prog contains logical function check_suite(test_suite) - use ccpp_static_api, only: ccpp_physics_suite_part_list - use ccpp_static_api, only: ccpp_physics_suite_variables + use test_host_ccpp_cap, only: ccpp_physics_suite_part_list + use test_host_ccpp_cap, only: ccpp_physics_suite_variables use test_utils, only: check_list ! Dummy argument @@ -108,15 +108,15 @@ subroutine test_host(retval, test_suites) #endif use test_host_mod, only: ncols, & num_time_steps - use ccpp_static_api, only: ccpp_register - use ccpp_static_api, only: ccpp_init - use ccpp_static_api, only: ccpp_physics_init - use ccpp_static_api, only: ccpp_physics_timestep_init - use ccpp_static_api, only: ccpp_physics_run - use ccpp_static_api, only: ccpp_physics_timestep_final - use ccpp_static_api, only: ccpp_physics_final - use ccpp_static_api, only: ccpp_final - use ccpp_static_api, only: ccpp_physics_suite_list + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final + use test_host_ccpp_cap, only: ccpp_physics_suite_list use test_host_mod, only: init_data, & compare_data, & check_model_times diff --git a/end-to-end-tests/chunked_data/CMakeLists.txt b/end-to-end-tests/chunked_data/CMakeLists.txt index 69fb5b90..894e6833 100644 --- a/end-to-end-tests/chunked_data/CMakeLists.txt +++ b/end-to-end-tests/chunked_data/CMakeLists.txt @@ -9,7 +9,6 @@ set(SCHEME_FILES "chunked_data_scheme") set(HOST_FILES "data" "main") set(SUITE_FILES "suite_chunked_data_suite.xml") set(HOST "test_host") - # By default, generated caps go in ccpp subdir set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") diff --git a/end-to-end-tests/chunked_data/main.F90 b/end-to-end-tests/chunked_data/main.F90 index 5bea870d..0845cfd2 100644 --- a/end-to-end-tests/chunked_data/main.F90 +++ b/end-to-end-tests/chunked_data/main.F90 @@ -11,7 +11,7 @@ program test_chunked_data use data, only: chunked_data_type, & chunked_data_instance - use ccpp_static_api, only: ccpp_register, & + use test_host_ccpp_cap, only: ccpp_register, & ccpp_init, & ccpp_physics_init, & ccpp_physics_timestep_init, & diff --git a/end-to-end-tests/cmake/ccpp_capgen.cmake b/end-to-end-tests/cmake/ccpp_capgen.cmake index 0b179bc6..e344bf9b 100644 --- a/end-to-end-tests/cmake/ccpp_capgen.cmake +++ b/end-to-end-tests/cmake/ccpp_capgen.cmake @@ -56,12 +56,18 @@ endfunction() # CMake wrapper for ccpp_capgen_ng.py # # TRACE - ON/OFF (Default: OFF) - Add --trace flag to capgen call -# HOST_NAME - String name of host +# HOST_NAME - String name of host (drives _ccpp_cap.F90 filename +# and module name; required) # OUTPUT_ROOT - String path to put generated caps # VERBOSITY - Number of --verbose flags to pass to capgen # HOSTFILES - CMake list of host metadata filenames # SCHEMEFILES - CMake list of scheme metadata files # SUITES - CMake list of suite xml files +# KIND_SPECS - Comma-separated kind mappings, e.g. "kind_phys=REAL32" or +# "kind_phys=my_mod:kind_r4,kind_dyn=REAL64". Each pair is +# forwarded as `--kind-type ` to capgen-ng (see the +# capgen-ng docstring for the `=[:]` +# grammar; bare ISO specs default to iso_fortran_env). function(ccpp_capgen) set(optionalArgs TRACE) set(oneValueArgs HOST_NAME OUTPUT_ROOT VERBOSITY KIND_SPECS) @@ -95,7 +101,7 @@ function(ccpp_capgen) list(APPEND CCPP_CAPGEN_CMD_LIST "--suites" "${SUITES_SEPARATED}") if(NOT DEFINED arg_HOST_NAME) - message(FATAL_ERROR "function(ccpp_capgen): HOSTNAME not set.") + message(FATAL_ERROR "function(ccpp_capgen): HOST_NAME not set.") endif() list(APPEND CCPP_CAPGEN_CMD_LIST "--host-name" "${arg_HOST_NAME}") @@ -112,16 +118,14 @@ function(ccpp_capgen) endif() if(DEFINED arg_KIND_SPECS) + # Accept either a comma-separated string ("kind_phys=REAL64,kind_dyn=REAL32") + # or a CMake list of pairs. Each pair becomes a separate + # `--kind-type ` argv pair so capgen-ng's argparse sees one + # `--kind-type` per pair (the flag is `action='append'`). string(REPLACE "," ";" KIND_SPEC_LIST "${arg_KIND_SPECS}") - set(KIND_ARGS "") # start empty foreach(pair IN LISTS KIND_SPEC_LIST) - # Append each pair prefixed with --kind-type and quoted. - # The surrounding double‑quotes are added explicitly so the - # resulting string contains them. - set(KIND_ARGS "${KIND_ARGS}--kind-type \"${pair}\"") - string(STRIP "${KIND_ARGS}" KIND_ARGS) + list(APPEND CCPP_CAPGEN_CMD_LIST "--kind-type" "${pair}") endforeach() - list(APPEND CCPP_CAPGEN_CMD_LIST ${KIND_SPEC_PARAMS}) endif() if(arg_TRACE) diff --git a/end-to-end-tests/ddthost/CMakeLists.txt b/end-to-end-tests/ddthost/CMakeLists.txt index 58d9b184..a07d795c 100644 --- a/end-to-end-tests/ddthost/CMakeLists.txt +++ b/end-to-end-tests/ddthost/CMakeLists.txt @@ -9,7 +9,6 @@ set(SCHEME_FILES "environ_conditions" "setup_coeffs" "temp_adjust" "temp_calc_ad set(HOST_FILES "host_ccpp_ddt" "test_host_data" "test_host_mod" "test_host") set(SUITE_FILES "ddt_suite.xml" "temp_suite.xml") set(HOST "test_host") - # By default, generated caps go in ccpp subdir set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") @@ -33,7 +32,6 @@ ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} SCHEMEFILES ${SCHEME_METADATA_FILES} SUITES ${SUITE_FILES} HOST_NAME ${HOST} - KIND_TYPES ${KIND_TYPES} OUTPUT_ROOT "${OUTPUT_ROOT}") # Retrieve the list of Fortran files required for test host from datatable.xml; diff --git a/end-to-end-tests/ddthost/test_host.F90 b/end-to-end-tests/ddthost/test_host.F90 index ef25b4cb..9b9b6204 100644 --- a/end-to-end-tests/ddthost/test_host.F90 +++ b/end-to-end-tests/ddthost/test_host.F90 @@ -25,8 +25,8 @@ module test_prog contains logical function check_suite(test_suite) - use ccpp_static_api, only: ccpp_physics_suite_part_list - use ccpp_static_api, only: ccpp_physics_suite_variables + use test_host_ccpp_cap, only: ccpp_physics_suite_part_list + use test_host_ccpp_cap, only: ccpp_physics_suite_variables use test_utils, only: check_list ! Dummy argument @@ -105,15 +105,15 @@ subroutine test_host(retval, test_suites) use test_host_mod, only: ncols, & num_time_steps - use ccpp_static_api, only: ccpp_register - use ccpp_static_api, only: ccpp_init - use ccpp_static_api, only: ccpp_physics_init - use ccpp_static_api, only: ccpp_physics_timestep_init - use ccpp_static_api, only: ccpp_physics_run - use ccpp_static_api, only: ccpp_physics_timestep_final - use ccpp_static_api, only: ccpp_physics_final - use ccpp_static_api, only: ccpp_final - use ccpp_static_api, only: ccpp_physics_suite_list + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final + use test_host_ccpp_cap, only: ccpp_physics_suite_list use test_host_mod, only: init_data, & compare_data, & check_model_times diff --git a/end-to-end-tests/instances/CMakeLists.txt b/end-to-end-tests/instances/CMakeLists.txt index 02a1fd3b..1b995067 100644 --- a/end-to-end-tests/instances/CMakeLists.txt +++ b/end-to-end-tests/instances/CMakeLists.txt @@ -9,7 +9,6 @@ set(SCHEME_FILES "unit_conv_scheme_1" "unit_conv_scheme_2") set(HOST_FILES "data" "main") set(SUITE_FILES "suite_unit_conv_suite.xml") set(HOST "test_host") - # By default, generated caps go in ccpp subdir set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") diff --git a/end-to-end-tests/instances/main.F90 b/end-to-end-tests/instances/main.F90 index 45643507..2d34bb51 100644 --- a/end-to-end-tests/instances/main.F90 +++ b/end-to-end-tests/instances/main.F90 @@ -9,7 +9,7 @@ program test_unit_conv nspecies, ninstances use data, only: instance_data - use ccpp_static_api, only: ccpp_register, & + use test_host_ccpp_cap, only: ccpp_register, & ccpp_init, & ccpp_physics_init, & ccpp_physics_timestep_init, & diff --git a/end-to-end-tests/nested_suite/CMakeLists.txt b/end-to-end-tests/nested_suite/CMakeLists.txt index c050f81a..f5f21345 100644 --- a/end-to-end-tests/nested_suite/CMakeLists.txt +++ b/end-to-end-tests/nested_suite/CMakeLists.txt @@ -9,7 +9,6 @@ set(SCHEME_FILES "effr_calc" "effrs_calc" "effr_diag" "effr_pre" "effr_post" "ra set(HOST_FILES "module_rad_ddt" "test_host_data" "test_host_mod" "test_host") set(SUITE_FILES "main_suite.xml") set(HOST "test_host") - # By default, generated caps go in ccpp subdir set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") @@ -33,7 +32,6 @@ ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} SCHEMEFILES ${SCHEME_METADATA_FILES} SUITES ${SUITE_FILES} HOST_NAME ${HOST} - KIND_TYPES ${KIND_TYPES} OUTPUT_ROOT "${OUTPUT_ROOT}") # Retrieve the list of Fortran files required for test host from datatable.xml; diff --git a/end-to-end-tests/nested_suite/test_host.F90 b/end-to-end-tests/nested_suite/test_host.F90 index 9871a827..0e3bade7 100644 --- a/end-to-end-tests/nested_suite/test_host.F90 +++ b/end-to-end-tests/nested_suite/test_host.F90 @@ -25,8 +25,8 @@ module test_prog contains logical function check_suite(test_suite) - use ccpp_static_api, only: ccpp_physics_suite_part_list - use ccpp_static_api, only: ccpp_physics_suite_variables + use test_host_ccpp_cap, only: ccpp_physics_suite_part_list + use test_host_ccpp_cap, only: ccpp_physics_suite_variables use test_utils, only: check_list ! Dummy argument @@ -104,15 +104,15 @@ end function check_suite subroutine test_host(retval, test_suites) use test_host_mod, only: ncols - use ccpp_static_api, only: ccpp_register - use ccpp_static_api, only: ccpp_init - use ccpp_static_api, only: ccpp_physics_init - use ccpp_static_api, only: ccpp_physics_timestep_init - use ccpp_static_api, only: ccpp_physics_run - use ccpp_static_api, only: ccpp_physics_timestep_final - use ccpp_static_api, only: ccpp_physics_final - use ccpp_static_api, only: ccpp_final - use ccpp_static_api, only: ccpp_physics_suite_list + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final + use test_host_ccpp_cap, only: ccpp_physics_suite_list use test_host_mod, only: init_data, & compare_data use test_utils, only: check_list diff --git a/end-to-end-tests/opt_arg/CMakeLists.txt b/end-to-end-tests/opt_arg/CMakeLists.txt index 83f68a9d..66d27712 100644 --- a/end-to-end-tests/opt_arg/CMakeLists.txt +++ b/end-to-end-tests/opt_arg/CMakeLists.txt @@ -9,7 +9,6 @@ set(SCHEME_FILES "opt_arg_scheme") set(HOST_FILES "data" "main") set(SUITE_FILES "suite_opt_arg_suite.xml") set(HOST "test_host") - # By default, generated caps go in ccpp subdir set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") @@ -27,12 +26,14 @@ ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} METADATA_FILES ${HOST_METADATA_FILES}) -# Run ccpp_capgen_ng +# Run ccpp_capgen_ng. Override kind_phys to REAL32 so the whole test +# runs in single precision; exercises the --kind-type plumbing. ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} SCHEMEFILES ${SCHEME_METADATA_FILES} SUITES ${SUITE_FILES} HOST_NAME ${HOST} + KIND_SPECS "kind_phys=REAL32" OUTPUT_ROOT "${OUTPUT_ROOT}") # Retrieve the list of Fortran files required for test host from datatable.xml; diff --git a/end-to-end-tests/opt_arg/main.F90 b/end-to-end-tests/opt_arg/main.F90 index b9928052..25701be4 100644 --- a/end-to-end-tests/opt_arg/main.F90 +++ b/end-to-end-tests/opt_arg/main.F90 @@ -9,7 +9,7 @@ program test_opt_arg opt_arg, & opt_arg_2 - use ccpp_static_api, only: ccpp_register, & + use test_host_ccpp_cap, only: ccpp_register, & ccpp_init, & ccpp_physics_init, & ccpp_physics_timestep_init, & diff --git a/end-to-end-tests/var_compat/CMakeLists.txt b/end-to-end-tests/var_compat/CMakeLists.txt index 6c2bb8b7..6a31d273 100644 --- a/end-to-end-tests/var_compat/CMakeLists.txt +++ b/end-to-end-tests/var_compat/CMakeLists.txt @@ -8,8 +8,7 @@ set(SCHEME_FILES "effr_calc" "effrs_calc" "effr_diag" "effr_pre" "effr_post" "rad_lw" "rad_sw") set(HOST_FILES "module_rad_ddt" "test_host_data" "test_host_mod" "test_host") set(SUITE_FILES "var_compatibility_suite.xml") -set(HOST "test_host") - +set(HOST "test_host") # By default, generated caps go in ccpp subdir set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") diff --git a/end-to-end-tests/var_compat/test_host.F90 b/end-to-end-tests/var_compat/test_host.F90 index 0d691174..d88c1d24 100644 --- a/end-to-end-tests/var_compat/test_host.F90 +++ b/end-to-end-tests/var_compat/test_host.F90 @@ -25,8 +25,8 @@ module test_prog contains logical function check_suite(test_suite) - use ccpp_static_api, only: ccpp_physics_suite_part_list - use ccpp_static_api, only: ccpp_physics_suite_variables + use test_host_ccpp_cap, only: ccpp_physics_suite_part_list + use test_host_ccpp_cap, only: ccpp_physics_suite_variables use test_utils, only: check_list ! Dummy argument @@ -104,15 +104,15 @@ end function check_suite subroutine test_host(retval, test_suites) use test_host_mod, only: ncols - use ccpp_static_api, only: ccpp_register - use ccpp_static_api, only: ccpp_init - use ccpp_static_api, only: ccpp_physics_init - use ccpp_static_api, only: ccpp_physics_timestep_init - use ccpp_static_api, only: ccpp_physics_run - use ccpp_static_api, only: ccpp_physics_timestep_final - use ccpp_static_api, only: ccpp_physics_final - use ccpp_static_api, only: ccpp_final - use ccpp_static_api, only: ccpp_physics_suite_list + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final + use test_host_ccpp_cap, only: ccpp_physics_suite_list use test_host_mod, only: init_data, & compare_data use test_utils, only: check_list diff --git a/unit-tests/test_ccpp_datafile.py b/unit-tests/test_ccpp_datafile.py index e66dc345..937f3ae6 100644 --- a/unit-tests/test_ccpp_datafile.py +++ b/unit-tests/test_ccpp_datafile.py @@ -35,7 +35,7 @@ from generator.suite_resolver import resolve_suite -def _build_datatable(tmpdir, host_name='test_host', +def _build_datatable(tmpdir, host_file_paths=None, utility_paths=None, suite_file_paths=None, scheme_file_paths=None, dependency_paths=None, @@ -62,7 +62,6 @@ def _build_datatable(tmpdir, host_name='test_host', suite_meta_paths=suite_meta_paths, expanded_sdf_paths=expanded_sdf_paths, host_dict=hd, - host_name=host_name, ) diff --git a/unit-tests/test_control_validation.py b/unit-tests/test_control_validation.py index e5f3ba1d..ebc6104b 100644 --- a/unit-tests/test_control_validation.py +++ b/unit-tests/test_control_validation.py @@ -249,19 +249,6 @@ def test_ninstances_alone_raises(self): self.assertIn('number_of_instances', msg) self.assertIn('paired', msg.lower()) - def test_host_name_in_error_message(self): - """Error message names the offending host so the developer knows which one.""" - host_dict = _build_host_dict( - host_files=[_sf('host_simple.meta')], - control_files=[_sf('bad_ctrl_missing_vars.meta')], - ) - try: - _validate_required_control_vars('my_special_host', host_dict) - self.fail("Expected CCPPError") - except CCPPError as exc: - self.assertIn('my_special_host', str(exc)) - - # --------------------------------------------------------------------------- # Tests for forbidden dimension names # --------------------------------------------------------------------------- diff --git a/unit-tests/test_datatable.py b/unit-tests/test_datatable.py index a1811296..a6c203be 100644 --- a/unit-tests/test_datatable.py +++ b/unit-tests/test_datatable.py @@ -24,8 +24,7 @@ def _resolve(suite_xml='suite_test_simple.xml'): def _write(tmpdir, suite_xml='suite_test_simple.xml', utility_paths=None, - suite_file_paths=None, host_file_paths=None, host_dict=None, - host_name='test_host', suite_meta_paths=None, + suite_file_paths=None, host_file_paths=None, host_dict=None, suite_meta_paths=None, expanded_sdf_paths=None): suite_resolution, store = _resolve(suite_xml) return ( @@ -39,7 +38,6 @@ def _write(tmpdir, suite_xml='suite_test_simple.xml', utility_paths=None, suite_meta_paths=suite_meta_paths, expanded_sdf_paths=expanded_sdf_paths, host_dict=host_dict, - host_name=host_name, ), suite_resolution, store, @@ -489,7 +487,6 @@ def setUp(self): self._path, self._sr, _ = _write( self._tmpdir, host_dict=self._hd, - host_name='test_host', ) self._root = ET.parse(self._path).getroot() @@ -516,7 +513,7 @@ def test_host_dictionary_present(self): None, ) self.assertIsNotNone(host_d) - self.assertEqual(host_d.get('name'), 'test_host') + self.assertEqual(host_d.get('name'), 'host') def test_host_dictionary_has_vars(self): host_d = next(vd for vd in self._vd().findall('var_dictionary') @@ -524,10 +521,12 @@ def test_host_dictionary_has_vars(self): names = {v.get('name') for v in host_d.find('variables').findall('var')} self.assertIn('air_temperature', names) - def test_api_dict_parent_is_host_name(self): + def test_api_dict_parent_is_host(self): api_d = next(vd for vd in self._vd().findall('var_dictionary') if vd.get('type') == 'api') - self.assertEqual(api_d.get('parent'), 'test_host') + # The 'host' string is a fixed internal label written by the + # generator; ccpp_datafile.py uses it only for the api->host walk. + self.assertEqual(api_d.get('parent'), 'host') def test_suite_dict_parent_is_api(self): suite_d = next(vd for vd in self._vd().findall('var_dictionary') diff --git a/unit-tests/test_integration.py b/unit-tests/test_integration.py index 85894712..60ca4acf 100644 --- a/unit-tests/test_integration.py +++ b/unit-tests/test_integration.py @@ -66,7 +66,7 @@ def _path(self, name): return os.path.join(self._tmpdir, name) def test_static_api_exists(self): - self.assertTrue(os.path.isfile(self._path('ccpp_static_api.F90'))) + self.assertTrue(os.path.isfile(self._path('test_host_ccpp_cap.F90'))) def test_suite_cap_exists(self): self.assertTrue(os.path.isfile(self._path('ccpp_test_simple_cap.F90'))) @@ -227,7 +227,7 @@ def test_cli_and_metadata_conflicting_kind_spec_raises(self): def _run_with_constituents(tmpdir, suite_xml='suite_consume_constituent.xml'): """Run capgen with a constituent-using fixture and return tmpdir.""" capgen( - host_name='host_consts', + host_name='test_host', host_files=[_sf('host_with_constituents.meta'), _sf('control_full.meta')], scheme_files=[_sf('scheme_consume_constituent.meta')], @@ -381,7 +381,7 @@ class TestStaticApiContent(unittest.TestCase): def setUp(self): self._tmpdir = tempfile.mkdtemp() _run_simple(self._tmpdir) - with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: self.text = fh.read() def tearDown(self): @@ -389,7 +389,7 @@ def tearDown(self): shutil.rmtree(self._tmpdir) def test_module_declaration(self): - self.assertIn('module ccpp_static_api', self.text) + self.assertIn('module test_host_ccpp_cap', self.text) def test_ccpp_register_always_present(self): # ccpp_register is mandatory in the new design and always emitted, @@ -525,7 +525,7 @@ def test_static_api_in_host_files(self): host_files = self._root.find('capgen_files').find('host_files') self.assertIsNotNone(host_files) names = [os.path.basename(f.text) for f in host_files.findall('file')] - self.assertIn('ccpp_static_api.F90', names) + self.assertIn('test_host_ccpp_cap.F90', names) def test_ccpp_kinds_in_utilities(self): # ccpp_kinds.F90 is always generated and must be discoverable by @@ -659,7 +659,7 @@ def test_both_suite_caps_exist(self): ) def test_static_api_dispatches_both(self): - with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: text = fh.read() self.assertIn("case('test_simple')", text) self.assertIn("case('test_subcycle')", text) @@ -697,7 +697,7 @@ def tearDown(self): def test_ccpp_init_minimal_signature(self): # Lifecycle signature for a multi-instance host carries the # paired (inst_num, ninstances) control vars. - with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: text = fh.read() self.assertIn( 'subroutine ccpp_init(suite_name, errflg, errmsg, inst_num, ninstances)', @@ -806,7 +806,7 @@ def tearDown(self): shutil.rmtree(self._tmpdir) def test_static_api_ccpp_init_omits_inst_num(self): - with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: text = fh.read() # ``suite_name_var`` is the local name declared by # ``control_no_instance.meta`` for the suite_name control var. @@ -818,7 +818,7 @@ def test_static_api_ccpp_init_omits_inst_num(self): self.assertNotIn('inst_num', text) def test_static_api_ccpp_register_omits_inst_num(self): - with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: text = fh.read() self.assertIn( 'subroutine ccpp_register(suite_name_var, errflg, errmsg)', @@ -826,7 +826,7 @@ def test_static_api_ccpp_register_omits_inst_num(self): ) def test_static_api_ccpp_final_omits_inst_num(self): - with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: text = fh.read() self.assertIn( 'subroutine ccpp_final(suite_name_var, errflg, errmsg)', @@ -922,7 +922,7 @@ def test_ninstances_alone_raises(self): def _run_opt_arg(tmpdir): capgen( - host_name='host', + host_name='test_host', host_files=[_sf('host_opt_arg.meta'), _sf('control_opt_arg.meta')], scheme_files=[_sf('scheme_opt_arg.meta')], suite_files=[_suite_file('suite_opt_arg.xml')], @@ -934,7 +934,7 @@ def _run_opt_arg(tmpdir): def _run_unit_conv(tmpdir): capgen( - host_name='host', + host_name='test_host', host_files=[_sf('host_unit_conv.meta'), _sf('control_unit_conv.meta')], scheme_files=[ _sf('scheme_unit_conv_1.meta'), @@ -949,7 +949,7 @@ def _run_unit_conv(tmpdir): def _run_chunked_data(tmpdir): capgen( - host_name='host', + host_name='test_host', host_files=[ _sf('host_chunked_data.meta'), _sf('ddt_chunked_data.meta'), @@ -2110,7 +2110,7 @@ def test_suite_cap_passes_inst_num_to_group_run(self): def test_static_api_physics_run_has_inst_num(self): """Static API ccpp_physics_run must include inst_num when groups need it.""" - with open(os.path.join(self._tmpdir, 'ccpp_static_api.F90')) as fh: + with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: text = fh.read() physics_run = text.split('subroutine ccpp_physics_run')[1] physics_run = physics_run.split('end subroutine')[0] diff --git a/unit-tests/test_static_api.py b/unit-tests/test_static_api.py index 48c78975..707fc90b 100644 --- a/unit-tests/test_static_api.py +++ b/unit-tests/test_static_api.py @@ -37,7 +37,7 @@ def _resolve(): def _generate(): suite_resolution = _resolve() - return _generate_static_api(['test_simple'], [suite_resolution]) + return _generate_static_api('test_host', ['test_simple'], [suite_resolution]) class TestAllCtrlArgsForPhase(unittest.TestCase): @@ -52,7 +52,7 @@ def test_only_error_ctrl_args_in_test_case(self): def test_mismatched_lengths_raises(self): from metadata.parse_tools import CCPPError with self.assertRaises(CCPPError): - _generate_static_api(['a', 'b'], [_resolve()]) + _generate_static_api('test_host', ['a', 'b'], [_resolve()]) class TestGenerateStaticApiModule(unittest.TestCase): @@ -65,11 +65,11 @@ def setUp(self): def test_module_header_comment(self): self.assertTrue(self.lines[0].startswith('!')) - self.assertIn('ccpp_static_api', self.lines[0]) + self.assertIn('test_host_ccpp_cap', self.lines[0]) def test_module_declaration(self): - self.assertIn('module ccpp_static_api', self.text) - self.assertIn('end module ccpp_static_api', self.text) + self.assertIn('module test_host_ccpp_cap', self.text) + self.assertIn('end module test_host_ccpp_cap', self.text) def test_does_not_use_constituent_mod(self): # Constituent merging is now opt-in via type=host (Task #6 follow-up). @@ -126,7 +126,7 @@ def setUp(self): suite = _parse_suite('suite_consume_constituent.xml') suite_resolution = resolve_suite(suite, store, hd) self.text = '\n'.join( - _generate_static_api(['consume_consts'], [suite_resolution], host_dict=hd, + _generate_static_api('test_host', ['consume_consts'], [suite_resolution], host_dict=hd, scheme_store=store), ) @@ -254,7 +254,7 @@ class TestCcppPhysicsUnknownSuiteErrors(unittest.TestCase): def setUp(self): hd = _load_full_host_dict() suite_resolution = _resolve() - self.text = '\n'.join(_generate_static_api(['test_simple'], [suite_resolution], hd)) + self.text = '\n'.join(_generate_static_api('test_host', ['test_simple'], [suite_resolution], hd)) def test_physics_run_has_default_case_with_errflg(self): run_block_start = self.text.index('subroutine ccpp_physics_run') @@ -282,7 +282,7 @@ def setUp(self): from copy import deepcopy sr2 = deepcopy(suite_resolution) sr2.suite_name = 'suite_b' - lines = _generate_static_api(['test_simple', 'suite_b'], [suite_resolution, sr2]) + lines = _generate_static_api('test_host', ['test_simple', 'suite_b'], [suite_resolution, sr2]) self.text = '\n'.join(lines) def test_both_suites_in_register(self): @@ -300,17 +300,17 @@ class TestWriteStaticApi(unittest.TestCase): def test_writes_file(self): suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: - path = write_static_api(['test_simple'], [suite_resolution], tmpdir) + path = write_static_api('test_host', ['test_simple'], [suite_resolution], tmpdir) self.assertTrue(os.path.isfile(path)) - self.assertEqual(os.path.basename(path), 'ccpp_static_api.F90') + self.assertEqual(os.path.basename(path), 'test_host_ccpp_cap.F90') def test_file_content(self): suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: - path = write_static_api(['test_simple'], [suite_resolution], tmpdir) + path = write_static_api('test_host', ['test_simple'], [suite_resolution], tmpdir) with open(path) as fh: content = fh.read() - self.assertIn('module ccpp_static_api', content) + self.assertIn('module test_host_ccpp_cap', content) # ccpp_register is now mandatory and always emitted. self.assertIn('subroutine ccpp_register', content) self.assertTrue(content.endswith('\n')) @@ -319,13 +319,13 @@ def test_creates_output_dir(self): suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: subdir = os.path.join(tmpdir, 'api') - write_static_api(['test_simple'], [suite_resolution], subdir) + write_static_api('test_host', ['test_simple'], [suite_resolution], subdir) self.assertTrue(os.path.isdir(subdir)) def test_returns_absolute_path(self): suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: - path = write_static_api(['test_simple'], [suite_resolution], tmpdir) + path = write_static_api('test_host', ['test_simple'], [suite_resolution], tmpdir) self.assertTrue(os.path.isabs(path)) @@ -336,7 +336,7 @@ class TestCcppInitMultiInstance(unittest.TestCase): def setUp(self): hd = _load_full_host_dict() suite_resolution = _resolve() - lines = _generate_static_api(['test_simple'], [suite_resolution], hd) + lines = _generate_static_api('test_host', ['test_simple'], [suite_resolution], hd) self.text = '\n'.join(lines) def test_init_signature_has_instance_pair(self): @@ -382,7 +382,7 @@ def setUp(self): hd = {k: v for k, v in _load_full_host_dict().items() if k not in ('number_of_instances', 'instance_number')} suite_resolution = _resolve() - lines = _generate_static_api(['test_simple'], [suite_resolution], hd) + lines = _generate_static_api('test_host', ['test_simple'], [suite_resolution], hd) self.text = '\n'.join(lines) def test_init_signature_no_instance_args(self): @@ -1205,6 +1205,7 @@ def test_module_imports_error_unit_unconditionally(self): # Stub-on and stub-off both include the same USE. for stub in (True, False): text = '\n'.join(_generate_static_api( + 'test_host', ['test_simple'], [self.suite_resolution], self.hd, no_host_introspection=stub, )) @@ -1218,6 +1219,7 @@ def test_public_declarations_unchanged_when_stubbed(self): # All five introspection routines remain public — callers must # still link against them. text = '\n'.join(_generate_static_api( + 'test_host', ['test_simple'], [self.suite_resolution], self.hd, no_host_introspection=True, )) @@ -1236,9 +1238,9 @@ def test_line_count_drops_dramatically(self): """The motivating case: 33k+ lines → ~800. We don't have 80 suites in unit-test fixtures, but even with one suite the stubbed module must be strictly shorter than the full one.""" - full = _generate_static_api(['test_simple'], [self.suite_resolution], + full = _generate_static_api('test_host', ['test_simple'], [self.suite_resolution], self.hd, no_host_introspection=False) - stub = _generate_static_api(['test_simple'], [self.suite_resolution], + stub = _generate_static_api('test_host', ['test_simple'], [self.suite_resolution], self.hd, no_host_introspection=True) self.assertLess(len(stub), len(full), 'stubbed module should be shorter than the full one ' @@ -1249,6 +1251,7 @@ def test_write_static_api_passes_flag_through(self): produce a file containing stub bodies, not full ones.""" with tempfile.TemporaryDirectory() as tmpdir: out_path = write_static_api( + 'test_host', ['test_simple'], [self.suite_resolution], tmpdir, self.hd, no_host_introspection=True, ) @@ -1274,6 +1277,7 @@ def setUp(self): def test_module_gate_default_off(self): text = '\n'.join(_generate_static_api( + 'test_host', ['test_simple'], [self.suite_resolution], self.hd, )) self.assertIn('logical, parameter :: trace = .false.', text) @@ -1281,6 +1285,7 @@ def test_module_gate_default_off(self): def test_module_gate_default_on(self): text = '\n'.join(_generate_static_api( + 'test_host', ['test_simple'], [self.suite_resolution], self.hd, trace=True, )) @@ -1289,6 +1294,7 @@ def test_module_gate_default_on(self): def test_trace_block_present_in_physics_phases(self): text = '\n'.join(_generate_static_api( + 'test_host', ['test_simple'], [self.suite_resolution], self.hd, )) # Every ccpp_physics_ dispatch has a gated write. @@ -1302,6 +1308,7 @@ def test_trace_block_present_in_physics_phases(self): def test_trace_block_present_in_lifecycle_routines(self): text = '\n'.join(_generate_static_api( + 'test_host', ['test_simple'], [self.suite_resolution], self.hd, )) for sub in ('ccpp_register', 'ccpp_init', 'ccpp_final'): @@ -1313,6 +1320,7 @@ def test_trace_block_present_in_lifecycle_routines(self): def test_write_static_api_threads_trace_flag(self): with tempfile.TemporaryDirectory() as tmpdir: out_path = write_static_api( + 'test_host', ['test_simple'], [self.suite_resolution], tmpdir, self.hd, trace=True, ) diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 308a7819..ebe1a835 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -566,6 +566,224 @@ def test_unknown_index_raises(self): 'run', hd, ) + def test_explicit_index_substitutes_scalar_idx_placeholder(self): + """Regression: an explicit subscript token like + ``index_of_water_vapor_specific_humidity`` whose ``access_path`` + carries a baked ``(instance_number)`` DDT-instance placeholder + must have that placeholder resolved to the host's local name + before being spliced into the subscript. Without the + substitution the generator emits Fortran like + ``qgrs(lb:ub, 1:nlev, GFS_Control(instance_number)%ntqv)`` and + the compiler rejects ``instance_number`` as untyped. Found + 2026-05-15 in the NEPTUNE phys_ps cap. + """ + from metadata.metadata_table import _parse_lines + ddt_src = ( + "[ccpp-table-properties]\n name = GFS_control_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_control_type\n type = ddt\n" + "[ ntqv ]\n" + " standard_name = index_of_water_vapor_specific_humidity\n" + " units = index\n dimensions = ()\n type = integer\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ GFS_Control ]\n standard_name = GFS_control_type_instance\n" + " units = DDT\n dimensions = (number_of_instances)\n" + " type = GFS_control_type\n" + "[ ncols ]\n standard_name = horizontal_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + "[ nlev ]\n standard_name = vertical_layer_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ instance ]\n standard_name = instance_number\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ ninstances ]\n standard_name = number_of_instances\n" + " units = count\n dimensions = ()\n type = integer\n intent = in\n" + ) + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + # Pre-condition: the inner ntqv entry's access path carries the + # baked ``(instance_number)`` placeholder. + self.assertEqual( + hd['index_of_water_vapor_specific_humidity'].access_path, + 'GFS_Control(instance_number)%ntqv', + ) + sub, _used = _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', 'index_of_water_vapor_specific_humidity'], + 'run', hd, + ) + # The emitted subscript must substitute the placeholder to the + # host's local name (``instance``). + self.assertEqual( + sub, '(lb:ub, 1:nlev, GFS_Control(instance)%ntqv)', + ) + self.assertNotIn('instance_number', sub) + + def test_explicit_index_nested_ddt_two_placeholder_levels(self): + """Recursive variant: the index token's access_path crosses TWO + DDT levels, each with its own registered scalar-index dim. The + baked path contains two distinct placeholders + (``(instance_number)`` outer + ``(thread_number)`` inner) plus + a third occurrence of one of them; ``_substitute_scalar_idx`` + must rewrite all of them in a single pass.""" + from metadata.metadata_table import _parse_lines + ddt_src = ( + # Innermost DDT — defines the leaf index variable. + "[ccpp-table-properties]\n name = scratch_type\n type = ddt\n" + "[ccpp-arg-table]\n name = scratch_type\n type = ddt\n" + "[ idx_qv ]\n" + " standard_name = index_of_water_vapor_specific_humidity\n" + " units = index\n dimensions = ()\n type = integer\n" + "\n" + # Middle DDT — sliced per OpenMP thread. + "[ccpp-table-properties]\n name = GFS_interstitial_type\n" + " type = ddt\n" + "[ccpp-arg-table]\n name = GFS_interstitial_type\n" + " type = ddt\n" + "[ scratch ]\n" + " standard_name = scratch_type_instance\n units = DDT\n" + " dimensions = ()\n type = scratch_type\n" + "\n" + # Outer DDT — sliced per model instance and itself carrying + # a per-thread Interstitial sub-DDT. + "[ccpp-table-properties]\n name = GFS_phys_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_phys_type\n type = ddt\n" + "[ Interstitial ]\n" + " standard_name = GFS_interstitial_type_instance\n" + " units = DDT\n dimensions = (number_of_threads)\n" + " type = GFS_interstitial_type\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ GFS_Phys ]\n standard_name = GFS_phys_type_instance\n" + " units = DDT\n dimensions = (number_of_instances)\n" + " type = GFS_phys_type\n" + "[ ncols ]\n standard_name = horizontal_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + "[ nlev ]\n standard_name = vertical_layer_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ mythread ]\n standard_name = thread_number\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ nthreads ]\n standard_name = number_of_threads\n" + " units = count\n dimensions = ()\n type = integer\n intent = in\n" + "[ instance ]\n standard_name = instance_number\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ ninstances ]\n standard_name = number_of_instances\n" + " units = count\n dimensions = ()\n type = integer\n intent = in\n" + ) + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + # Pre-condition: the leaf entry's access path carries BOTH + # registered-scalar-index placeholders, one per DDT level. + self.assertEqual( + hd['index_of_water_vapor_specific_humidity'].access_path, + 'GFS_Phys(instance_number)%Interstitial(thread_number)%scratch%idx_qv', + ) + sub, _used = _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', 'index_of_water_vapor_specific_humidity'], + 'run', hd, + ) + # All placeholders must be resolved in one pass. + self.assertEqual( + sub, + '(lb:ub, 1:nlev, ' + 'GFS_Phys(instance)%Interstitial(mythread)%scratch%idx_qv)', + ) + self.assertNotIn('instance_number', sub) + self.assertNotIn('thread_number', sub) + + def test_multiple_explicit_index_tokens_each_with_placeholder(self): + """Two distinct scheme-arg subscript tokens, each resolving to a + DDT-walked access path with its own ``(instance_number)`` + placeholder. Both must be substituted independently.""" + from metadata.metadata_table import _parse_lines + ddt_src = ( + "[ccpp-table-properties]\n name = GFS_control_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_control_type\n type = ddt\n" + "[ ntqv ]\n" + " standard_name = index_of_water_vapor_specific_humidity\n" + " units = index\n dimensions = ()\n type = integer\n" + "[ ntcw ]\n" + " standard_name = index_of_cloud_liquid_water_mixing_ratio\n" + " units = index\n dimensions = ()\n type = integer\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ GFS_Control ]\n standard_name = GFS_control_type_instance\n" + " units = DDT\n dimensions = (number_of_instances)\n" + " type = GFS_control_type\n" + "[ ncols ]\n standard_name = horizontal_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + "[ nlev ]\n standard_name = vertical_layer_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ instance ]\n standard_name = instance_number\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ ninstances ]\n standard_name = number_of_instances\n" + " units = count\n dimensions = ()\n type = integer\n intent = in\n" + ) + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + # Both index entries should carry the (instance_number) placeholder. + self.assertEqual( + hd['index_of_water_vapor_specific_humidity'].access_path, + 'GFS_Control(instance_number)%ntqv', + ) + self.assertEqual( + hd['index_of_cloud_liquid_water_mixing_ratio'].access_path, + 'GFS_Control(instance_number)%ntcw', + ) + # A subscript with TWO explicit index tokens — both placeholders + # must be rewritten. + sub, _used = _build_merged_subscript( + ['horizontal_dimension', 'vertical_layer_dimension'], + [':', ':', 'index_of_water_vapor_specific_humidity', + 'index_of_cloud_liquid_water_mixing_ratio'], + 'run', hd, + ) + self.assertEqual( + sub, + '(lb:ub, 1:nlev, ' + 'GFS_Control(instance)%ntqv, GFS_Control(instance)%ntcw)', + ) + self.assertNotIn('instance_number', sub) + ######################################################################## # Tests: _translate_active_expr From 1043666832b2ede09e9663325656580161702dfb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 May 2026 15:54:48 -0600 Subject: [PATCH 21/74] Bug fix: local_subscript lost in active expressions (and three sibling sites) --- capgen-ng/generator/suite_resolver.py | 71 ++++++++++++------ unit-tests/test_suite_resolver.py | 100 ++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 21 deletions(-) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 3c268329..21cf6487 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -68,7 +68,7 @@ scalar_index_for, is_scalar_index_dim, ) -from metadata.variable_resolver import HostVarEntry +from metadata.variable_resolver import HostVarEntry, _resolve_subscript # Dimension standard names that map to horizontal loop bounds. _HORIZ_LOOP_DIMS: frozenset = frozenset({ @@ -344,14 +344,11 @@ def _resolve_single_bound( # so the emitted subscript references the actual storage and # the USE statement (which walks back to the root via # ``_root_symbol``) imports the right top-level symbol. - # The DDT-instance walk bakes registered scalar-index std - # names (e.g. ``(thread_number)``, ``(instance_number)``) into - # the access path as placeholders; resolve them to the host's - # local Fortran names here so a bound that turns into a - # subscript on a per-thread/per-instance DDT field doesn't - # leak the std-name placeholder through to the emitted cap - # code (Fortran rejects it as "no IMPLICIT type"). - return _substitute_scalar_idx(entry.access_path, host_dict) + # ``_render_value_expr`` (a) resolves baked + # ``(instance_number)`` / ``(thread_number)`` placeholders and + # (b) re-attaches any literal subscript stripped from a + # ``local_name = foo(1)``-style declaration. + return _render_value_expr(entry, host_dict) if suite_vars: suite_var = suite_vars.get(bound) if suite_var is not None: @@ -594,15 +591,11 @@ def _build_merged_subscript( # Use ``access_path`` so DDT-component subscript indices # (e.g. ``q(:,:,index_of_)`` where index_of_X lives # on a DDT) resolve to the full DDT walk, not the bare - # leaf name. Identical to ``local_name`` for plain - # module-level host vars. The DDT walk bakes registered - # scalar-index std-name placeholders (e.g. - # ``GFS_Control(instance_number)%ntqv``) into the - # access path; rewrite them to host local names here so - # the inner ``(instance_number)`` doesn't leak through - # to the emitted Fortran when this access path is itself - # used as a subscript token. - parts.append(_substitute_scalar_idx(entry.access_path, host_dict)) + # leaf name. ``_render_value_expr`` (a) resolves baked + # ``(instance_number)``/``(thread_number)`` placeholders + # and (b) re-attaches any literal subscript stripped + # from a ``local_name = foo(1)``-style declaration. + parts.append(_render_value_expr(entry, host_dict)) used.add(key) elif suite_vars and key in suite_vars: parts.append( @@ -683,6 +676,40 @@ def _substitute_scalar_idx( _substitute_instance_idx = _substitute_scalar_idx +def _render_value_expr( + entry: HostVarEntry, + host_dict: Dict[str, HostVarEntry], +) -> str: + """Render *entry*'s full Fortran value-read expression. + + Combines two steps that callers usually need together: + + 1. Resolve any baked registered scalar-index placeholders in the + access path (``(instance_number)``, ``(thread_number)``) to the + host's local Fortran names via :func:`_substitute_scalar_idx`. + 2. Re-attach any literal subscript that was stripped from the + declared ``local_name`` at parse time (e.g. host metadata + declaring ``local_name = nstf_name(1)`` parses into + ``base='nstf_name'`` + ``local_subscript=['1']``; reading the + value requires re-appending ``(1)``). Std-name tokens inside + the subscript are themselves resolved to host local names via + :func:`metadata.variable_resolver._resolve_subscript`. + + Use this helper anywhere a host entry is rendered as a Fortran + expression in generator output (active-expression translation, + dimension-bound resolution, subscript-index tokens, subcycle + loop-count expressions, etc.). The scheme-arg base_expr + + _build_merged_subscript path is the exception — that path consumes + *entry.local_subscript* directly and interleaves it with scheme + dimensions, so it must not be pre-joined here. + """ + expr = _substitute_scalar_idx(entry.access_path, host_dict) + if entry.local_subscript: + sub = _resolve_subscript(', '.join(entry.local_subscript), host_dict) + expr = '{}({})'.format(expr, sub) + return expr + + def _translate_active_expr(active: str, host_dict: Dict[str, HostVarEntry]) -> str: """Translate standard names in an ``active`` expression to local Fortran. @@ -701,7 +728,7 @@ def _replace(m: re.Match) -> str: entry = host_dict.get(word) if entry is None: return word - return _substitute_instance_idx(entry.access_path, host_dict) + return _render_value_expr(entry, host_dict) return FORTRAN_CONDITIONAL_REGEX.sub(_replace, active) @@ -970,8 +997,10 @@ def _resolve_subcycle_loop_bound( # just the local name, but for a DDT-component the access path is # ``%`` (or # ``(instance_number)%`` when the parent is - # in an instance-dimensioned array; resolve that template here). - return _substitute_instance_idx(entry.access_path, host_dict), key + # in an instance-dimensioned array; resolve that template here, + # and re-attach any literal local_subscript from a + # ``local_name = foo(1)``-style declaration). + return _render_value_expr(entry, host_dict), key if suite_vars and key in suite_vars: suite_var = suite_vars[key] return suite_var.access_path, key diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index ebe1a835..ded760dc 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -717,6 +717,55 @@ def test_explicit_index_nested_ddt_two_placeholder_levels(self): self.assertNotIn('instance_number', sub) self.assertNotIn('thread_number', sub) + def test_explicit_index_with_literal_local_subscript(self): + """Regression 2026-05-15: a subscript-token entry whose declared + local_name carries a literal subscript (e.g. ``nstf_name(1)``) + must render the full ``(1)`` form, not bare ````. + Companion to the active-expression bug for the same root cause. + """ + from metadata.metadata_table import _parse_lines + ddt_src = ( + "[ccpp-table-properties]\n name = GFS_control_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_control_type\n type = ddt\n" + "[ nstf_name(1) ]\n" + " standard_name = control_for_nsstm\n" + " units = flag\n dimensions = ()\n type = integer\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ GFS_Control ]\n standard_name = GFS_control_type_instance\n" + " units = DDT\n dimensions = (number_of_instances)\n" + " type = GFS_control_type\n" + "[ ncols ]\n standard_name = horizontal_dimension\n" + " units = count\n dimensions = ()\n type = integer\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ instance ]\n standard_name = instance_number\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ ninstances ]\n standard_name = number_of_instances\n" + " units = count\n dimensions = ()\n type = integer\n intent = in\n" + ) + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + sub, _used = _build_merged_subscript( + ['horizontal_dimension'], + [':', 'control_for_nsstm'], + 'run', hd, + ) + self.assertEqual( + sub, '(lb:ub, GFS_Control(instance)%nstf_name(1))', + ) + def test_multiple_explicit_index_tokens_each_with_placeholder(self): """Two distinct scheme-arg subscript tokens, each resolving to a DDT-walked access path with its own ``(instance_number)`` @@ -872,6 +921,57 @@ def test_ddt_component_flag_uses_full_access_path(self): # No instance_number declared in this fixture → falls back to (1). self.assertEqual(result, '(instance_data(1)%opt_array_flag)') + def test_literal_subscript_in_local_name_preserved(self): + """Regression 2026-05-15: a DDT-component variable declared with a + literal subscript in its local_name (e.g. ``nstf_name(1)``) must + translate to ``(1)`` in an active expression — the + ``(1)`` carries semantic information (selects element 1 of an + integer array) and must not be dropped. Found in NEPTUNE + GFS_Statein metadata where ``tref`` had + ``active = (control_for_nsstm > 0)`` and ``control_for_nsstm`` + was declared as ``local_name = nstf_name(1)`` on GFS_Control; + the cap emitted ``GFS_Control(instance)%nstf_name > 0`` (rank + mismatch) instead of ``GFS_Control(instance)%nstf_name(1) > 0``. + """ + from metadata.metadata_table import _parse_lines + ddt_src = ( + "[ccpp-table-properties]\n name = GFS_control_type\n type = ddt\n" + "[ccpp-arg-table]\n name = GFS_control_type\n type = ddt\n" + "[ nstf_name(1) ]\n" + " standard_name = control_for_nsstm\n" + " units = flag\n dimensions = ()\n type = integer\n" + ) + host_src = ( + "[ccpp-table-properties]\n name = scm_type_defs\n type = host\n" + "[ccpp-arg-table]\n name = scm_type_defs\n type = host\n" + "[ GFS_Control ]\n standard_name = GFS_control_type_instance\n" + " units = DDT\n dimensions = (number_of_instances)\n" + " type = GFS_control_type\n" + ) + ctrl_src = ( + "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" + "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" + "[ instance ]\n standard_name = instance_number\n units = index\n" + " dimensions = ()\n type = integer\n intent = in\n" + "[ ninstances ]\n standard_name = number_of_instances\n" + " units = count\n dimensions = ()\n type = integer\n intent = in\n" + ) + hd = build_flat_host_dict( + _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), + _parse_lines(ctrl_src.splitlines(keepends=True), 'ctrl.meta'), + _parse_lines(ddt_src.splitlines(keepends=True), 'ddt.meta'), + ) + # Pre-conditions: access_path strips the literal subscript; + # local_subscript captures it for re-attachment at render time. + self.assertEqual( + hd['control_for_nsstm'].access_path, + 'GFS_Control(instance_number)%nstf_name', + ) + self.assertEqual(hd['control_for_nsstm'].local_subscript, ['1']) + # Active expression must emit the full ``(1)`` subscript. + result = _translate_active_expr('(control_for_nsstm > 0)', hd) + self.assertEqual(result, '(GFS_Control(instance)%nstf_name(1) > 0)') + ######################################################################## # Tests: _substitute_instance_idx From 96ffd0c54765c96557e7aabf7e5c03789cf4dd66 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 May 2026 20:00:00 -0600 Subject: [PATCH 22/74] ccpp_physics_final and ccpp_final idempotent --- capgen-ng/generator/group_cap.py | 10 +++++++-- capgen-ng/generator/suite_cap.py | 31 +++++++++++++++++++++------- doc/migration.md | 35 ++++++++++++++++++++++---------- 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/capgen-ng/generator/group_cap.py b/capgen-ng/generator/group_cap.py index a9e30e74..9c92c578 100644 --- a/capgen-ng/generator/group_cap.py +++ b/capgen-ng/generator/group_cap.py @@ -726,7 +726,7 @@ def _state_entry_guard( ``timestep_init`` ``== INITIALIZED`` ``run`` ``== IN_TIMESTEP`` ``timestep_final`` ``== IN_TIMESTEP`` - ``final`` ``>= INITIALIZED`` + ``final`` ``>= INITIALIZED`` (idempotent skip if ``UNINITIALIZED``) ===================== ============================================ Invalid state sets ``errflg = 1``, populates ``errmsg``, and returns. @@ -769,7 +769,13 @@ def _err_block(condition: str) -> List[str]: if phase == 'timestep_final': return _err_block('{} /= CCPP_GROUP_IN_TIMESTEP'.format(state_var)) if phase == 'final': - return _err_block('{} < CCPP_GROUP_INITIALIZED'.format(state_var)) + # Idempotent skip when already UNINITIALIZED; INITIALIZED and + # IN_TIMESTEP are both valid entry states (UNINITIALIZED is the only + # state value < INITIALIZED, so no error block is reachable here). + return [ + '{}if ({} == CCPP_GROUP_UNINITIALIZED) return'.format(indent, state_var), + '', + ] return [] diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 9d384fd4..6d6fab37 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -850,14 +850,31 @@ def _physics_dispatch_lines( inst_idx = _instance_idx(host_dict) sub_label = '{}_physics_{}'.format(suite_name, phase) + if phase == 'final': + # ``physics_final`` is silently idempotent: a repeat call (or a + # call issued after ``ccpp_final``) must return cleanly with + # ``errflg=0`` rather than erroring. The group-level guard + # handles the per-group skip when ``ccpp_final`` has not been + # called; the two checks below cover the post-``ccpp_final`` + # cases (state array deallocated on the last instance, or set + # to ``UNREGISTERED`` on any other instance). + lines += [ + '{}if (.not. allocated(ccpp_suite_state)) return'.format(i2), + '{}if (ccpp_suite_state({}) == CCPP_SUITE_UNREGISTERED) return'.format( + i2, inst_idx + ), + ] + else: + lines += [ + '{}if (.not. allocated(ccpp_suite_state)) then'.format(i2), + "{} {} = '{}: ccpp_register has not been called'".format( + i2, errmsg_local, sub_label + ), + '{} {} = 1'.format(i2, errflg_local), + '{} return'.format(i2), + '{}end if'.format(i2), + ] lines += [ - '{}if (.not. allocated(ccpp_suite_state)) then'.format(i2), - "{} {} = '{}: ccpp_register has not been called'".format( - i2, errmsg_local, sub_label - ), - '{} {} = 1'.format(i2, errflg_local), - '{} return'.format(i2), - '{}end if'.format(i2), '{}if (ccpp_suite_state({}) /= CCPP_SUITE_FRAMEWORK_INITIALIZED) then'.format( i2, inst_idx ), diff --git a/doc/migration.md b/doc/migration.md index f90fe668..50a29cbc 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -115,13 +115,18 @@ matches. ### 1.7 Optional `instance_number` / `number_of_instances` pair -These two control variables are now **paired optional**: +These two control variables are now **paired optional** and both live +in the host's `type=control` table (symmetric with the +`thread_number` / `number_of_threads` pair): -- Declare **both** (`instance_number` in `type=control`, - `number_of_instances` in `type=host`) → multi-instance API. +- Declare **both** in `type=control` → multi-instance API. Both flow + as control dummies through every lifecycle and physics-phase + signature. - Declare **neither** → single-instance API. Public entry points drop - `instance_number`; internal per-instance arrays size to length 1. + both args; internal per-instance arrays size to length 1. - Declare exactly one → hard error from the validator. +- Declare `number_of_instances` in `type=host` → hard error + (must be `type=control`). Hosts that don't need multi-instance bookkeeping can drop both declarations. @@ -332,14 +337,14 @@ Optional (paired — see §1.7): | Standard name | Fortran type | Table type | Purpose | |-------------------------|--------------|------------|--------------------------------| | `instance_number` | integer | control | Current instance index | -| `number_of_instances` | integer | host | Total instance count | +| `number_of_instances` | integer | control | Total instance count | ### 3.2 Required entry-point call sequence ``` -ccpp_register(suite_name, errflg, errmsg, [instance_number]) +ccpp_register(suite_name, errflg, errmsg, [instance_number, number_of_instances]) └── per scheme that declares a register phase -ccpp_init(suite_name, errflg, errmsg, [instance_number]) +ccpp_init(suite_name, errflg, errmsg, [instance_number, number_of_instances]) └── per scheme that declares an init phase ccpp_physics_init(...) └── physics phase routines per group: @@ -348,11 +353,14 @@ ccpp_physics_init(...) ccpp_physics_run ← run-loop phase ccpp_physics_timestep_final ccpp_physics_final -ccpp_final(suite_name, errflg, errmsg, [instance_number]) +ccpp_final(suite_name, errflg, errmsg, [instance_number, number_of_instances]) ``` -`instance_number` appears in every signature only when the host -declares the `instance_number` / `number_of_instances` pair (§1.7). +The `(instance_number, number_of_instances)` pair appears in every +signature only when the host declares it (§1.7). Both flow uniformly +through lifecycle and physics-phase calls; the framework consumes +`number_of_instances` only at register/init time but carries it +elsewhere for API symmetry with `(thread_number, number_of_threads)`. ### 3.3 Host module convention @@ -513,7 +521,12 @@ file lives in the target's parent directory (always under Always generated: - `ccpp_kinds.F90` — kind parameters. Listed under ``. -- `ccpp_static_api.F90` — public host-facing entry points + introspection routines. +- `_ccpp_cap.F90` — public host-facing entry points + introspection routines. + Filename and emitted `module _ccpp_cap` name are both driven by the + required `--host-name ` CLI argument so multiple host integrations + can co-exist in one executable. The public sub names inside + (`ccpp_register`, `ccpp_init`, `ccpp_physics_*`, `ccpp_final`) are + unchanged regardless of ``. - `ccpp__cap.F90` — per-suite dispatcher. - `ccpp___cap.F90` — per-group phase implementations. - `ccpp__data.F90` — suite-owned interstitial DDT + module-level array. From 2704aea28435abe814460a811f03f45d9de8f4df Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 May 2026 20:08:45 -0600 Subject: [PATCH 23/74] Fix final phase idempotency --- capgen-ng/generator/suite_cap.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 6d6fab37..63bf4a0c 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -695,14 +695,13 @@ def _final_lines( "{}{} = ''".format(i2, errmsg_local), '{}{} = 0'.format(i2, errflg_local), '', - '{}if (.not. allocated(ccpp_suite_state)) then'.format(i2), - "{} {} = '{}_final: ccpp_register has not been called'".format( - i2, errmsg_local, suite_name - ), - '{} {} = 1'.format(i2, errflg_local), - '{} return'.format(i2), - '{}end if'.format(i2), - '', + # ``_final`` is silently idempotent: a repeat call must return + # cleanly with ``errflg=0``. After the first call's last-to-leave + # teardown the state array is deallocated, so the unallocated path is + # the normal post-final state — silent-return rather than error. The + # per-instance ``UNREGISTERED`` skip covers any other instance that + # was already finalized before the last-to-leave dealloc fired. + '{}if (.not. allocated(ccpp_suite_state)) return'.format(i2), '{}if (ccpp_suite_state({}) == CCPP_SUITE_UNREGISTERED) return'.format( i2, inst_idx ), From abe5f112fd2c08e6cef4facf1718f9af760efdab Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 15 May 2026 20:55:50 -0600 Subject: [PATCH 24/74] Save documentation updates --- doc/migration.md | 10 ++++++++++ doc/redesign_analysis.md | 29 +++++++++++++++++++++++++++++ doc/redesign_prompt.md | 24 ++++++++++++++++++++---- unit-tests/test_suite_cap.py | 6 ------ 4 files changed, 59 insertions(+), 10 deletions(-) diff --git a/doc/migration.md b/doc/migration.md index 50a29cbc..2bb77ac4 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -600,6 +600,16 @@ Per-instance integer state arrays: Single-instance hosts get length-1 arrays indexed with literal `1`. See `doc/redesign_prompt.md` §7. +**Idempotent entry points.** `ccpp_physics_init`, `ccpp_physics_final`, and +`ccpp_final` are all silently idempotent — repeat calls return cleanly with +`errflg=0` rather than erroring. `ccpp_physics_final` additionally silent-skips +when issued *after* `ccpp_final` has torn the suite down (state array +deallocated on the last instance, or `== UNREGISTERED` on any other instance). +The other physics phases (`timestep_init`, `run`, `timestep_final`) still +hard-error on a state mismatch. `ccpp_init` does *not* silent-skip when the +state array is unallocated — there, "not allocated" really does mean +"you forgot `ccpp_register`" and continues to be a hard error. + --- ## 6. Framework changes (constituents) diff --git a/doc/redesign_analysis.md b/doc/redesign_analysis.md index 67252a5f..287e6ac2 100644 --- a/doc/redesign_analysis.md +++ b/doc/redesign_analysis.md @@ -1221,6 +1221,35 @@ and `horizontal_loop_end` are in scope for all phases — a `_init` scheme that `horizontal_dimension` correctly receives `(lb:ub)` slicing just as a `_run` scheme would, with the host responsible for passing the right values. +**Both `ccpp_physics_final` and `ccpp_final` are silently idempotent.** +Symmetric to `ccpp_physics_init`'s silent skip when already `INITIALIZED`, +both final-path entry points return cleanly with `errflg=0` on every repeat +invocation. Three cap levels participate: + +- The suite-cap `_physics_final` dispatcher silent-returns when + `ccpp_suite_state` is unallocated (last-instance post-`ccpp_final` deallocation) + or `ccpp_suite_state(inst_num) == CCPP_SUITE_UNREGISTERED` (any other + instance post-`ccpp_final`). The `state /= FRAMEWORK_INITIALIZED` error is + retained so calling `physics_final` after only `ccpp_register` (no `ccpp_init`) + still errors. +- The group-cap `_final` entry guard silent-returns when + `ccpp_group_state(inst_num) == CCPP_GROUP_UNINITIALIZED`. (Since `UNINITIALIZED` + is the only value `< INITIALIZED`, the previously generated error block became + unreachable and is no longer emitted.) +- The suite-cap `_final` body itself silent-returns on the same two + conditions (unallocated state array, or `== UNREGISTERED` for this instance). + After the first call's last-to-leave block deallocates `ccpp_suite_state`, + the unallocated state *is* the normal post-final condition — so on a + single-instance host the second call would otherwise trip a misleading + "`ccpp_register` has not been called" error. + +`_init` is intentionally *not* made idempotent on unallocated — there, +the unallocated state really does mean "you forgot `ccpp_register`", and +emitting an error is the correct behavior. + +The other physics phases (`init`, `timestep_init`, `run`, `timestep_final`) are +unchanged — they still hard-error with `errflg=1` on any state mismatch. + --- ## 9. Real-world example: CCPP Single Column Model (SCM) diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index 906c7838..8135ae92 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -493,8 +493,9 @@ integer, parameter :: CCPP_SUITE_FRAMEWORK_INITIALIZED = 2 |---|---|---| | `ccpp_register` | `== UNREGISTERED` | `REGISTERED` | | `ccpp_init` | `== REGISTERED` | `FRAMEWORK_INITIALIZED` | -| `ccpp_physics_*` | `== FRAMEWORK_INITIALIZED` | (unchanged) | -| `ccpp_final` | `== FRAMEWORK_INITIALIZED` | `REGISTERED` | +| `ccpp_physics_*` (non-final) | `== FRAMEWORK_INITIALIZED` | (unchanged) | +| `ccpp_physics_final` | (idempotent: silent skip if state array unallocated or `== UNREGISTERED`); otherwise `== FRAMEWORK_INITIALIZED` | (unchanged) | +| `ccpp_final` | (idempotent: silent skip if state array unallocated or `== UNREGISTERED`); otherwise any `>= REGISTERED` | `UNREGISTERED` (state array deallocated on last-to-leave) | ### 7.2 Group-level state (in each group cap) @@ -506,17 +507,32 @@ integer, parameter :: CCPP_GROUP_IN_TIMESTEP = 2 | Entry point | Required state | State after | |---|---|---| -| `ccpp_physics_init` | `< INITIALIZED` (idempotent if `== INITIALIZED`) | `INITIALIZED` | +| `ccpp_physics_init` | `< INITIALIZED` (idempotent silent skip if `== INITIALIZED`) | `INITIALIZED` | | `ccpp_physics_timestep_init` | `== INITIALIZED` | `IN_TIMESTEP` | | `ccpp_physics_run` | `== IN_TIMESTEP` | `IN_TIMESTEP` | | `ccpp_physics_timestep_final` | `== IN_TIMESTEP` | `INITIALIZED` | -| `ccpp_physics_final` | `>= INITIALIZED` | `UNINITIALIZED` | +| `ccpp_physics_final` | `>= INITIALIZED` (idempotent silent skip if `== UNINITIALIZED`) | `UNINITIALIZED` | The idempotency rule for `ccpp_physics_init`: if the group is already in state `INITIALIZED`, return immediately without calling any scheme `_init` routines. This allows the host to call `ccpp_physics_init` multiple times safely. Any further call after the first must result in no change (idempotency is a scheme contract). +The same rule applies to `ccpp_physics_final`: a repeat call (or a call issued +after `ccpp_final` has torn the suite down) must return cleanly with `errflg=0` +rather than erroring. This is enforced at both levels — the suite-cap dispatcher +silent-returns when `ccpp_suite_state` is unallocated or `== UNREGISTERED`, and +the group cap silent-returns when `ccpp_group_state(inst) == UNINITIALIZED`. + +`ccpp_final` itself is also silently idempotent for the same reason: the +first call's last-to-leave block deallocates `ccpp_suite_state`, so on a +single-instance host the unallocated state *is* the normal post-`ccpp_final` +condition. Both checks (`.not. allocated(ccpp_suite_state)` and +`ccpp_suite_state(inst) == CCPP_SUITE_UNREGISTERED`) silent-return rather +than erroring. By contrast, `ccpp_init`'s "not allocated" branch keeps +erroring with "ccpp_register has not been called" — there, the unallocated +state really does indicate a missed `ccpp_register` call. + #### 7.2.1 State array allocation and instance indexing Each group cap declares an allocatable module-level array: diff --git a/unit-tests/test_suite_cap.py b/unit-tests/test_suite_cap.py index 9fa300c8..b5d14890 100644 --- a/unit-tests/test_suite_cap.py +++ b/unit-tests/test_suite_cap.py @@ -320,12 +320,6 @@ class TestFinalSubroutineStateMachine(unittest.TestCase): def setUp(self): self.text = '\n'.join(_generate()) - def test_state_check_register_message(self): - # The not-allocated guard now refers to ccpp_register, not ccpp_init. - self.assertIn( - 'ccpp_register has not been called', self.text, - ) - def test_idempotent_unregistered_skip(self): self.assertIn('== CCPP_SUITE_UNREGISTERED', self.text) From 523ac77ba1ee5c763bf9ccaddcee85f1ded4077e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 18 May 2026 20:27:12 -0600 Subject: [PATCH 25/74] Add end-to-end-tests/instances_advection and bug fixes/updates to capgen-ng: entire constituent state now per instance --- capgen-ng/ccpp_validator.py | 5 +- capgen-ng/generator/datatable.py | 2 + capgen-ng/generator/host_constituents.py | 58 +++- capgen-ng/generator/kinds_writer.py | 2 +- capgen-ng/generator/suite_cap.py | 54 ++-- capgen-ng/generator/trace.py | 23 +- capgen-ng/metadata/metadata_table.py | 2 +- doc/constituents_overhaul.md | 65 ++++- end-to-end-tests/CMakeLists.txt | 1 + .../instances_advection/CMakeLists.txt | 63 +++++ .../apply_constituent_tendencies.F90 | 39 +++ .../apply_constituent_tendencies.meta | 36 +++ .../instances_advection/cld_liq.F90 | 99 +++++++ .../instances_advection/cld_liq.meta | 135 +++++++++ .../instances_advection/cld_suite.xml | 8 + end-to-end-tests/instances_advection/data.F90 | 128 +++++++++ .../instances_advection/data.meta | 83 ++++++ end-to-end-tests/instances_advection/main.F90 | 256 ++++++++++++++++++ .../instances_advection/main.meta | 77 ++++++ unit-tests/test_host_constituents.py | 19 +- unit-tests/test_suite_resolver.py | 14 +- unit-tests/test_trace.py | 21 +- 22 files changed, 1141 insertions(+), 49 deletions(-) create mode 100644 end-to-end-tests/instances_advection/CMakeLists.txt create mode 100644 end-to-end-tests/instances_advection/apply_constituent_tendencies.F90 create mode 100644 end-to-end-tests/instances_advection/apply_constituent_tendencies.meta create mode 100644 end-to-end-tests/instances_advection/cld_liq.F90 create mode 100644 end-to-end-tests/instances_advection/cld_liq.meta create mode 100644 end-to-end-tests/instances_advection/cld_suite.xml create mode 100644 end-to-end-tests/instances_advection/data.F90 create mode 100644 end-to-end-tests/instances_advection/data.meta create mode 100644 end-to-end-tests/instances_advection/main.F90 create mode 100644 end-to-end-tests/instances_advection/main.meta diff --git a/capgen-ng/ccpp_validator.py b/capgen-ng/ccpp_validator.py index 486f3543..749e1102 100755 --- a/capgen-ng/ccpp_validator.py +++ b/capgen-ng/ccpp_validator.py @@ -146,6 +146,9 @@ def _line_optional_names(line: str) -> List[str]: >>> _line_optional_names(' ! a comment, optional :: not_a_decl') [] """ + # Strip any inline ``!`` comment so an ``optional`` token inside a + # comment can't be misread as an attribute declaration. + line = _COMMENT_RE.sub('', line) if '::' not in line: return [] before, _, after = line.partition('::') @@ -206,7 +209,7 @@ def _join_continuation( [' foo bar', ' baz'] >>> _join_continuation([' foo &\\n', ' & bar\\n', ... ' & )\\n', ' baz\\n']) - [' foo bar )', ' baz'] + [' foo bar )', ' baz'] """ # First pass: normalise each line — strip trailing newlines and any # inline ``!`` comment. Keep blank/comment-only lines as ``''`` so diff --git a/capgen-ng/generator/datatable.py b/capgen-ng/generator/datatable.py index efec3fcf..d838f82c 100644 --- a/capgen-ng/generator/datatable.py +++ b/capgen-ng/generator/datatable.py @@ -433,6 +433,8 @@ def write_datatable( >>> suite_resolution = MagicMock() >>> suite_resolution.suite_name = 'test' >>> suite_resolution.groups = [] + >>> suite_resolution.suite_init_call = None + >>> suite_resolution.suite_final_call = None >>> store = MagicMock() >>> with tempfile.TemporaryDirectory() as d: ... path = write_datatable([suite_resolution], store, [], [], d) diff --git a/capgen-ng/generator/host_constituents.py b/capgen-ng/generator/host_constituents.py index 7c9a3e64..55b29a3e 100644 --- a/capgen-ng/generator/host_constituents.py +++ b/capgen-ng/generator/host_constituents.py @@ -166,7 +166,15 @@ def _register_constituents_lines( for sname in register_suites: buf = _dyn_const_array_name(sname) lines.append('{}if (allocated({})) then'.format(i2, buf)) - lines.append('{}num_consts = num_consts + size({}, 1)'.format(i3, buf)) + lines.append( + '{}if (allocated({}({})%items)) then'.format(i3, buf, inst_idx) + ) + lines.append( + '{}num_consts = num_consts + size({}({})%items, 1)'.format( + i3 + _INDENT, buf, inst_idx, + ) + ) + lines.append('{}end if'.format(i3)) lines.append('{}end if'.format(i2)) lines.append('') lines.append('{}call {}({})%initialize_table(num_consts)'.format( @@ -189,16 +197,28 @@ def _register_constituents_lines( lines.append('') lines.append("{}! Merge {} dynamic constituents".format(i2, sname)) lines.append('{}if (allocated({})) then'.format(i2, buf)) - lines.append('{}do index = 1, size({}, 1)'.format(i3, buf)) - lines.append('{}const_prop => {}(index)'.format(i3 + _INDENT, buf)) + lines.append( + '{}if (allocated({}({})%items)) then'.format(i3, buf, inst_idx) + ) + lines.append( + '{}do index = 1, size({}({})%items, 1)'.format( + i3 + _INDENT, buf, inst_idx, + ) + ) + lines.append( + '{}const_prop => {}({})%items(index)'.format( + i3 + _INDENT * 2, buf, inst_idx, + ) + ) lines.append( '{}call {}({})%new_field(const_prop, errcode=errflg, errmsg=errmsg)'.format( - i3 + _INDENT, _CONST_OBJ, inst_idx, + i3 + _INDENT * 2, _CONST_OBJ, inst_idx, ) ) - lines.append('{}nullify(const_prop)'.format(i3 + _INDENT)) - lines.append('{}if (errflg /= 0) return'.format(i3 + _INDENT)) - lines.append('{}end do'.format(i3)) + lines.append('{}nullify(const_prop)'.format(i3 + _INDENT * 2)) + lines.append('{}if (errflg /= 0) return'.format(i3 + _INDENT * 2)) + lines.append('{}end do'.format(i3 + _INDENT)) + lines.append('{}end if'.format(i3)) lines.append('{}end if'.format(i2)) lines.append('') lines.append( @@ -592,6 +612,26 @@ def _generate_host_constituents( lines.append('{}public :: {}'.format(_INDENT, p)) lines.append('') + # Per-instance wrapper around the per-suite scheme-registered + # constituent buffer. Each instance owns its own slot in the outer + # array; the inner ``items(:)`` array is filled independently by + # ``_register`` per instance so that the property objects + # (which acquire a const_ind during ``ccpp_register_constituents``) + # are not shared across instances. + if register_suites: + lines.append( + '{}type :: ccpp_dyn_const_buffer_t'.format(_INDENT) + ) + lines.append( + '{}type({}), allocatable :: items(:)'.format( + _INDENT * 2, _CONST_PROP_TYPE, + ) + ) + lines.append( + '{}end type ccpp_dyn_const_buffer_t'.format(_INDENT) + ) + lines.append('') + # State declarations. lines.append( '{}type({}), target, allocatable :: {}(:)'.format( @@ -602,8 +642,8 @@ def _generate_host_constituents( for sname in register_suites: buf = _dyn_const_array_name(sname) lines.append( - '{}type({}), allocatable, target :: {}(:)'.format( - _INDENT, _CONST_PROP_TYPE, buf, + '{}type(ccpp_dyn_const_buffer_t), allocatable, target :: {}(:)'.format( + _INDENT, buf, ) ) if register_suites: diff --git a/capgen-ng/generator/kinds_writer.py b/capgen-ng/generator/kinds_writer.py index 99e868d7..9597c9ca 100644 --- a/capgen-ng/generator/kinds_writer.py +++ b/capgen-ng/generator/kinds_writer.py @@ -168,7 +168,7 @@ def _generate_ccpp_kinds(kind_types: KindMap) -> List[str]: Empty mapping is an error: - >>> _generate_ccpp_kinds({}) + >>> _generate_ccpp_kinds({}) # doctest: +ELLIPSIS Traceback (most recent call last): ... metadata.parse_tools.parse_source.CCPPError: ccpp_kinds requires at least ... diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 63bf4a0c..900a7175 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -389,50 +389,62 @@ def _register_lines( # instance's ``ccpp_model_constituents_obj(inst)`` happens later # when the host calls ``ccpp_register_constituents`` per instance. # - # The buffer itself is shared across instances (registration is - # identical per instance) — gated by ``.not. allocated`` so that - # only the first instance to enter does the two-pass count+pack. - # Subsequent instances reuse the same buffer. The state-array - # transition still runs per instance (after this block). + # Each instance owns its own slot ``(inst)%items(:)``: the + # property objects are independent across instances so that + # ``ccpp_register_constituents`` can ``set_const_index`` on each + # without conflicting with other instances. The outer wrapper + # array is allocated once on first call (any instance); each + # instance then runs its own two-pass count+pack into its slot. + # The state-machine guard above this block ensures each instance + # runs the fill at most once. const_scheme_names = {scheme_name for scheme_name, _ in suite_res.constituent_register_calls} buf = '{}_dynamic_constituents'.format(suite_name) + # Allocate the outer wrapper array on first call (any instance). lines.append( '{}if (.not. allocated({})) then'.format(i2, buf) ) - lines.append('{}num_consts = 0'.format(i2 + _INDENT)) - lines.append('{}! First pass: count constituents'.format(i2 + _INDENT)) + lines.append('{}allocate({}({}))'.format( + i2 + _INDENT, buf, ninstances_arg, + )) + lines.append('{}end if'.format(i2)) + lines.append('') + + # Per-instance two-pass count+pack into this instance's slot. + lines.append('{}num_consts = 0'.format(i2)) + lines.append('{}! First pass: count constituents'.format(i2)) for _gname, resolved_call in _register_calls(suite_res): if resolved_call.scheme_name in const_scheme_names: - _emit_register_call(resolved_call, i2 + _INDENT, errflg_local, lines) + _emit_register_call(resolved_call, i2, errflg_local, lines) lines.append( '{}num_consts = num_consts + size(scheme_consts, 1)'.format( - i2 + _INDENT, + i2, ) ) - lines.append('{}deallocate(scheme_consts)'.format(i2 + _INDENT)) + lines.append('{}deallocate(scheme_consts)'.format(i2)) lines.append('') - lines.append('{}allocate({}(num_consts))'.format(i2 + _INDENT, buf)) - lines.append('{}num_consts = 0'.format(i2 + _INDENT)) + lines.append('{}allocate({}({})%items(num_consts))'.format( + i2, buf, inst_idx, + )) + lines.append('{}num_consts = 0'.format(i2)) lines.append('') - lines.append('{}! Second pass: copy into per-suite buffer'.format(i2 + _INDENT)) + lines.append('{}! Second pass: copy into per-instance buffer'.format(i2)) for _gname, resolved_call in _register_calls(suite_res): if resolved_call.scheme_name in const_scheme_names: - _emit_register_call(resolved_call, i2 + _INDENT, errflg_local, lines) - lines.append('{}do i = 1, size(scheme_consts, 1)'.format(i2 + _INDENT)) + _emit_register_call(resolved_call, i2, errflg_local, lines) + lines.append('{}do i = 1, size(scheme_consts, 1)'.format(i2)) lines.append( - '{}{}(num_consts + i) = scheme_consts(i)'.format( - i2 + _INDENT * 2, buf, + '{}{}({})%items(num_consts + i) = scheme_consts(i)'.format( + i2 + _INDENT, buf, inst_idx, ) ) - lines.append('{}end do'.format(i2 + _INDENT)) + lines.append('{}end do'.format(i2)) lines.append( '{}num_consts = num_consts + size(scheme_consts, 1)'.format( - i2 + _INDENT, + i2, ) ) - lines.append('{}deallocate(scheme_consts)'.format(i2 + _INDENT)) - lines.append('{}end if'.format(i2)) + lines.append('{}deallocate(scheme_consts)'.format(i2)) lines.append('') # Emit any non-constituent register calls in addition (always, per instance). for _gname, resolved_call in _register_calls(suite_res): diff --git a/capgen-ng/generator/trace.py b/capgen-ng/generator/trace.py index 4fa3b1a6..7a7c146d 100644 --- a/capgen-ng/generator/trace.py +++ b/capgen-ng/generator/trace.py @@ -9,12 +9,18 @@ ``intent(inout)`` control dummy emits a guarded write as the very first line of its body:: - if (trace) write(error_unit, *) & + if (trace) write(error_unit, '(a,a,a,a,1x,i0)') & 'CCPP TRACE :', & - ' =', , & - ' =', trim(), & + ' =', trim(), & + ' =', , & ... +The format string is built per-call: each character item contributes an +``a`` descriptor (the label literal and any ``trim()``-wrapped value), +and each integer item contributes ``1x,i0`` so the value is printed +flush against a single space separator rather than the wide default +field of list-directed I/O. + Two effects: 1. With ``trace = .false.`` (the default) the compiler eliminates the @@ -178,8 +184,17 @@ def emit_trace_block( if not items: return [] + # Build a per-call format: one ``a`` for the trace name, then for each + # item one ``a`` for the ``' label='`` literal plus ``a`` (character) + # or ``1x,i0`` (integer) for the value. + fmt_parts: List[str] = ['a'] + for (_, is_char) in items: + fmt_parts.append('a') + fmt_parts.append('a' if is_char else '1x,i0') + fmt = "'({})'".format(','.join(fmt_parts)) + lines: List[str] = [] - lines.append('{}if (trace) write(error_unit, *) &'.format(indent)) + lines.append('{}if (trace) write(error_unit, {}) &'.format(indent, fmt)) lines.append("{} 'CCPP TRACE {}:', &".format(indent, sub_name)) for i, (local_name, is_char) in enumerate(items): expr = 'trim({})'.format(local_name) if is_char else local_name diff --git a/capgen-ng/metadata/metadata_table.py b/capgen-ng/metadata/metadata_table.py index 3f62d89f..aa762b1e 100644 --- a/capgen-ng/metadata/metadata_table.py +++ b/capgen-ng/metadata/metadata_table.py @@ -215,7 +215,7 @@ def _parse_kind_spec_value( ('kind_r8', 'host_kinds', 'kind_r8') >>> _parse_kind_spec_value(' temp_kinds : kind_temp => temp_r8 ', ctx) ('kind_temp', 'temp_kinds', 'temp_r8') - >>> _parse_kind_spec_value('not_a_kind_spec', ctx) + >>> _parse_kind_spec_value('not_a_kind_spec', ctx) # doctest: +ELLIPSIS Traceback (most recent call last): ... metadata.parse_tools.parse_source.CCPPError: Malformed kind_spec ... diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index 717d548c..834e07c0 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -2,14 +2,16 @@ **Authors:** Dom Heinzeller (lead), Claude (assistant) **Date drafted:** 2026-05-12 -**Last revised:** 2026-05-13 +**Last revised:** 2026-05-18 **Intended audience:** CCPP framework team, CAM-SIMA team **Status:** Discussion document — no decisions are final. Proposals A/B/C below remain pending the upcoming meeting; the bug fix from Proposal A (the `ccpt_deallocate` ownership flag) and the capgen-ng internal cleanup from Proposal B (§4.8) have landed; the missing setters from Proposal A and the `is_match` relaxation from Proposal B -have not. +have not. Independent of A/B/C, the per-suite dynamic_constituents +buffer was made per-instance on 2026-05-18 to fix a multi-instance +mutation conflict — see §4.13. --- @@ -231,8 +233,9 @@ the standard-name catalog is identical across instances. ``` ccpp_register(suite_name, instance_number, ...) └─ _register → packs scheme-dynamic constituents into - _dynamic_constituents (shared buffer, - first instance wins) + _dynamic_constituents(instance)%items + (per-instance wrapper-DDT array; each instance + allocates and fills its own slot — see §4.13) ↓ ccpp_register_constituents(host_constituents, instance_number, ...) └─ initialize_table(num_host_consts + num_suite_consts) @@ -548,6 +551,60 @@ added on the framework side. Hosts that want runtime override get spirit to the existing `horizontal_loop_extent → horizontal_dimension` shim. Remove the rewrite once known consumers are migrated. +### 4.13 Capgen-ng: per-suite `dynamic_constituents` buffer was shared across instances (FIXED 2026-05-18) + +- **Location**: `capgen-ng/generator/host_constituents.py` (buffer + declaration + `ccpp_register_constituents` iteration); + `capgen-ng/generator/suite_cap.py::_register_lines` (the two-pass + count→allocate→pack inside `_register`). +- **Symptom**: with two or more instances and any register-phase + scheme that produces constituents, the second per-instance + `ccpp_register_constituents` call fails with `ccp_set_const_index + ccpp_constituent_properties_t const index is already set`. +- **Root cause**: the per-suite buffer + `_dynamic_constituents(:)` was declared as a single shared + 1-D array of `ccpp_constituent_properties_t`, filled exactly once on + first instance entry (`.not. allocated(buf)` gate). + `ccpp_register_constituents` then iterates that shared buffer per + instance and calls `%new_field(const_prop)` on each property + object. `%new_field` calls `ccp_set_const_index`, which **mutates + the property object** by writing `const_ind`. Instance 1 set + `const_ind` on every shared object; instance 2's call tripped the + "set exactly once" guard. +- **Latent companion bug**: the same shared-mutation pattern means + that once Proposal B's class-B setters (`set_advected`, + `set_diagnostic_name`, `set_water_species` per-instance, etc.) are + exercised, instance 1's setter call would silently corrupt instance + 2's view of the property. No "already set" guard exists on those + setters today. +- **Why it didn't surface earlier**: the advection end-to-end test is + single-instance; the instances end-to-end test has no constituents. + Surfaced by the new `instances_advection` combined test + (`end-to-end-tests/instances_advection/`) on first run. +- **Fix landed 2026-05-18**: the per-suite buffer is now a wrapper-DDT + array indexed by `instance_number`: + ```fortran + type :: ccpp_dyn_const_buffer_t + type(ccpp_constituent_properties_t), allocatable :: items(:) + end type + type(ccpp_dyn_const_buffer_t), allocatable, target :: _dynamic_constituents(:) + ``` + The outer array is allocated to `number_of_instances` on first call; + each instance independently runs the two-pass count+pack into its + own `%items` slot. `ccpp_register_constituents` iterates + `_dynamic_constituents(instance)%items` so each instance's + `new_field` calls operate on **distinct** property objects. + Scheme `_register` routines are now called N times instead of once + (negligible cost — typical register bodies are a few `%instantiate` + calls), in exchange for clean per-instance isolation. +- **Cost**: ~50 lines across the two generator emitters, plus updates + to six pinned unit tests. No CAM-SIMA / NEPTUNE / SCM coordination + needed (host-facing API unchanged). +- **Status**: framework tests pass; full unit-test suite (1319 tests) + is green; all 10 end-to-end tests pass. +- **Position relative to Proposals A/B/C**: orthogonal — none of the + three proposed touching the buffer. Independently adopted. + --- ## 5. Property classification (Class A vs Class B) diff --git a/end-to-end-tests/CMakeLists.txt b/end-to-end-tests/CMakeLists.txt index 8e5945fc..84f4b984 100644 --- a/end-to-end-tests/CMakeLists.txt +++ b/end-to-end-tests/CMakeLists.txt @@ -80,3 +80,4 @@ add_subdirectory(instances) add_subdirectory(capgen_ng) add_subdirectory(var_compat) add_subdirectory(advection) +add_subdirectory(instances_advection) diff --git a/end-to-end-tests/instances_advection/CMakeLists.txt b/end-to-end-tests/instances_advection/CMakeLists.txt new file mode 100644 index 00000000..25bc8687 --- /dev/null +++ b/end-to-end-tests/instances_advection/CMakeLists.txt @@ -0,0 +1,63 @@ +#------------------------------------------------------------------------------ +# +# Multi-instance + constituents combined test +# +# Exercises per-instance state separation (from end-to-end-tests/instances) and +# host + scheme-registered constituents (boiled-down from end-to-end-tests/ +# advection). Verification is mass conservation per instance and cross-instance +# distinctness of final state. +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "cld_liq" "apply_constituent_tendencies") +set(HOST_FILES "data" "main") +set(SUITE_FILES "cld_suite.xml") +set(HOST "test_host") +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES}) + +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_instances_advection.x + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES}) +target_link_libraries(test_instances_advection.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_instances_advection.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_instances_advection.x PROPERTIES LINKER_LANGUAGE Fortran) + +add_test(NAME test_instances_advection + COMMAND test_instances_advection.x) diff --git a/end-to-end-tests/instances_advection/apply_constituent_tendencies.F90 b/end-to-end-tests/instances_advection/apply_constituent_tendencies.F90 new file mode 100644 index 00000000..63a1881c --- /dev/null +++ b/end-to-end-tests/instances_advection/apply_constituent_tendencies.F90 @@ -0,0 +1,39 @@ +module apply_constituent_tendencies + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: apply_constituent_tendencies_run + +contains + + !> \section arg_table_apply_constituent_tendencies_run Argument Table + !!! \htmlinclude apply_constituent_tendencies_run.html + subroutine apply_constituent_tendencies_run(const_tend, const, errcode, errmsg) + ! Dummy arguments + real(kind=kind_phys), intent(inout) :: const_tend(:, :, :) ! constituent tendency array + real(kind=kind_phys), intent(inout) :: const(:, :, :) ! constituent state array + integer, intent(out) :: errcode + character(len=512), intent(out) :: errmsg + + ! Local variables + integer :: klev, jcnst, icol + + errcode = 0 + errmsg = '' + + do icol = 1, size(const_tend, 1) + do klev = 1, size(const_tend, 2) + do jcnst = 1, size(const_tend, 3) + const(icol, klev, jcnst) = const(icol, klev, jcnst) + const_tend(icol, klev, jcnst) + end do + end do + end do + + const_tend = 0._kind_phys + + end subroutine apply_constituent_tendencies_run + +end module apply_constituent_tendencies diff --git a/end-to-end-tests/instances_advection/apply_constituent_tendencies.meta b/end-to-end-tests/instances_advection/apply_constituent_tendencies.meta new file mode 100644 index 00000000..ac02e5e4 --- /dev/null +++ b/end-to-end-tests/instances_advection/apply_constituent_tendencies.meta @@ -0,0 +1,36 @@ +##################################################################### +[ccpp-table-properties] + name = apply_constituent_tendencies + type = scheme +[ccpp-arg-table] + name = apply_constituent_tendencies_run + type = scheme +[ const_tend ] + standard_name = ccpp_constituent_tendencies + long_name = ccpp constituent tendencies + units = none + type = real | kind = kind_phys + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + intent = inout +[ const ] + standard_name = ccpp_constituents + long_name = ccpp constituents + units = none + type = real | kind = kind_phys + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + intent = inout +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + type = integer + dimensions = () + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + type = character | kind = len=512 + dimensions = () + intent = out +######################################################### diff --git a/end-to-end-tests/instances_advection/cld_liq.F90 b/end-to-end-tests/instances_advection/cld_liq.F90 new file mode 100644 index 00000000..25b05b9f --- /dev/null +++ b/end-to-end-tests/instances_advection/cld_liq.F90 @@ -0,0 +1,99 @@ +! Test parameterization with advected species +! + +module cld_liq + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + private + + public :: cld_liq_register + public :: cld_liq_init + public :: cld_liq_run + +contains + + !> \section arg_table_cld_liq_register Argument Table + !! \htmlinclude arg_table_cld_liq_register.html + !! + subroutine cld_liq_register(dyn_const, errmsg, errflg) + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + allocate(dyn_const(1), stat=errflg) + if (errflg /= 0) then + errmsg = 'Error allocating dyn_const in cld_liq_register' + return + end if + call dyn_const(1)%instantiate(std_name="cloud_liquid_dry_mixing_ratio", long_name='Cloud liquid dry mixing ratio', & + diag_name='CLDLIQ', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + mixing_ratio_type='dry', & + errcode=errflg, errmsg=errmsg) + + end subroutine cld_liq_register + + !> \section arg_table_cld_liq_run Argument Table + !! \htmlinclude arg_table_cld_liq_run.html + !! + subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & + cld_liq_array, cld_liq_tend, errmsg, errflg) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(in) :: tcld + real(kind=kind_phys), intent(inout) :: temp(:, :) + real(kind=kind_phys), intent(inout) :: qv(:, :) + real(kind=kind_phys), intent(in) :: ps(:) + real(kind=kind_phys), intent(inout) :: cld_liq_array(:, :) + real(kind=kind_phys), intent(out) :: cld_liq_tend(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: icol + integer :: ilev + real(kind=kind_phys) :: cond + + errmsg = '' + errflg = 0 + + do icol = 1, ncol + do ilev = 1, size(temp, 2) + cld_liq_array(icol, ilev) = max(0.0_kind_phys, cld_liq_array(icol, ilev)) + if ((qv(icol, ilev) > 0.0_kind_phys) .and. & + (temp(icol, ilev) <= tcld)) then + cond = min(qv(icol, ilev), 0.1_kind_phys) + cld_liq_tend(icol, ilev) = cond + qv(icol, ilev) = qv(icol, ilev) - cond + if (cond > 0.0_kind_phys) then + temp(icol, ilev) = temp(icol, ilev) + (cond * 5.0_kind_phys) + end if + end if + end do + end do + + end subroutine cld_liq_run + + !> \section arg_table_cld_liq_init Argument Table + !! \htmlinclude arg_table_cld_liq_init.html + !! + subroutine cld_liq_init(tfreeze, tcld, errmsg, errflg) + + real(kind=kind_phys), intent(in) :: tfreeze + real(kind=kind_phys), intent(out) :: tcld + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + tcld = tfreeze - 20.0_kind_phys + + end subroutine cld_liq_init + +end module cld_liq diff --git a/end-to-end-tests/instances_advection/cld_liq.meta b/end-to-end-tests/instances_advection/cld_liq.meta new file mode 100644 index 00000000..db43c5ef --- /dev/null +++ b/end-to-end-tests/instances_advection/cld_liq.meta @@ -0,0 +1,135 @@ +# cld_liq is a scheme that produces a cloud liquid amount +[ccpp-table-properties] + name = cld_liq + type = scheme +[ccpp-arg-table] + name = cld_liq_register + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_cld_liq + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out + allocatable = true +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_liq_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ tcld] + standard_name = minimum_temperature_for_cloud_liquid + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ temp ] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_LAYER_dimension) + type = real + kind = kind_phys + intent = inout +[ qv ] + standard_name = water_vapor_specific_humidity + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = hPa + dimensions = (horizontal_dimension) + intent = in +[ cld_liq_array ] + standard_name = cloud_liquid_dry_mixing_ratio + diagnostic_name = CLDLIQ + advected = .true. + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = inout +[ cld_liq_tend ] + standard_name = tendency_of_cloud_liquid_dry_mixing_ratio + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_liq_init + type = scheme +[ tfreeze] + standard_name = water_temperature_at_freezing + long_name = Freezing temperature of water at sea level + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ tcld] + standard_name = minimum_temperature_for_cloud_liquid + units = K + dimensions = () + type = real | kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/instances_advection/cld_suite.xml b/end-to-end-tests/instances_advection/cld_suite.xml new file mode 100644 index 00000000..65e98802 --- /dev/null +++ b/end-to-end-tests/instances_advection/cld_suite.xml @@ -0,0 +1,8 @@ + + + + + cld_liq + apply_constituent_tendencies + + diff --git a/end-to-end-tests/instances_advection/data.F90 b/end-to-end-tests/instances_advection/data.F90 new file mode 100644 index 00000000..1145dc83 --- /dev/null +++ b/end-to-end-tests/instances_advection/data.F90 @@ -0,0 +1,128 @@ +module data + + use ccpp_kinds, only: kind_phys + + implicit none + public + + ! Sizing parameters (shared across instances) + integer, parameter :: ncols = 4 + integer, parameter :: pver = 3 + integer, parameter :: ninstances = 3 + + ! Time-step + freezing-point constants (shared across instances) + real(kind=kind_phys), parameter :: dt = 1.0_kind_phys + real(kind=kind_phys), parameter :: tfreeze = 273.15_kind_phys + integer, parameter :: num_time_steps = 2 + + ! qv index in the constituent state array; filled at runtime after + ! ccpp_register_constituents. Identical across instances because all + ! instances register the same constituents in the same order. + integer, protected :: index_qv = -1 + + ! Per-instance mutable physics state + ! + ! \section arg_table_physics_state Argument Table + !! \htmlinclude arg_table_physics_state.html + type physics_state + real(kind=kind_phys), allocatable :: ps(:) ! surface pressure + real(kind=kind_phys), allocatable :: temp(:, :) ! temperature + real(kind=kind_phys), pointer :: q(:, :, :) => null() ! constituent array + end type physics_state + + type(physics_state), target :: phys_state(ninstances) + + ! Per-instance distinct initial qv values (used to drive distinct results) + real(kind=kind_phys), parameter, dimension(ninstances) :: & + qv_init = (/ 1.0_kind_phys, 2.0_kind_phys, 3.0_kind_phys /) + + ! Tolerance for the final verification check + real(kind=kind_phys), parameter :: tolerance = 1.0e-12_kind_phys + +contains + + subroutine set_index_qv(idx) + integer, intent(in) :: idx + index_qv = idx + end subroutine set_index_qv + + subroutine allocate_physics_state(ins, constituents_ptr) + ! Wire phys_state(ins) to its per-instance constituent array and allocate + ! the per-instance temp/ps storage. + integer, intent(in) :: ins + real(kind=kind_phys), pointer :: constituents_ptr(:, :, :) + + if (allocated(phys_state(ins)%ps)) then + deallocate(phys_state(ins)%ps) + end if + allocate(phys_state(ins)%ps(ncols)) + phys_state(ins)%ps = 1000.0_kind_phys + + if (allocated(phys_state(ins)%temp)) then + deallocate(phys_state(ins)%temp) + end if + allocate(phys_state(ins)%temp(ncols, pver)) + ! Start cold so cld_liq_run will produce a tendency on the first call. + phys_state(ins)%temp = tfreeze - 30.0_kind_phys + + if (associated(phys_state(ins)%q)) nullify(phys_state(ins)%q) + phys_state(ins)%q => constituents_ptr + + end subroutine allocate_physics_state + + subroutine init_qv(ins) + ! Seed the per-instance constituent array with a distinct qv value. + integer, intent(in) :: ins + phys_state(ins)%q(:, :, :) = 0.0_kind_phys + phys_state(ins)%q(:, :, index_qv) = qv_init(ins) + end subroutine init_qv + + logical function verify_results(num_consts) + ! Check per-instance mass conservation and cross-instance distinctness. + integer, intent(in) :: num_consts + + real(kind=kind_phys) :: q_sum(ninstances) + real(kind=kind_phys) :: cld_liq_max(ninstances) + integer :: ins, ins2, k + logical :: ok + + verify_results = .true. + + ! Mass conservation per instance: total constituent mass per instance + ! should equal ncols * pver * qv_init(ins) (because all qv either + ! stays as qv or is moved into cld_liq by cld_liq_run; nothing else + ! produces or consumes mass in this minimal scheme set). + do ins = 1, ninstances + q_sum(ins) = 0.0_kind_phys + do k = 1, num_consts + q_sum(ins) = q_sum(ins) + sum(phys_state(ins)%q(:, :, k)) + end do + cld_liq_max(ins) = maxval(phys_state(ins)%q(:, :, :)) + ok = abs(q_sum(ins) - real(ncols, kind_phys) * real(pver, kind_phys) & + * qv_init(ins)) < tolerance * real(ncols * pver, kind_phys) & + * qv_init(ins) + if (.not. ok) then + write(6, '(a,i0,a,es15.7,a,es15.7)') & + 'FAIL mass conservation for instance ', ins, & + ': q_sum=', q_sum(ins), ' expected~', & + real(ncols, kind_phys) * real(pver, kind_phys) * qv_init(ins) + verify_results = .false. + end if + end do + + ! Cross-instance distinctness: instance i must end up with a different + ! state than instance j, since they started from different qv. + do ins = 1, ninstances + do ins2 = ins + 1, ninstances + if (abs(q_sum(ins) - q_sum(ins2)) < tolerance) then + write(6, '(a,i0,a,i0,a)') & + 'FAIL distinctness: instance ', ins, ' and instance ', ins2, & + ' have identical totals (state leaked between instances?)' + verify_results = .false. + end if + end do + end do + + end function verify_results + +end module data diff --git a/end-to-end-tests/instances_advection/data.meta b/end-to-end-tests/instances_advection/data.meta new file mode 100644 index 00000000..f860b346 --- /dev/null +++ b/end-to-end-tests/instances_advection/data.meta @@ -0,0 +1,83 @@ +[ccpp-table-properties] + name = physics_state + type = ddt +[ccpp-arg-table] + name = physics_state + type = ddt +[ps] + standard_name = surface_air_pressure + units = hPa + dimensions = (horizontal_dimension) + type = real + kind = kind_phys +[temp] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys +[q] + standard_name = state_constituent_mixing_ratio + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + type = real + kind = kind_phys +[q(:,:,index_of_water_vapor_specific_humidity)] + standard_name = water_vapor_specific_humidity + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + + +[ccpp-table-properties] + name = data + type = host + dependencies = + +[ccpp-arg-table] + name = data + type = host +[ncols] + standard_name = horizontal_dimension + long_name = horizontal dimension + units = count + dimensions = () + type = integer + protected = True +[pver] + standard_name = vertical_layer_dimension + long_name = vertical layer dimension + units = count + dimensions = () + type = integer + protected = True +[dt] + standard_name = time_step_for_physics + long_name = time step for physics + units = s + dimensions = () + type = real + kind = kind_phys + protected = True +[tfreeze] + standard_name = water_temperature_at_freezing + long_name = freezing temperature of water at sea level + units = K + dimensions = () + type = real + kind = kind_phys + protected = True +[index_qv] + standard_name = index_of_water_vapor_specific_humidity + long_name = index of water vapor specific humidity in the constituent array + units = index + dimensions = () + type = integer + protected = True +[phys_state] + standard_name = physics_state_derived_type + long_name = per-instance physics state DDT + units = none + dimensions = (number_of_instances) + type = physics_state diff --git a/end-to-end-tests/instances_advection/main.F90 b/end-to-end-tests/instances_advection/main.F90 new file mode 100644 index 00000000..f84f44ae --- /dev/null +++ b/end-to-end-tests/instances_advection/main.F90 @@ -0,0 +1,256 @@ +program test_instances_advection + + use, intrinsic :: iso_fortran_env, only: error_unit +#ifdef _OPENMP + use omp_lib +#endif + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + use data, only: ncols, pver, ninstances, dt, tfreeze, num_time_steps + use data, only: phys_state, qv_init, index_qv + use data, only: allocate_physics_state, init_qv, set_index_qv, & + verify_results + + use test_host_ccpp_cap, only: ccpp_register, ccpp_init, ccpp_final + use test_host_ccpp_cap, only: ccpp_physics_init, ccpp_physics_run, & + ccpp_physics_timestep_init, ccpp_physics_timestep_final, & + ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_register_constituents, & + ccpp_initialize_constituents, ccpp_deallocate_dynamic_constituents + use test_host_ccpp_cap, only: ccpp_const_get_index, ccpp_constituents_array + use test_host_ccpp_cap, only: ccpp_number_constituents + + implicit none + + character(len=*), parameter :: ccpp_suite = 'cld_suite' + character(len=512) :: errmsg + integer :: errflg + integer :: nphys_threads + integer :: ins + integer :: tstep + integer :: num_consts + integer :: idx + type(ccpp_constituent_properties_t), target, allocatable :: host_consts(:) + real(kind=kind_phys), pointer :: constituents_ptr(:, :, :) + + ! Use OpenMP threading in physics (internally) where available. +#ifdef _OPENMP + nphys_threads = omp_get_max_threads() +#else + nphys_threads = 1 +#endif + + !----------------------------------------------------------------- + ! 1. CCPP register: per-instance, populates per-suite dynamic + ! constituent buffer on first instance, idempotent thereafter. + !----------------------------------------------------------------- + do ins = 1, ninstances + call ccpp_register(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(error_unit, '(2a)') 'ccpp_register failed: ', trim(errmsg) + stop 1 + end if + end do + + !----------------------------------------------------------------- + ! 2. Build the host-registered constituent list (specific_humidity) + ! and call ccpp_register_constituents per instance. Each call + ! consumes the host_consts array (new_field sets const_index on + ! the input objects), so we re-instantiate per instance. + !----------------------------------------------------------------- + do ins = 1, ninstances + if (allocated(host_consts)) deallocate(host_consts) + allocate(host_consts(1)) + call host_consts(1)%instantiate( & + std_name='water_vapor_specific_humidity', & + long_name='Water vapor specific humidity', & + diag_name='QV', units='kg kg-1', & + vertical_dim='vertical_layer_dimension', advected=.true., & + default_value=0.0_kind_phys, mixing_ratio_type='wet', & + errcode=errflg, errmsg=errmsg) + if (errflg /= 0) then + write(error_unit, '(2a)') 'instantiate failed: ', trim(errmsg) + stop 1 + end if + call ccpp_register_constituents(host_constituents=host_consts, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_register_constituents failed for instance ', ins, & + ': ', trim(errmsg) + stop 1 + end if + end do + + !----------------------------------------------------------------- + ! 3. ccpp_initialize_constituents per instance — allocates the + ! per-instance constituent storage. + !----------------------------------------------------------------- + do ins = 1, ninstances + call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, & + instance=ins, errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_initialize_constituents failed for instance ', ins, & + ': ', trim(errmsg) + stop 1 + end if + end do + + !----------------------------------------------------------------- + ! 4. Resolve the qv constituent index and number of constituents + ! (identical across instances). + !----------------------------------------------------------------- + + call ccpp_const_get_index(stdname='water_vapor_specific_humidity', & + const_index=idx, instance=1, errflg=errflg, errmsg=errmsg) + if (errflg /= 0) then + write(error_unit, '(2a)') 'ccpp_const_get_index(qv) failed: ', & + trim(errmsg) + stop 1 + end if + call set_index_qv(idx) + + call ccpp_number_constituents(num_flds=num_consts, instance=1, & + errflg=errflg, errmsg=errmsg) + if (errflg /= 0) then + write(error_unit, '(2a)') 'ccpp_number_constituents failed: ', & + trim(errmsg) + stop 1 + end if + + !----------------------------------------------------------------- + ! 5. For each instance, wire phys_state(ins) to its constituent + ! storage and seed distinct initial qv values. + !----------------------------------------------------------------- + + do ins = 1, ninstances + constituents_ptr => ccpp_constituents_array(ins) + call allocate_physics_state(ins, constituents_ptr) + call init_qv(ins) + end do + + !----------------------------------------------------------------- + ! 6. ccpp_init and ccpp_physics_init per instance. + !----------------------------------------------------------------- + do ins = 1, ninstances + call ccpp_init(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(error_unit, '(a,i0,2a)') 'ccpp_init failed for instance ', & + ins, ': ', trim(errmsg) + stop 1 + end if + end do + do ins = 1, ninstances + call ccpp_physics_init(suite_name=ccpp_suite, group_name='physics', & + lb=1, ub=ncols, thread_num=1, nthreads=1, & + nphys_threads=nphys_threads, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_physics_init failed for instance ', ins, ': ', trim(errmsg) + stop 1 + end if + end do + + !----------------------------------------------------------------- + ! 7. Timestep loop. + !----------------------------------------------------------------- + do tstep = 1, num_time_steps + do ins = 1, ninstances + call ccpp_physics_timestep_init(suite_name=ccpp_suite, & + group_name='physics', lb=1, ub=ncols, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_physics_timestep_init failed for instance ', ins, & + ': ', trim(errmsg) + stop 1 + end if + end do + do ins = 1, ninstances + call ccpp_physics_run(suite_name=ccpp_suite, group_name='physics', & + lb=1, ub=ncols, thread_num=1, nthreads=1, & + nphys_threads=nphys_threads, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_physics_run failed for instance ', ins, ': ', trim(errmsg) + stop 1 + end if + end do + do ins = 1, ninstances + call ccpp_physics_timestep_final(suite_name=ccpp_suite, & + group_name='physics', lb=1, ub=ncols, & + thread_num=1, nthreads=1, nphys_threads=nphys_threads, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_physics_timestep_final failed for instance ', ins, & + ': ', trim(errmsg) + stop 1 + end if + end do + end do + + !----------------------------------------------------------------- + ! 8. ccpp_physics_final per instance. + !----------------------------------------------------------------- + do ins = 1, ninstances + call ccpp_physics_final(suite_name=ccpp_suite, group_name='physics', & + lb=1, ub=ncols, thread_num=1, nthreads=1, & + nphys_threads=nphys_threads, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(error_unit, '(a,i0,2a)') & + 'ccpp_physics_final failed for instance ', ins, ': ', trim(errmsg) + stop 1 + end if + end do + + !----------------------------------------------------------------- + ! 9. Verify per-instance results BEFORE constituent teardown. + ! phys_state(:)%q points into the framework's per-instance + ! constituent storage; ccpp_deallocate_dynamic_constituents + ! frees that storage, so any verification must run first. + !----------------------------------------------------------------- + if (.not. verify_results(num_consts)) then + write(6, '(a)') 'FAIL: per-instance + constituents test' + stop 1 + end if + + !----------------------------------------------------------------- + ! 10. Teardown. + !----------------------------------------------------------------- + do ins = 1, ninstances + call ccpp_final(suite_name=ccpp_suite, & + instance=ins, ninstances=ninstances, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(error_unit, '(a,i0,2a)') 'ccpp_final failed for instance ', & + ins, ': ', trim(errmsg) + stop 1 + end if + end do + do ins = 1, ninstances + call ccpp_deallocate_dynamic_constituents(instance=ins) + end do + deallocate(host_consts) + + write(6, '(a)') 'PASS: per-instance + constituents test' + stop 0 + +end program test_instances_advection diff --git a/end-to-end-tests/instances_advection/main.meta b/end-to-end-tests/instances_advection/main.meta new file mode 100644 index 00000000..9623129f --- /dev/null +++ b/end-to-end-tests/instances_advection/main.meta @@ -0,0 +1,77 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[suite_name] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[group_name] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[lb] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ub] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[thread_num] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[nthreads] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[nphys_threads] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[errmsg] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[errflg] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer +[instance] + standard_name = instance_number + long_name = current model instance number + units = index + dimensions = () + type = integer +[ninstances] + standard_name = number_of_instances + long_name = number of instances for multi-instance test + units = count + dimensions = () + type = integer diff --git a/unit-tests/test_host_constituents.py b/unit-tests/test_host_constituents.py index 828f7175..685ec874 100644 --- a/unit-tests/test_host_constituents.py +++ b/unit-tests/test_host_constituents.py @@ -172,8 +172,16 @@ def test_index_of_X_declared(self): ) def test_per_suite_buffer_declared_for_producer(self): + # The per-suite buffer is a per-instance wrapper-DDT array so each + # instance owns its own scheme-registered constituent property + # objects (the wrapper type is emitted once at module scope). + self.assertIn('type :: ccpp_dyn_const_buffer_t', self.register_text) self.assertIn( - 'type(ccpp_constituent_properties_t), allocatable, target :: ' + 'type(ccpp_constituent_properties_t), allocatable :: items(:)', + self.register_text, + ) + self.assertIn( + 'type(ccpp_dyn_const_buffer_t), allocatable, target :: ' 'reg_consts_dynamic_constituents(:)', self.register_text, ) @@ -230,8 +238,11 @@ def test_initializes_table_per_instance(self): 'call ccpp_model_constituents_obj(inst_num)%initialize_table(num_consts)', self.text, ) + # Count of scheme-registered constituents comes from THIS instance's + # slot in the wrapper-DDT array, not the (no-longer-shared) buffer. self.assertIn( - 'num_consts = num_consts + size(reg_consts_dynamic_constituents, 1)', + 'num_consts = num_consts + size(' + 'reg_consts_dynamic_constituents(inst_num)%items, 1)', self.text, ) @@ -240,7 +251,9 @@ def test_iterates_host_then_suite_per_instance(self): 'end subroutine ccpp_register_constituents' )[0] host_pos = body.find('host_constituents(index)') - suite_pos = body.find('reg_consts_dynamic_constituents(index)') + suite_pos = body.find( + 'reg_consts_dynamic_constituents(inst_num)%items(index)' + ) self.assertGreater(host_pos, 0) self.assertGreater(suite_pos, host_pos) # All %new_field calls go through obj(inst_num). diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index ded760dc..220f8ae0 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -3039,20 +3039,28 @@ def test_two_pass_packs_into_buffer(self): 'end subroutine reg_consts_register' )[0] self.assertIn('First pass: count', register_body) - self.assertIn('Second pass: copy into per-suite buffer', register_body) + self.assertIn('Second pass: copy into per-instance buffer', register_body) # The constituent-providing scheme is called twice (one per pass). self.assertEqual( register_body.count('call register_constituents_register'), 2, ) def test_buffer_allocate(self): + # Outer wrapper-DDT array is sized to number_of_instances on first + # call; each instance allocates its own ``%items(num_consts)`` slot. self.assertIn( - 'allocate(reg_consts_dynamic_constituents(num_consts))', self.text, + 'allocate(reg_consts_dynamic_constituents(', + self.text, + ) + self.assertIn( + 'allocate(reg_consts_dynamic_constituents(', + self.text, ) + self.assertIn('%items(num_consts))', self.text) def test_buffer_populate_loop(self): self.assertIn( - 'reg_consts_dynamic_constituents(num_consts + i) = scheme_consts(i)', + '%items(num_consts + i) = scheme_consts(i)', self.text, ) diff --git a/unit-tests/test_trace.py b/unit-tests/test_trace.py index 28023317..7ed78390 100644 --- a/unit-tests/test_trace.py +++ b/unit-tests/test_trace.py @@ -88,7 +88,8 @@ def _ctrl_out(self): def test_emits_for_intent_in_dummies(self): out = emit_trace_block('my_sub', self._ctrl_in(), ' ') self.assertEqual(out, [ - ' if (trace) write(error_unit, *) &', + " if (trace) write(error_unit, " + "'(a,a,1x,i0,a,1x,i0,a,1x,i0)') &", " 'CCPP TRACE my_sub:', &", " ' lb=', lb, &", " ' ub=', ub, &", @@ -110,7 +111,7 @@ def test_character_wrapped_in_trim(self): entries = [_FakeEntry('suite_name', 'suite_name', 'character')] out = emit_trace_block('my_sub', entries, ' ') self.assertEqual(out, [ - ' if (trace) write(error_unit, *) &', + " if (trace) write(error_unit, '(a,a,a)') &", " 'CCPP TRACE my_sub:', &", " ' suite_name=', trim(suite_name)", ]) @@ -154,6 +155,22 @@ def test_signature_order_preserved(self): self.assertLess(first, second) self.assertLess(second, third) + def test_format_string_mixes_a_and_i0(self): + entries = [ + _FakeEntry('suite_name', 'suite_name', 'character'), + _FakeEntry('horizontal_loop_begin', 'lb', 'integer'), + _FakeEntry('horizontal_loop_end', 'ub', 'integer'), + ] + out = emit_trace_block('my_sub', entries, ' ') + # First line carries the format: ``a`` for trace name, then for + # each item ``a`` (label) followed by ``a`` (char) or + # ``1x,i0`` (integer). + self.assertEqual( + out[0], + " if (trace) write(error_unit, " + "'(a,a,a,a,1x,i0,a,1x,i0)') &", + ) + def test_continuation_only_after_last_var_omitted(self): entries = [ _FakeEntry('a', 'one', 'integer'), From eae0d48505710a8319cc423361f8327047eab58e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 18 May 2026 21:00:49 -0600 Subject: [PATCH 26/74] Add doc/briefing_pm.md --- doc/briefing_pm.md | 350 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 350 insertions(+) create mode 100644 doc/briefing_pm.md diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md new file mode 100644 index 00000000..030e2a68 --- /dev/null +++ b/doc/briefing_pm.md @@ -0,0 +1,350 @@ +# capgen-ng — Briefing for Project Management + +*Companion to `doc/briefing.md` (the developer walk-through) and +`doc/redesign_analysis.md` (the deep-dive technical comparison of +prebuild and capgen). This document targets project leadership and +program managers; it summarises the case for `capgen-ng` in terms of +product risk, schedule, and cross-organisation impact rather than +implementation detail.* + +*Last revised: 2026-05-18.* + +--- + +## TL;DR + +The CCPP Framework today ships **two** code generators that solve the +same problem differently: + +- **`ccpp-prebuild`** powers NOAA UFS, Navy NEPTUNE, and CCPP-SCM. + Simple and reliable, but feature-light — does not support features + CAM-SIMA needs (constituents, framework-owned variables, + introspection). +- **`ccpp-capgen`** powers NCAR CAM-SIMA. Feature-rich, but built on + technical choices that **do not scale** to UFS or NEPTUNE and that + **do not support multi-instance hosts** at all. + +Neither generator can be the basis for a single shared toolchain. +**`capgen-ng`** is a third generator, started in early May 2026, +designed to do everything both other generators do, in code small +enough for a few people to own, with the architectural choices that +make it work at UFS/NEPTUNE scale and beyond. The redesign is +running on the SCM as proving ground; UFS / NEPTUNE / CAM-SIMA +re-integration is sequenced behind that. + +This document explains, in plain language, **why we did not extend +capgen instead**, what risks the redesign retires, and where things +stand. + +--- + +## 1. The three generators in one paragraph each + +**`ccpp-prebuild`** (NOAA, in production for UFS, NEPTUNE, SCM). +Procedural Python; reads metadata; emits Fortran caps; passes +host-defined derived-type (DDT) arguments to scheme call sites. In +production for several years; bug rate is low; the team understands +it (those who worked with it). What it doesn't do: framework-owned +variables, the constituent mechanism CAM-SIMA needs, and runtime +introspection. Treated as the **baseline for simplicity and +reliability**. + +**`ccpp-capgen`** (NCAR, in production for CAM-SIMA). Heavy +object-oriented Python (deep class hierarchy, ~tens of thousands of +lines); reads metadata; emits Fortran caps that pass **flat scalar +fields** to scheme call sites instead of DDTs; supports the +constituent mechanism, suite-owned variables, introspection, and a +few other features prebuild lacks. **It is the only existing +generator with those features.** But — see §3 — it has structural +limits that make it impractical for UFS, NEPTUNE, or multi-instance +hosts, and the implementation is concentrated enough that few people +can extend it safely (if at all - primary developer gone). + +**`capgen-ng`** (new, 2026-05). Procedural Python (a few thousand +lines; flat data classes); reads the same metadata format; passes +arguments like prebuild; supports the features capgen pioneered +(constituents, suite-owned variables, introspection); adds +multi-instance, an integer state machine, six explicit scheme +phases, vertical-flip / unit / kind transforms, registered +scalar-index dimensions for threading and ensembles, write-if-changed +build integration, and a separate Fortran-vs-metadata validator +tool. Designed so the same generator works for prebuild-style hosts +(UFS / NEPTUNE / SCM) and capgen-style hosts (CAM-SIMA, future). + +--- + +## 2. Why this matters now + +Three pressures converged in 2025/26: + +1. **Framework unification heavily delayed.** A fully-functional + capgen that supports UFS / NEPTUNE / SCM and replaces prebuild + was promised for years, and never delivered. Pressure from + project management and sponsors is building. +2. **UFS / NEPTUNE want the capgen feature set.** Constituents + in particular are increasingly central to atmospheric physics + (chemistry, aerosols, deep atmosphere), and re-implementing the + prebuild-side glue per host is duplicated effort. Extending + capgen to UFS-scale runs into the flat-field problem (§3.1) — + not a small refactor, a fundamental data-shape change. The + performance of capgen generating multi-suite caps is up to + 20 times slower than that of prebuild for the CCPP SCM. This + is caused by fundamental design choices (five layers of + classes inheriting from each other) that are integral to capgen. +3. **The team owning capgen has limited bandwidth to extend it.** + The class hierarchy is intricate; understanding the + `ConstituentVarDict` scope-chain or the auto-clone path requires + reading several modules together. Realistically, only one or two + people on the framework team can change capgen without breaking + something downstream. One of them now lives overseas and rejects + simplification attempts from the others. This is a + **bus-factor risk** that the redesign retires. + +--- + +## 3. What capgen does that does not extend to UFS / NEPTUNE / multi-instance + +This section is for the project lead who came from the capgen side: +none of these are critiques of capgen as a *product*. They are +specific architectural choices that worked for CAM-SIMA's +single-instance design and don't generalise. Each is sourced from +the technical analysis in `doc/redesign_analysis.md` and validated +by the SCM / multi-instance test work this month. + +### 3.1 Flat-field argument passing fails at UFS / NEPTUNE scale + +CAM-SIMA's group caps pass **every individual variable as a separate +argument** to the scheme dispatch routine. At CAM-SIMA's roughly +two-hundred-variable scale this works. At UFS scale (~1200 +variables per group), the generated Fortran exceeds compiler limits +under strict error-checking flags (`-check all`, `-fcheck=all`), +fails to compile, and even when it does compile produces +unmaintainably large source files. **This is the technical reason +capgen cannot drive UFS today**, independent of any other concern. + +`capgen-ng` reverts to prebuild's DDT-argument convention. Host +authors pass their physics DDTs by reference (one or a few arguments +per scheme call); component access happens **inside the scheme**. +This works at every scale we've measured. + +### 3.2 Single-instance constituents are baked into the generated code + +CAM-SIMA runs one host per executable, so capgen generates a single +module-level `ccpp_model_constituents_obj`. The constituent +mechanism — the central feature capgen-ng inherited from capgen — +references that global directly. Re-targeting capgen to +multi-instance is not a configuration toggle; it requires +re-emitting the constituent module per-instance throughout the +generator, plus refactoring the framework setters. + +`capgen-ng` was multi-instance **from day one**: every constituent +entry point takes an `instance_number` argument; the property +storage, the state machine, the dynamic-constituent buffers are all +per-instance. As of 2026-05-18, the per-suite dynamic-constituents +buffer was also moved per-instance after the new combined +multi-instance + constituents end-to-end test surfaced a latent bug +that capgen would never have hit (because capgen never supported +multi-instance). **The redesign is finding bugs the legacy +toolchain hid.** + +### 3.3 Constituent registration has three competing paths in capgen + +capgen accepts constituent declarations from (a) host-supplied +arrays, (b) scheme `register`-phase Fortran subroutines, and (c) an +**auto-clone path** that scans scheme metadata for the +`is_constituent` attribute and silently generates a registration in +the host cap. The auto-clone path is invisible from the scheme +Fortran — to know whether a scheme registers a constituent you have +to know the generator semantics. This makes scheme code harder to +read, harder to port between hosts, and harder to debug when +registrations collide. + +`capgen-ng` keeps only the first two (explicit) paths. Auto-clone +is deliberately gone — see `doc/constituents_overhaul.md` §2.3. + +### 3.4 Host-specific values baked into scheme metadata + +capgen requires `diagnostic_name` (host's diagnostic-output label, +e.g. `CLDLIQ` for CAM-SIMA but something else for UFS) at +constituent instantiation time. Schemes therefore embed +host-specific strings into their own metadata. Porting a scheme +between hosts requires either editing the scheme or maintaining a +fork. + +`capgen-ng` is moving `diagnostic_name` (and a handful of other +host-configuration properties) to a host-side override mechanism; +schemes carry physics-portable defaults only. The reform is +documented in `doc/constituents_overhaul.md`; the decision is on the +agenda for one of the next framework-team meetings. + +### 3.5 Synthetic variable-resolution scopes are hard to extend + +capgen introduces a synthetic dictionary (`ConstituentVarDict`) +between the suite and host scopes during variable matching. The +mechanism works for capgen's use cases but is a code path most +contributors don't read. Extending the resolver to handle +multi-instance dimensions, scalar-index substitution, or +constituent host-wins semantics required undoing parts of the +synthetic scope. + +`capgen-ng`'s resolver is flat: each scheme arg is classified into +exactly one source (control / host / suite / constituent), recorded +on a small data class (`ResolvedArg`), and used directly by the +emitter. No synthetic dictionary. + +### 3.6 Code volume and team coverage + +capgen is roughly an order of magnitude larger than prebuild, with a +deeply layered class hierarchy. This is not a moral failing — it +reflects the feature set — but the practical consequence is that +the maintenance burden falls on a small subset of the framework +team. capgen-ng is comparable to prebuild in size (a few thousand +lines of mostly procedural Python with small data classes), so the +"who can fix this" pool is closer to "anyone with framework +context". capgen-ng comes with over a thousand docstring tests +and unit tests, as well as a comprehensive end-to-end test suite +that covers all of prebuild's and capgen's existing end to end +tests. capgen-ng adds additional end-to-end tests for new +features such as the multi-instance constituents test. + +--- + +## 4. What `capgen-ng` does better than capgen — at any scale + +For audiences who already accept the multi-instance and UFS-scale +arguments, the day-to-day quality-of-life improvements that apply +even to CAM-SIMA-shape problems: + +| Topic | capgen | capgen-ng | +|---|---|---| +| Scheme call argument shape | Flat fields | DDT references | +| Variable resolution | Scope-chain promotion via synthetic dict | Flat 4-source classification on `ResolvedArg` | +| Suite state runtime check | String comparison | Integer-named-parameter state machine | +| Fortran-vs-metadata validation | Embedded in generator | Standalone tool (`ccpp_validator.py`) — run by developers or CMake before generation | +| Generator code style | Deep class hierarchy | Flat data classes + procedural resolver | +| Error reporting | Variable amount of context | "Loud, specific, actionable" enforced — every parse-time error names file, line, variable, attribute, value, and reason | +| Constituent registration | Three sources (one invisible) | Two sources, both explicit | +| `is_constituent` auto-clone | Yes (host-specific values baked into scheme metadata) | Removed | +| `_finalize` vs `_final` phase name | `_finalize` | `_final` (renamed to keep symmetry with init/timestep_init/timestep_final) | + +--- + +## 5. Additional features of `capgen-ng` compared to `capgen` + +Features that exist only in capgen-ng (some exist in prebuild): + +| Capability | Why it matters | +|---|---| +| **Multi-instance host support** (per-instance state machine, per-instance constituent objects, per-instance dynamic-constituents buffers as of 2026-05-18) | Required by NEPTUNE (prebuild has basic solution) | +| **Registered scalar-index dimensions** | When metadata says a variable is dimensioned by `number_of_threads` or `number_of_instances`, capgen-ng injects the right per-call subscript automatically; the host's OpenMP-thread-private DDT layout works unchanged | +| **Subcycle loop-counter automation** | Schemes inside a `` element can access `ccpp_loop_counter` / `ccpp_loop_extent` directly; the generator emits the Fortran `do` loop and binds the locals | +| **`--legacy-mode` migration shim** | One CLI flag enables silent rewrite of two known-good deprecated standard names (`horizontal_loop_extent` → `horizontal_dimension`, `number_of_openmp_threads` → `number_of_threads`) with a loud warning — buys time for host metadata to migrate | +| **`--no-host-introspection` flag** | The five runtime introspection routines (`ccpp_physics_suite_list`, etc.) emit large `select case` blocks at SCM scale; this flag stubs the bodies, dropping the generated static API from ~33,000 lines to ~800 for the SCM build (the introspection routines were making `-O3` compilation effectively hang) | + +--- + +## 6. Where things stand right now (2026-05-18) + +- **Unit tests**: 1319 passing. No known failures. +- **End-to-end tests**: 10 passing — `chunked_data`, `opt_arg`, + `nested_suite`, `ddthost`, `instances`, `capgen_ng`, + `var_compat`, `advection`, and the new `instances_advection` + (multi-instance + constituents). +- **CCPP-SCM**: actively driving development. Each build / runtime + issue surfaced this week landed as a fix in capgen-ng rather than + a host-side workaround. All available suites in CCPP-SCM now + build and run end-to-end via `--legacy-mode`. +- **Multi-instance + constituents fix landed 2026-05-18**. The new + combined end-to-end test surfaced a latent shared-buffer mutation + bug; the fix moves the per-suite dynamic-constituents buffer + per-instance. No coordination with CAM-SIMA / UFS / NEPTUNE + required (host-facing API unchanged). +- **NEPTUNE**: cleanup and acceptance testing in progress this week. +- **UFS Weather Model**: not yet attempted; SCM is the proving + ground first. +- **CAM-SIMA**: not yet re-connected; the constituent overhaul + decision (see §7) and the availability of CAM-SIMA developer + are the gating items. + +--- + +## 7. What is intentionally NOT decided yet + +The redesign is opinionated about the architectural choices (DDT +arguments, per-instance everything, integer state machine, two-tool +split). It is **not** opinionated about the framework-level +constituent reform. + +`doc/constituents_overhaul.md` lays out three reform proposals on +the table: + +- **Proposal A** (mostly landed): bug-fix on the deallocate path + + add missing host setters for properties the host wants to + override. Conservative. +- **Proposal B** (recommended for the next 4–6 weeks): relax the + identity-equality check, formally classify properties as + "scheme-intrinsic" (immutable) vs "host-configuration" (mutable + after registration). Physics schemes become genuinely portable + across hosts. +- **Proposal C** (tabled): drop scheme-side constituent + registration entirely; only the host registers. Cleaner but + requires coordinated PRs across the framework, both generators, + the CAM-SIMA atmospheric_physics tree, and CAM-SIMA itself. + +These are open questions for the framework-team meeting, not +capgen-ng decisions. capgen-ng is structured so all three +proposals are implementable on top of it. + +--- + +## 8. Risk register (project-management view) + +| Risk | Status | Mitigation | +|---|---|---| +| capgen-ng diverges from capgen feature set | LOW | Cross-checked by `doc/redesign_analysis.md`; the feature comparison table in §4 / §5 is exhaustive | +| Host metadata break for UFS / NEPTUNE / CAM-SIMA | MEDIUM | `--legacy-mode` shim covers the known incompatible standard-name pair; remaining required changes (e.g., `_finalize` → `_final`) are mechanical and listed in `doc/migration.md` §3 | +| Constituent overhaul stalls | MEDIUM | Proposal A unblocks the immediate bug; capgen-ng works with the current framework today; the overhaul is a separate decision track | +| Bus-factor on capgen-ng itself | MEDIUM | Procedural code style + flat data classes + 1319-test safety net; significantly lower than capgen's bus factor | +| Two host call-shape conventions (prebuild-style vs capgen-style) coexist forever | LOW | capgen-ng emits one shape; downstream host conversions are tracked in `doc/migration.md` | +| Regression discovered during NEPTUNE / UFS testing | EXPECTED | SCM proving ground catches most; remaining issues become capgen-ng tickets, not host-side patches | +| ccpp-prebuild end-of-life requires a sunset plan | OPEN | Not yet scoped; both generators currently coexist in the framework repo | + +--- + +## 9. The pragmatic case (for the meeting) + +Three points worth raising explicitly: + +1. **Extending capgen to UFS/NEPTUNE scale is not a configuration + change — it is a refactor of the same magnitude as a redesign.** + The flat-field convention is load-bearing throughout capgen's + variable-matching, resolution, and emission code. Once that + change is made, the resulting generator looks substantially + like capgen-ng anyway. +2. **The features capgen pioneered (constituents, suite-owned + variables, introspection) are kept and improved — not + discarded.** capgen-ng is genuinely the successor, not a + parallel project. The contributions made on the capgen side are + what made the capgen-ng feature set possible. A significant + portion of capgen's code, in particular metadata parsine, + Fortran-metadata validation, and constituents, were imported + into capgen-ng. +3. **The team owning capgen-ng can be larger than the team owning + capgen.** This is the most important practical point for + long-term programme health. A framework that two organisations + can maintain is more resilient than a framework that one + organisation (or one individual in that organisation) can maintain. + +--- + +## 10. References + +- `doc/briefing.md` — developer walk-through; same outline, more + technical detail. +- `doc/redesign_analysis.md` — deep-dive technical comparison of + prebuild and capgen with named-product examples. +- `doc/migration.md` — host-author migration guide. +- `doc/constituents_overhaul.md` — the constituent-reform discussion + document. +- `end-to-end-tests/` — the working examples (`instances_advection` + is the newest, exercises everything end-to-end). From 68eff476fd745dd83499bebb6b2c3fe23ff4d347 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 May 2026 07:29:08 -0600 Subject: [PATCH 27/74] Clean up dead code in capgen-ng/** --- capgen-ng/ccpp_datafile.py | 3 - capgen-ng/metadata/parse_tools/__init__.py | 11 +- .../metadata/parse_tools/parse_checkers.py | 899 +++--------------- capgen-ng/metadata/parse_tools/parse_log.py | 34 +- .../metadata/parse_tools/parse_object.py | 166 ---- .../metadata/parse_tools/parse_source.py | 257 +---- capgen-ng/metadata/parse_tools/xml_tools.py | 685 ++++++------- 7 files changed, 434 insertions(+), 1621 deletions(-) delete mode 100644 capgen-ng/metadata/parse_tools/parse_object.py diff --git a/capgen-ng/ccpp_datafile.py b/capgen-ng/ccpp_datafile.py index 777a3e8c..dd55d872 100755 --- a/capgen-ng/ccpp_datafile.py +++ b/capgen-ng/ccpp_datafile.py @@ -36,16 +36,13 @@ scheme entries. """ -# Python library imports import argparse import sys import xml.etree.ElementTree as ET from typing import List, Optional -# Module-level indent string used by --show; overridden via --indent. _INDENT_STR = " " -## datatable_report must have an action for each report type _VALID_REPORTS = [ {"report": "host_files", "type": bool, "help": "Return a list of host CAP Fortran files created by capgen"}, diff --git a/capgen-ng/metadata/parse_tools/__init__.py b/capgen-ng/metadata/parse_tools/__init__.py index 24b357f8..9131cf34 100644 --- a/capgen-ng/metadata/parse_tools/__init__.py +++ b/capgen-ng/metadata/parse_tools/__init__.py @@ -4,11 +4,8 @@ CCPPError, ParseSyntaxError, ParseInternalError, - ParseContextError, ParseContext, - ParseSource, context_string, - type_name, ) from .parse_log import init_log, set_log_level, set_log_to_null, set_log_to_stdout from .parse_checkers import ( @@ -22,15 +19,9 @@ check_fortran_type, check_fortran_intrinsic, check_molar_mass, - FORTRAN_ID, FORTRAN_SCALAR_REF_RE, - FORTRAN_INTRINSIC_TYPES, -) -from .parse_object import ParseObject -from .fortran_conditional import ( - FORTRAN_CONDITIONAL_REGEX, - FORTRAN_CONDITIONAL_REGEX_WORDS, ) +from .fortran_conditional import FORTRAN_CONDITIONAL_REGEX from .xml_tools import ( read_xml_file, find_schema_version, diff --git a/capgen-ng/metadata/parse_tools/parse_checkers.py b/capgen-ng/metadata/parse_tools/parse_checkers.py index dd4d1124..f0621fa4 100644 --- a/capgen-ng/metadata/parse_tools/parse_checkers.py +++ b/capgen-ng/metadata/parse_tools/parse_checkers.py @@ -1,13 +1,10 @@ #!/usr/bin/env python3 -"""Helper functions to validate parsed input""" +"""Helper functions to validate parsed input.""" -# Python library imports import re -# CCPP framework imports -from .parse_source import CCPPError, ParseInternalError -######################################################################## +from .parse_source import CCPPError _UNITLESS_REGEX = "1" _NON_LEADING_ZERO_NUM = r"[1-9]\d*" @@ -20,6 +17,7 @@ _UNITS_RE = re.compile(_UNITS_REGEX) _MAX_MOLAR_MASS = 10000.0 + def check_units(test_val, prop_dict, error): """Return if a valid unit, otherwise, None if is True, raise an Exception if is not valid. @@ -38,9 +36,6 @@ def check_units(test_val, prop_dict, error): >>> check_units('', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: '' is not a valid unit - >>> check_units(' ', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '' is not a valid unit >>> check_units(['foo'], None, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: ['foo'] is invalid; not a string @@ -49,19 +44,14 @@ def check_units(test_val, prop_dict, error): if _UNITS_RE.match(test_val.strip()) is None: if error: raise CCPPError("'{}' is not a valid unit".format(test_val)) - else: - test_val = None - # end if - # end if + test_val = None else: if error: raise CCPPError("'{}' is invalid; not a string".format(test_val)) - else: - test_val = None - # end if - # end if + test_val = None return test_val + def check_dimensions(test_val, prop_dict, error, max_len=0): """Return if a valid dimensions list, otherwise, None If > 0, each string in must not be longer than @@ -71,176 +61,103 @@ def check_dimensions(test_val, prop_dict, error, max_len=0): ['dim1', 'dim2name'] >>> check_dimensions([":", ":"], None, False) [':', ':'] - >>> check_dimensions([":", "dim2"], None, False) - [':', 'dim2'] - >>> check_dimensions(["dim1", ":"], None, False) - ['dim1', ':'] >>> check_dimensions(["8", "::"], None, False) ['8', '::'] >>> check_dimensions(['start1:end1', 'start2:end2'], None, False) ['start1:end1', 'start2:end2'] - >>> check_dimensions(['start1:', 'start2:end2'], None, False) - ['start1:', 'start2:end2'] - >>> check_dimensions(['start1 :end1', 'start2: end2'], None, False) - ['start1 :end1', 'start2: end2'] >>> check_dimensions(['size(foo)'], None, False) ['size(foo)'] - >>> check_dimensions(['size(foo,1) '], None, False) - ['size(foo,1) '] >>> check_dimensions(['size(foo,1'], None, False) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: Invalid dimension component, size(foo,1 - >>> check_dimensions(["dim1", "dim2name"], None, False, max_len=5) - >>> check_dimensions(["dim1", "dim2name"], None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: 'dim2name' is too long (> 5 chars) >>> check_dimensions("hi_mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: 'hi_mom' is invalid; not a list - >>> check_dimensions(["1:dim1", "dim2name"], None, True) - ['1:dim1', 'dim2name'] - >>> check_dimensions(["2:dim1", "dim2name"], None, True) - ['2:dim1', 'dim2name'] >>> check_dimensions(["ccpp_constant_one:1", "dim2name"], None, True) ['ccpp_constant_one:1', 'dim2name'] """ if not isinstance(test_val, list): if error: raise CCPPError("'{}' is invalid; not a list".format(test_val)) - else: - test_val = None - # end if - else: - for item in test_val: - isplit = item.split(':') - # Check for too many colons - if (len(isplit) > 3): + return None + for item in test_val: + isplit = item.split(':') + if len(isplit) > 3: + if error: + raise CCPPError("'{}' is an invalid dimension range".format(item)) + return None + # Integer literals are valid in any bound position; semantic + # restrictions (e.g. horizontal_dimension lower bound must be 1) + # are enforced by the resolver, not here. + tdims = [x.strip() for x in isplit if len(x) > 0] + for tdim in tdims: + try: + int(tdim) + valid = True + except ValueError: + valid = check_fortran_id(tdim, None, error, + max_len=max_len) is not None + if not valid and tdim.strip().lower()[0:4] == 'size': + if -1 in check_balanced_paren(tdim[4:]): + raise CCPPError( + 'Invalid dimension component, {}'.format(tdim)) + valid = True + if not valid: if error: - errmsg = "'{}' is an invalid dimension range" - raise CCPPError(errmsg.format(item)) - else: - test_val = None - # end if - break - # end if - # Check possible dim styles (a, a:b, a:, :b, :, ::, a:b:c, a::c). - # Integer literals are valid in any bound position; semantic - # restrictions (e.g. horizontal_dimension lower bound must be 1) - # are enforced by the resolver, not here. - tdims = [x.strip() for x in isplit if len(x) > 0] - for tdim in tdims: - try: - valid = isinstance(int(tdim), int) - except ValueError as ve: - # Not an integer, try a Fortran ID - valid = check_fortran_id(tdim, None, - error, max_len=max_len) is not None - if not valid: - # Check for size entry -- simple check - tcheck = tdim.strip().lower() - if tcheck[0:4] == 'size': - ploc = check_balanced_paren(tdim[4:]) - if -1 in ploc: - emsg = 'Invalid dimension component, {}' - raise CCPPError(emsg.format(tdim)) - else: - valid = tdim - # end if - # end if - # end if - # End try - if not valid: - if error: - raise CCPPError(f"'{item}' is an invalid dimension name") - else: - test_val = None - # end if - break - # end if - # end for - # end for - # end if + raise CCPPError(f"'{item}' is an invalid dimension name") + return None return test_val -######################################################################## -# CF_ID is a string representing the regular expression for CF Standard Names CF_ID = r"(?i)[a-z][a-z0-9_]*" -__CFID_RE = re.compile(CF_ID+r"$") +__CFID_RE = re.compile(CF_ID + r"$") + def check_cf_standard_name(test_val, prop_dict, error): - """Return if a valid CF Standard Name, otherwise, None + """Return if a valid CF Standard Name, otherwise, None. http://cfconventions.org/Data/cf-standard-names/docs/guidelines.html if is True, raise an Exception if is not valid. >>> check_cf_standard_name("hi_mom", None, False) 'hi_mom' >>> check_cf_standard_name("hi mom", None, False) - >>> check_cf_standard_name("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is not a valid CF Standard Name >>> check_cf_standard_name("", None, False) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: CCPP Standard Name cannot be blank - >>> check_cf_standard_name("_hi_mom", None, False) - - >>> check_cf_standard_name("2pac", None, False) - >>> check_cf_standard_name("Agood4tranID", None, False) 'agood4tranid' - >>> check_cf_standard_name("agoodcfid", None, False) - 'agoodcfid' """ if len(test_val) == 0: raise CCPPError("CCPP Standard Name cannot be blank") - else: - match = __CFID_RE.match(test_val) - # end if - if match is None: + if __CFID_RE.match(test_val) is None: if error: - errmsg = "'{}' is not a valid CCPP Standard Name" - raise CCPPError(errmsg.format(test_val)) - else: - test_val = None - # end if - else: - test_val = test_val.lower() - # end if - return test_val - -######################################################################## - -### Fortran-specific parsing helper variables and functions + raise CCPPError( + "'{}' is not a valid CCPP Standard Name".format(test_val)) + return None + return test_val.lower() -######################################################################## -# FORTRAN_ID is a string representing the regular expression for Fortran names FORTRAN_ID = r"([A-Za-z][A-Za-z0-9_]*)" -__FID_RE = re.compile(FORTRAN_ID+r"$") -# Note that the scalar array reference expressions below are not really for -# scalar references because a colon can be a placeholder, unlike in Fortran code +__FID_RE = re.compile(FORTRAN_ID + r"$") +# Scalar array-reference pattern below allows `:` placeholders, unlike +# real Fortran code, so the same regex covers both scalar refs and +# slice descriptors in metadata. __FORTRAN_AID = r"(?:[A-Za-z][A-Za-z0-9_]*)" __FORT_INT = r"[0-9]+" -__FORT_DIM = r"(?:"+__FORTRAN_AID+r"|[:]|"+__FORT_INT+r")" -__REPEAT_DIM = r"(?:,\s*"+__FORT_DIM+r"\s*)" -__FORTRAN_SCALAR_ARREF = r"[(]\s*("+__FORT_DIM+r"\s*"+__REPEAT_DIM+r"{0,6})[)]" -# FORTRAN_SCALAR_REF: Pattern of a valid Fortran array reference -# NB: Only allows symbols, no expressions and/or function calls -FORTRAN_SCALAR_REF = r"(?:"+FORTRAN_ID+r"\s*"+__FORTRAN_SCALAR_ARREF+r")" -FORTRAN_SCALAR_REF_RE = re.compile(FORTRAN_SCALAR_REF+r"$") -# FORTRAN_FUNCTION_REF: A Fortran function reference -# NB: Currenly does not support function arguments -FORTRAN_FUNCTION_REF = r"(?:"+FORTRAN_ID+r"\s*[(]\s*[)])" -FORTRAN_FUNCTION_REF_RE = re.compile(FORTRAN_FUNCTION_REF) +__FORT_DIM = r"(?:" + __FORTRAN_AID + r"|[:]|" + __FORT_INT + r")" +__REPEAT_DIM = r"(?:,\s*" + __FORT_DIM + r"\s*)" +__FORTRAN_SCALAR_ARREF = r"[(]\s*(" + __FORT_DIM + r"\s*" + __REPEAT_DIM + r"{0,6})[)]" +FORTRAN_SCALAR_REF_RE = re.compile( + r"(?:" + FORTRAN_ID + r"\s*" + __FORTRAN_SCALAR_ARREF + r")$") FORTRAN_INTRINSIC_TYPES = ["integer", "real", "logical", "complex", "double precision", "character"] FORTRAN_DP_RE = re.compile(r"(?i)double\s*precision") -FORTRAN_TYPE_RE = re.compile(r"(?i)type\s*\(\s*("+FORTRAN_ID+r")\s*\)") _REGISTERED_FORTRAN_DDT_NAMES = ["ccpp_constituent_prop_ptr_t"] -######################################################################## def check_fortran_id(test_val, prop_dict, error, max_len=0): """Return if a valid Fortran identifier, otherwise, None @@ -248,566 +165,126 @@ def check_fortran_id(test_val, prop_dict, error, max_len=0): if is True, raise an Exception if is not valid. >>> check_fortran_id("hi_mom", None, False) 'hi_mom' - >>> check_fortran_id("hi_mom", None, False, max_len=5) - >>> check_fortran_id("hi_mom", None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: 'hi_mom' is too long (> 5 chars) - >>> check_fortran_id("hi mom", None, False) - >>> check_fortran_id("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: 'hi_mom' is not a valid Fortran identifier - >>> check_fortran_id("", None, False) - - >>> check_fortran_id("_hi_mom", None, False) - >>> check_fortran_id("2pac", None, False) >>> check_fortran_id("Agood4tranID", None, False) 'Agood4tranID' """ - match = __FID_RE.match(test_val) - if match is None: + if __FID_RE.match(test_val) is None: if error: - raise CCPPError("'{}' is not a valid Fortran identifier".format(test_val)) - else: - test_val = None - # end if - elif (max_len > 0) and (len(test_val) > max_len): + raise CCPPError( + "'{}' is not a valid Fortran identifier".format(test_val)) + return None + if max_len > 0 and len(test_val) > max_len: if error: - raise CCPPError("'{}' is too long (> {} chars)".format(test_val, max_len)) - else: - test_val = None - # end if - # end if + raise CCPPError( + "'{}' is too long (> {} chars)".format(test_val, max_len)) + return None return test_val -######################################################################## - -def fortran_list_match(test_str): - """Check if could be a list of Fortran expressions. - The list must be enclosed in parentheses and separated by commas. - If the list appears okay, return the items (for further checking) - >>> fortran_list_match('(ccpp_constant_one:dim1)') - ['ccpp_constant_one:dim1'] - >>> fortran_list_match('(foo, bar)') - ['foo', 'bar'] - >>> fortran_list_match('()') - [''] - >>> fortran_list_match('(foo, ,)') - - >>> fortran_list_match('foo, bar') - - >>> fortran_list_match('(foo, bar') - - """ - parens, parene = check_balanced_paren(test_str) - if (parens >= 0) and (parene > parens): - litems = [x.strip() for x in test_str[parens+1:parene].split(',')] - if (len(litems) > 1) and (min([len(x) for x in litems]) == 0): - litems = None - # end if - else: - litems = None - # end if - return litems - -######################################################################## def check_fortran_ref(test_val, prop_dict, error, max_len=0): """Return if a valid simple Fortran variable reference, otherwise, None. A simple Fortran variable reference is defined as a scalar id or a scalar array reference. if is True, raise an Exception if is not valid. - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(1) - 'foo' - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2) - 'bar, baz ' - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2).split(',')[0].strip() - 'bar' - >>> FORTRAN_SCALAR_REF_RE.match("foo( :, baz )").group(2).split(',')[0].strip() - ':' - >>> FORTRAN_SCALAR_REF_RE.match("foo( bar, baz )").group(2).split(',')[1].strip() - 'baz' >>> check_fortran_ref("hi_mom", None, False) 'hi_mom' - >>> check_fortran_ref("hi_mom", None, False, max_len=5) - - >>> check_fortran_ref("hi_mom", None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'hi_mom' is too long (> 5 chars) - >>> check_fortran_ref("hi mom", None, False) - >>> check_fortran_ref("hi mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: 'hi_mom' is not a valid Fortran identifier - >>> check_fortran_ref("", None, False) - - >>> check_fortran_ref("_hi_mom", None, False) - - >>> check_fortran_ref("2pac", None, False) - - >>> check_fortran_ref("Agood4tranID", None, False) - 'Agood4tranID' >>> check_fortran_ref("foo(bar)", None, False) 'foo(bar)' >>> check_fortran_ref("foo( bar, baz )", None, False) 'foo( bar, baz )' >>> check_fortran_ref("foo( :, baz )", None, False) 'foo( :, baz )' - >>> check_fortran_ref("foo( bar, )", None, False) - >>> check_fortran_ref("foo( bar, )", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: 'foo( bar, )' is not a valid Fortran scalar reference - >>> check_fortran_ref("foo()", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'foo()' is not a valid Fortran scalar reference >>> check_fortran_ref("foo(bar, bazz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: 'bazz' is too long (> 3 chars) in foo(bar, bazz) - >>> check_fortran_ref("foo(barr, baz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'bazr' is too long (> 3 chars) in foo(barr, baz) - >>> check_fortran_ref("fooo(bar, baz)", None, True, max_len=3) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'foo' is too long (> 3 chars) in fooo(bar, baz) """ idval = check_fortran_id(test_val, prop_dict, False, max_len=max_len) - if idval is None: - match = FORTRAN_SCALAR_REF_RE.match(test_val) - if match is None: - if error: - emsg = "'{}' is not a valid Fortran scalar reference" - raise CCPPError(emsg.format(test_val)) - else: - test_val = None - # end if - elif max_len > 0: - tokens = test_val.strip().rstrip(')').split('(') - tokens = [tokens[0].strip()] + [x.strip() - for x in tokens[1].split(',')] - for token in tokens: - if len(token) > max_len: - if error: - emsg = "'{}' is too long (> {} chars) in {}" - raise CCPPError(emsg.format(token, max_len, test_val)) - else: - test_val = None - break - # end if - # end if - # end for - # end if - # end if + if idval is not None: + return test_val + if FORTRAN_SCALAR_REF_RE.match(test_val) is None: + if error: + raise CCPPError( + "'{}' is not a valid Fortran scalar reference".format(test_val)) + return None + if max_len > 0: + tokens = test_val.strip().rstrip(')').split('(') + tokens = [tokens[0].strip()] + [x.strip() for x in tokens[1].split(',')] + for token in tokens: + if len(token) > max_len: + if error: + raise CCPPError( + "'{}' is too long (> {} chars) in {}".format( + token, max_len, test_val)) + return None return test_val -######################################################################## - -def check_local_name(test_val, prop_dict, error, max_len=0): - """Return if a valid simple Fortran variable reference, - or Fortran constant, otherwise, None. - A simple Fortran variable reference is defined as a scalar id or a - scalar array reference. - A constant is only valid if is not None, the 'protected' - property is present and True, and the 'type' property matches the - type of . - if is True, raise an Exception if is not valid. - >>> check_local_name("hi_mom", None, error=False) - 'hi_mom' - >>> check_local_name('122', {'protected':True,'type':'integer'}, error=False) - '122' - >>> check_local_name('122', None, error=False) - - >>> check_local_name('122', {}, error=False) - - >>> check_local_name('122', {'protected':False,'type':'integer'}, error=False) - - >>> check_local_name('122', {'protected':True,'type':'real'}, error=False) - - >>> check_local_name('-122.e4', {'protected':True,'type':'real'}, error=False) - '-122.e4' - >>> check_local_name('-122.', {'protected':True,'type':'real','kind':'kp'}, error=False) - - >>> check_local_name('-122._kp', {'protected':True,'type':'real','kind':'kp'}, error=False) - '-122._kp' - >>> check_local_name('q(:,:,index_of_water_vapor_specific_humidity)', {}, error=False) - 'q(:,:,index_of_water_vapor_specific_humidity)' - """ - valid_val = None - # First check for a constant - if (prop_dict is not None) and ('protected' in prop_dict): - protected = prop_dict['protected'] - else: - protected = False - # end if - if (prop_dict is not None) and ('type' in prop_dict): - vtype = prop_dict['type'] - else: - vtype = "" - # end if - if (prop_dict is not None) and ('kind' in prop_dict): - kind = prop_dict['kind'] - else: - kind = "" - # end if - if protected and vtype and check_fortran_literal(test_val, vtype, kind): - valid_val = test_val - # end if - if valid_val is None: - valid_val = check_fortran_ref(test_val, prop_dict, error, max_len=max_len) - # end if - return valid_val - - -######################################################################## def check_fortran_intrinsic(typestr, error=False): - """Return if a valid Fortran intrinsic type, otherwise, None - if is True, raise an Exception if is not valid. + """Return if a valid Fortran intrinsic type, otherwise, None + if is True, raise an Exception if is not valid. >>> check_fortran_intrinsic("real", error=False) 'real' - >>> check_fortran_intrinsic("complex") - 'complex' - >>> check_fortran_intrinsic("integer") - 'integer' >>> check_fortran_intrinsic("InteGer") 'InteGer' - >>> check_fortran_intrinsic("logical") - 'logical' - >>> check_fortran_intrinsic("character") - 'character' >>> check_fortran_intrinsic("double precision") 'double precision' - >>> check_fortran_intrinsic("double precision") - 'double precision' >>> check_fortran_intrinsic("doubleprecision") 'doubleprecision' >>> check_fortran_intrinsic("char", error=True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: 'char' is not a valid Fortran type - >>> check_fortran_intrinsic("int") - - >>> check_fortran_intrinsic("char", error=False) - - >>> check_fortran_intrinsic("type") - >>> check_fortran_intrinsic("complex(kind=r8)") """ chk_type = typestr.strip().lower() match = chk_type in FORTRAN_INTRINSIC_TYPES - if (not match) and (chk_type[0:6] == 'double'): - # Special case for double precision + if not match and chk_type[0:6] == 'double': match = FORTRAN_DP_RE.match(chk_type) is not None - # End if if not match: if error: raise CCPPError("'{}' is not a valid Fortran type".format(typestr)) - else: - typestr = None - # end if - # end if + return None return typestr -######################################################################## def check_fortran_type(typestr, prop_dict, error): """Return if a valid Fortran type, otherwise, None if is True, raise an Exception if is not valid. >>> check_fortran_type("real", None, False) 'real' - >>> check_fortran_type("integer", None, False) - 'integer' - >>> check_fortran_type("InteGer", None, False) - 'InteGer' - >>> check_fortran_type("character", None, False) - 'character' - >>> check_fortran_type("double precision", None, False) - 'double precision' - >>> check_fortran_type("double precision", None, False) - 'double precision' - >>> check_fortran_type("doubleprecision", None, False) - 'doubleprecision' - >>> check_fortran_type("complex", None, False) - 'complex' >>> check_fortran_type("char", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: 'char' is not a valid Fortran type - >>> check_fortran_type("int", None, False) - - >>> check_fortran_type("char", {}, False) - - >>> check_fortran_type("type", None, False) - >>> check_fortran_type("type", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: 'type' is not a valid derived Fortran type - >>> check_fortran_type("type(hi mom)", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'type(hi mom)' is not a valid derived Fortran type """ dt = "" match = check_fortran_intrinsic(typestr, error=False) if match is None: match = registered_fortran_ddt_name(typestr) dt = " derived" - # end if if match is None: if error: - emsg = "'{}' is not a valid{} Fortran type" - raise CCPPError(emsg.format(typestr, dt)) - else: - typestr = None - # end if - # end if + raise CCPPError( + "'{}' is not a valid{} Fortran type".format(typestr, dt)) + return None return typestr -######################################################################## - -def check_fortran_literal(value, typestr, kind): - """Return True iff is a valid Fortran literal of type, . - Note: no attempt is made to handle the older D syntax for real literals. - To promote clean coding, real values MUST have a decimal point, however, - this check is not available for the complex type so we just require - the two components to either both be integers or both be reals. - If is not an empty string, it is required to be present (i.e., if - == 'kind_phys', should be of the form, 123.4_kind_phys) - >>> check_fortran_literal("123", "integer", "") - True - >>> check_fortran_literal("123", "INTEGER", "") - True - >>> check_fortran_literal("-123", "integer", "") - True - >>> check_fortran_literal("+123", "integer", "") - True - >>> check_fortran_literal("+123", "integer", "kind_int") - False - >>> check_fortran_literal("+123_kind_int", "integer", "kind_int") - True - >>> check_fortran_literal("+123_int", "integer", "kind_int") - False - >>> check_fortran_literal("123", "real", "") - False - >>> check_fortran_literal("123.", "real", "") - True - >>> check_fortran_literal("123.45", "real", "kind_phys") - False - >>> check_fortran_literal("123.45_8", "real", "kind_phys") - False - >>> check_fortran_literal("123.45_kind_phys", "real", "kind_phys") - True - >>> check_fortran_literal("123", "double precision", "") - False - >>> check_fortran_literal("123.", "doubleprecision", "") - True - >>> check_fortran_literal("123.45", "double precision", "kind_phys") - False - >>> check_fortran_literal("123.45_8", "doubleprecision", "kind_phys") - False - >>> check_fortran_literal("123.45_kp", "doubleprecision", "kp") - True - >>> check_fortran_literal("123", "logical", "") - False - >>> check_fortran_literal(".true.", "logical", "") - True - >>> check_fortran_literal(".false.", "logical", "") - True - >>> check_fortran_literal("T", "logical", "") - False - >>> check_fortran_literal("F", "logical", "") - False - >>> check_fortran_literal(".TRUE.", "logical", "kind_log") - False - >>> check_fortran_literal(".TRUE._kind_log", "logical", "kind_log") - True - >>> check_fortran_literal("(123.,456.)", "complex", "") - True - >>> check_fortran_literal("(123. , 456.)", "complex", "") - True - >>> check_fortran_literal("(123.,456", "complex", "") - False - >>> check_fortran_literal("(123. , 456.)", "complex", "kp") - False - >>> check_fortran_literal("(123._kp , 456)", "complex", "kp") - False - >>> check_fortran_literal("(123._kp , 456._kp)", "complex", "kp") - True - >>> check_fortran_literal("'hi mom'", "character", "") - True - >>> check_fortran_literal("'hi mom", "character", "") - False - >>> check_fortran_literal('"hi mom"', "character", "") - True - >>> check_fortran_literal('"hi""mom"', "character", "") - True - >>> check_fortran_literal('"hi" "mom"', "character", "") - False - >>> check_fortran_literal("'hi''there''mom'", "character", "") - True - >>> check_fortran_literal("'hi mom'", "character", "kc") - False - >>> check_fortran_literal("kc_'hi mom'", "character", "kc") - True - >>> check_fortran_literal("123._kp", "float", "kp") #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ParseInternalError: ERROR: 'float' is not a Fortran intrinsic type - """ - valid = True - if FORTRAN_DP_RE.match(typestr.strip()) is not None: - vtype = 'real' - else: - vtype = typestr.lower() - # end if - # Check complex first - if vtype == 'complex': - cvals = value.strip().split(',') - if len(cvals) == 2: - tp = 'integer' - if ('.' in cvals[0]) and ('.' in cvals[1]): - tp = 'real' - elif ('.' in cvals[0]) or ('.' in cvals[1]): - valid = False - # end if - if (cvals[0][0] == '(') and (cvals[1][-1] == ')'): - valid = valid and check_fortran_literal(cvals[0][1:], tp, kind) - valid = valid and check_fortran_literal(cvals[1][:-1], tp, kind) - else: - valid = False - # end if - else: - valid = False - elif valid: - vparts = value.strip().split('_') - if vtype == 'character': - if len(vparts) > 1: - val = vparts[-1] - vkind = '_'.join(vparts[0:-1]) - else: - val = vparts[0] - vkind = '' - # end if - else: - val = vparts[0] - if len(vparts) > 1: - vkind = '_'.join(vparts[1:]) - else: - vkind = '' - # end if - # end if - if vkind != kind.lower(): - valid = False - # end if, kind is okay, check value - if valid and (vtype == 'integer'): - try: - vtest = int(val) - except ValueError as ve: - valid = False - # End try - elif valid and (vtype == 'real'): - if '.' not in val: - valid = False - else: - try: - vtest = float(val) - except ValueError as ve: - valid = False - # End try - # end if - elif valid and (vtype == 'logical'): - valid = (val.upper() == '.TRUE.') or (val.upper() == '.FALSE.') - elif valid and (vtype == 'character'): - sep = val[0] - cparts = val.split(sep) - # We must have balanced delimiters - if len(cparts)%2 == 0: - valid = False - else: - for index in range(len(cparts)): - if (index%2 == 0) and (len(cparts[index]) > 0): - valid = False - break - # end if - # end for - # end if (else okay) - elif valid: - errmsg = "ERROR: '{}' is not a Fortran intrinsic type" - raise ParseInternalError(errmsg.format(typestr)) - # end if (no else) - # end if - return valid - -def check_default_value(test_val, prop_dict, error): - """Return if a valid default value for a CCPP field, - otherwise, None. - If is True, raise an Exception if is not valid. - A valid value is determined by the 'type' of the variable. It is an - error for there to be no 'type' property in . - >>> check_default_value('314', {'type':'integer'}, False) - '314' - >>> check_default_value('314', {'type':'integer'}, True) - '314' - >>> check_default_value('314', {'type':'integer', 'kind':'ikind'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 314 is not a valid Fortran integer of kind, ikind - >>> check_default_value('314_ikind', {'type':'integer', 'kind':'ikind'}, True) - '314_ikind' - >>> check_default_value('314', {'type':'real'}, False) - - >>> check_default_value('314', {'type':'real'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 314 is not a valid Fortran real - >>> check_default_value('3.14', {'type':'real'}, False) - '3.14' - >>> check_default_value('314', {'tipe':'integer'}, False) - - >>> check_default_value('314', {'local_name':'foo'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: foo does not have a 'type' attribute - >>> check_default_value('314', {'tipe':'integer'}, False) - - >>> check_default_value('314', None, True) - '314' - """ - valid = None - if prop_dict and ('type' in prop_dict): - valid = test_val - var_type = prop_dict['type'].lower().strip() - if 'kind' in prop_dict: - vkind = prop_dict['kind'].lower().strip() - else: - vkind = '' - # end if - if not check_fortran_literal(test_val, var_type, vkind): - valid = None - if error: - emsg = '{} is not a valid Fortran {}' - if vkind: - emsg += ' of kind, {}' - raise CCPPError(emsg.format(test_val, var_type, vkind)) - # end if - # end if (no else, is okay) - elif prop_dict is None: - # Special case for checks during parsing, always pass - valid = test_val - elif error: - emsg = "{} does not have a 'type' attribute" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(lname)) - # end if - return valid - -def check_valid_values(test_val, prop_dict, error): - """Return if a valid 'valid_values' attribute value, - otherwise, None. - If is True, raise an Exception if is not valid. - """ - raise ParseInternalError("NOT IMPLEMENTED") def check_diagnostic_fixed(test_val, prop_dict, error): """Return if a valid descriptor for a CCPP diagnostic, @@ -817,12 +294,6 @@ def check_diagnostic_fixed(test_val, prop_dict, error): an error to specify both 'diagnostic_name' and 'diagnostic_name_fixed'. >>> check_diagnostic_fixed("foo", {'diagnostic_name_fixed' : 'foo'}, False) 'foo' - >>> check_diagnostic_fixed("foo", {'diagnostic_name_fixed' : 'foo'}, True) - 'foo' - >>> check_diagnostic_fixed("foo", {'diagnostic_name' : 'foo'}, False) - - >>> check_diagnostic_fixed("foo", {'diagnostic_name':'','local_name':'hi','standard_name':'mom'}, True) - 'foo' >>> check_diagnostic_fixed("foo", {'diagnostic_name':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: hi (mom) cannot have both 'diagnostic_name' and 'diagnostic_name_fixed' attributes @@ -830,45 +301,30 @@ def check_diagnostic_fixed(test_val, prop_dict, error): Traceback (most recent call last): CCPPError: '2foo' (hi) is not a valid fixed diagnostic name """ - valid = test_val if (prop_dict and ('diagnostic_name' in prop_dict) and prop_dict['diagnostic_name']): - valid = None if error: - emsg = "{} ({}) cannot have both 'diagnostic_name' and " - emsg += "'diagnostic_name_fixed' attributes" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - if 'standard_name' in prop_dict: - sname = prop_dict['standard_name'] - else: - sname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(lname, sname)) - # end if - elif check_fortran_id(test_val, prop_dict, False) is None: - valid = None + lname = prop_dict.get('local_name', 'UNKNOWN') + sname = prop_dict.get('standard_name', 'UNKNOWN') + raise CCPPError( + "{} ({}) cannot have both 'diagnostic_name' and " + "'diagnostic_name_fixed' attributes".format(lname, sname)) + return None + if check_fortran_id(test_val, prop_dict, False) is None: if error: - emsg = "'{}' ({}) is not a valid fixed diagnostic name" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(test_val, lname)) - # end if - # end if - return valid + lname = prop_dict.get('local_name', 'UNKNOWN') + raise CCPPError( + "'{}' ({}) is not a valid fixed diagnostic name".format( + test_val, lname)) + return None + return test_val -######################################################################## -_DIAG_PRE = r"("+FORTRAN_ID+")?" +_DIAG_PRE = r"(" + FORTRAN_ID + ")?" _DIAG_SUFF = r"([_0-9A-Za-z]+)?" -_DIAG_PROP = r"((\${process}|\${scheme_name})"+_DIAG_SUFF+r")" -_DIAG_RE = re.compile(_DIAG_PRE+_DIAG_PROP+r"?$") +_DIAG_PROP = r"((\${process}|\${scheme_name})" + _DIAG_SUFF + r")" +_DIAG_RE = re.compile(_DIAG_PRE + _DIAG_PROP + r"?$") + def check_diagnostic_id(test_val, prop_dict, error): """Return if a valid descriptor for a CCPP diagnostic, @@ -886,124 +342,61 @@ def check_diagnostic_id(test_val, prop_dict, error): 'diagnostic_name_fixed'. >>> check_diagnostic_id("foo", {'diagnostic_name' : 'foo'}, False) 'foo' - >>> check_diagnostic_id("foo", {'diagnostic_name' : 'foo'}, True) - 'foo' - >>> check_diagnostic_id("foo", {'diagnostic_name_fixed' : 'foo'}, False) - >>> check_diagnostic_id("foo_${process}", {}, False) 'foo_${process}' - >>> check_diagnostic_id("foo_${process}_2bad", {}, False) - 'foo_${process}_2bad' - >>> check_diagnostic_id("${process}_2bad", {}, False) - '${process}_2bad' - >>> check_diagnostic_id("foo_${scheme_name}", {}, False) - 'foo_${scheme_name}' >>> check_diagnostic_id("foo_${scheme_name}_2bad", {}, False) 'foo_${scheme_name}_2bad' - >>> check_diagnostic_id("${scheme_name}_suff", {}, False) - '${scheme_name}_suff' >>> check_diagnostic_id("pref_${scheme}_suff", {}, False) - >>> check_diagnostic_id("pref_${scheme_name_suff", {}, False) - - >>> check_diagnostic_id("pref_$scheme_name}_suff", {}, False) - - >>> check_diagnostic_id("pref_{scheme_name}_suff", {}, False) - - >>> check_diagnostic_id("foo", {'diagnostic_name_fixed':'','local_name':'hi','standard_name':'mom'}, True) - 'foo' >>> check_diagnostic_id("foo", {'diagnostic_name_fixed':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: hi (mom) cannot have both 'diagnostic_name' and 'diagnostic_name_fixed' attributes - >>> check_diagnostic_id("2foo", {'diagnostic_name':'foo','local_name':'hi','standard_name':'mom'}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '2foo' (hi) is not a valid diagnostic name """ if (prop_dict and ('diagnostic_name_fixed' in prop_dict) and prop_dict['diagnostic_name_fixed']): - valid = None if error: - emsg = "{} ({}) cannot have both 'diagnostic_name' and " - emsg += "'diagnostic_name_fixed' attributes" - if 'local_name' in prop_dict: - lname = prop_dict['local_name'] - else: - lname = 'UNKNOWN' - # end if - if 'standard_name' in prop_dict: - sname = prop_dict['standard_name'] - else: - sname = 'UNKNOWN' - # end if - raise CCPPError(emsg.format(lname, sname)) - # end if - else: - match = _DIAG_RE.match(test_val) - if match is None: - valid = None - if error: - emsg = "'{}' is not a valid diagnostic_name value" - raise CCPPError(emsg.format(test_val)) - # end if - else: - valid = test_val - # end if - # end if - return valid + lname = prop_dict.get('local_name', 'UNKNOWN') + sname = prop_dict.get('standard_name', 'UNKNOWN') + raise CCPPError( + "{} ({}) cannot have both 'diagnostic_name' and " + "'diagnostic_name_fixed' attributes".format(lname, sname)) + return None + if _DIAG_RE.match(test_val) is None: + if error: + raise CCPPError( + "'{}' is not a valid diagnostic_name value".format(test_val)) + return None + return test_val -######################################################################## def check_molar_mass(test_val, prop_dict, error): """Return if valid molar mass, otherwise, None if is True, raise an Exception if is not valid. >>> check_molar_mass('1', None, True) 1.0 - >>> check_molar_mass('1.0', None, True) - 1.0 >>> check_molar_mass('1.0', None, False) 1.0 >>> check_molar_mass('-1', None, False) - >>> check_molar_mass('-1.0', None, False) - - >>> check_molar_mass('string', None, False) - - >>> check_molar_mass(10001, None, False) - >>> check_molar_mass('-1', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: '-1' is not a valid molar mass - >>> check_molar_mass('-1.0', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '-1.0' is not a valid molar mass - >>> check_molar_mass('string', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: '-1.0' is not a valid molar mass >>> check_molar_mass(10001, None, True) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: '10001' is not a valid molar mass """ - # Check if input value is an int or float try: test_val = float(test_val) - if test_val < 0.0 or test_val > _MAX_MOLAR_MASS: - if error: - raise CCPPError(f"{test_val} is not a valid molar mass") - else: - test_val = None - # end if - # end if - except: - # not an int or float, conditionally throw error + except (TypeError, ValueError): if error: - raise CCPPError(f"{test_val} is invalid; not a float or int") - else: - test_val=None - # end if - # end try + raise CCPPError(f"{test_val} is invalid; not a float or int") + return None + if test_val < 0.0 or test_val > _MAX_MOLAR_MASS: + if error: + raise CCPPError(f"{test_val} is not a valid molar mass") + return None return test_val -######################################################################## def check_balanced_paren(string, start=0, error=False): """Return indices delineating a balance set of parentheses. @@ -1018,8 +411,6 @@ def check_balanced_paren(string, start=0, error=False): (-1, -1) >>> check_balanced_paren("(foo, bar)") (0, 9) - >>> check_balanced_paren("( (foo, bar) )", start=1) - (2, 11) >>> check_balanced_paren("(size(foo,1), qux)") (0, 17) >>> check_balanced_paren("(foo('bar()'))") @@ -1037,64 +428,30 @@ def check_balanced_paren(string, start=0, error=False): inchar = None str_len = len(string) while index < str_len: - if (string[index] == '"') or (string[index] == "'"): - if inchar == string[index]: + c = string[index] + if c in ('"', "'"): + if inchar == c: inchar = None elif inchar is None: - inchar = string[index] - # else in character context, keep going - # end if + inchar = c elif inchar is not None: - # In character context, keep going pass - elif string[index] == '(': + elif c == '(': if depth == 0: begin = index - # end if - depth = depth + 1 - if depth == 0: - break - # end if - elif string[index] == ')': - depth = depth - 1 + depth += 1 + elif c == ')': + depth -= 1 if depth == 0: end = index break - # end if - # else just keep going - # end if - index = index + 1 - # End while - if (begin >= 0) and (end < 0) and error: + index += 1 + if begin >= 0 and end < 0 and error: raise CCPPError("ERROR: Unbalanced parenthesis in '{}'".format(string)) - # end if return begin, end -######################################################################## - -def registered_fortran_ddt_names(): - return _REGISTERED_FORTRAN_DDT_NAMES - -######################################################################## def registered_fortran_ddt_name(name): if name in _REGISTERED_FORTRAN_DDT_NAMES: return name - else: - return None - -######################################################################## - -def register_fortran_ddt_name(name): - if name not in _REGISTERED_FORTRAN_DDT_NAMES: - _REGISTERED_FORTRAN_DDT_NAMES.append(name) - -######################################################################## - -if __name__ == "__main__": - # pylint: disable=ungrouped-imports - import doctest - # pylint: enable=ungrouped-imports - fail, _ = doctest.testmod() - sys.exit(fail) -# end if + return None diff --git a/capgen-ng/metadata/parse_tools/parse_log.py b/capgen-ng/metadata/parse_tools/parse_log.py index fec65161..fe080cc6 100644 --- a/capgen-ng/metadata/parse_tools/parse_log.py +++ b/capgen-ng/metadata/parse_tools/parse_log.py @@ -1,9 +1,4 @@ -#!/usr/bin/env python3 - -"""Shared logger utilities for parse processes. - -Copied from scripts/parse_tools/parse_log.py. -""" +"""Shared logger utilities for parse processes.""" import logging @@ -29,40 +24,19 @@ def init_log(name, level=None): def set_log_level(logger, level): - """Set *logger*'s level to *level*.""" logger.setLevel(level) -def remove_handlers(logger): - """Remove all handlers from *logger*.""" +def _remove_handlers(logger): for handler in list(logger.handlers): logger.removeHandler(handler) def set_log_to_stdout(logger): - """Direct *logger* output to standard output.""" - remove_handlers(logger) + _remove_handlers(logger) logger.addHandler(logging.StreamHandler()) def set_log_to_null(logger): - """Suppress all *logger* output.""" - remove_handlers(logger) + _remove_handlers(logger) logger.addHandler(logging.NullHandler()) - - -def set_log_to_file(logger, filename): - """Direct *logger* output to *filename*.""" - remove_handlers(logger) - logger.addHandler(logging.FileHandler(filename)) - - -def flush_log(logger): - """Flush all pending output from *logger*.""" - for handler in list(logger.handlers): - handler.flush() - - -def verbose(logger): - """Return True if *logger* is at DEBUG level.""" - return logger.isEnabledFor(logging.DEBUG) diff --git a/capgen-ng/metadata/parse_tools/parse_object.py b/capgen-ng/metadata/parse_tools/parse_object.py deleted file mode 100644 index 1095c210..00000000 --- a/capgen-ng/metadata/parse_tools/parse_object.py +++ /dev/null @@ -1,166 +0,0 @@ -#!/usr/bin/env python3 -"""A module for the base, ParseObject class""" - -# CCPP framework imports -from .parse_source import ParseContext, CCPPError, context_string - -######################################################################## - -class ParseObject(ParseContext): - """ParseObject is a simple class that keeps track of an object's - place in a file and safely produces lines from an array of lines - >>> ParseObject('foobar.F90', []) #doctest: +ELLIPSIS - - >>> ParseObject('foobar.F90', []).filename - 'foobar.F90' - >>> ParseObject('foobar.F90', ["##hi mom",], line_start=1).curr_line() - (None, 1) - >>> ParseObject('foobar.F90', ["first line","## hi mom"], line_start=1).curr_line() - ('## hi mom', 1) - >>> ParseObject('foobar.F90', ["##hi mom",], line_start=1).next_line() - (None, 1) - >>> ParseObject('foobar.F90', ["##first line","## hi mom"], line_start=1).next_line() - ('## hi mom', 1) - >>> ParseObject('foobar.F90', ["## hi \\\\","mom"], line_start=0).next_line() - ('## hi mom', 0) - >>> ParseObject('foobar.F90', ["line1","##line2","## hi mom"], line_start=2).next_line() - ('## hi mom', 2) - >>> ParseObject('foobar.F90', ["## hi \\\\","there \\\\","mom"], line_start=0).next_line() - ('## hi there mom', 0) - >>> ParseObject('foobar.F90', ["!! line1","!! hi mom"], line_start=1).next_line() - ('!! hi mom', 1) - """ - - _max_errors = 32 - - def __init__(self, filename, lines_in, line_start=0): - """Initialize this ParseObject""" - self.__lines = lines_in - self.__line_start = line_start - self.__line_end = line_start - self.__line_next = line_start - self.__num_lines = len(self.__lines) - self.__error_message = "" - self.__num_errors = 0 - super().__init__(linenum=line_start, filename=filename) - - @property - def first_line_num(self): - """Return the first line parsed""" - return self.__line_start - - @property - def last_line_num(self): - """Return the last line parsed""" - return self.__line_end - - def valid_line(self): - """Return True if the current line is valid""" - return (self.line_num >= 0) and (self.line_num < self.__num_lines) - - @property - def error_message(self): - """Return this object's error message""" - return self.__error_message - - def curr_line(self): - """Return the current line (if valid) and the current line number. - If the current line is invalid, return None""" - valid_line = self.valid_line() - _curr_line = None - _my_curr_lineno = self.line_num - if valid_line: - try: - _curr_line = self.__lines[self.line_num].rstrip() - self.__line_next = self.line_num + 1 - self.__line_end = self.__line_next - except CCPPError: - self.add_syntax_err("line", self.line_num) - valid_line = False - # end if - # We allow continuation self.__lines (ending with a single backslash) - if valid_line and _curr_line.endswith('\\'): - next_line, _ = self.next_line() - if next_line is None: - # We ran out of lines, just strip the backslash - _curr_line = _curr_line[0:len(_curr_line)-1] - else: - _curr_line = _curr_line[0:len(_curr_line)-1] + next_line - # end if - # end if - # curr_line should not change the line number - self.line_num = _my_curr_lineno - return _curr_line, self.line_num - - def next_line(self): - """Return the next line in our file (if valid) and the next line's - line number. If the next line is not valid, return None""" - self.line_num = self.__line_next - return self.curr_line() - - def peek_line(self, line_num): - """Return the text of without advancing to that line. - if is out of bounds, return None.""" - if (line_num >= 0) and (line_num < len(self.__lines)): - return self.__lines[line_num] - # end if - return None - - def add_syntax_err(self, token_type, token=None, skip_context=False): - """Add a ParseSyntaxError-type message to this object's error - log, separating it from any previous messages with a newline.""" - if self.__error_message: - if self.__num_errors == self._max_errors: - self.__error_message += '\nMaximum number of errors exceeded' - self.line_num = self.__num_lines # Intentionally walk off end - self.__line_next = self.line_num - elif self.__num_errors > self._max_errors: - # Oops, something went wrong, panic! - raise CCPPError(self.error_message) - # end if - self.__error_message += '\n' - # end if - if self.__num_errors < self._max_errors: - if skip_context: - cstr = "" - else: - cstr = context_string(self) - # end if - if token is None: - self.__error_message += "{}{}".format(token_type, cstr) - else: - self.__error_message += "Invalid {}, '{}'{}".format(token_type, - token, cstr) - # end if - # end if - self.__num_errors += 1 - - def reset_pos(self, line_start=0): - """Attempt to set the current file position to . - If is out of bounds, raise an exception.""" - if (line_start < 0) or (line_start >= self.__num_lines): - emsg = 'Attempt to reset_pos to non-existent line, {}' - raise CCPPError(emsg.format(line_start)) - # end if - self.line_num = line_start - self.__line_next = line_start - - def write_line(self, line_num, line): - """Overwrite line, with . - If is out of bounds, raise an exception.""" - if (line_num < 0) or (line_num >= len(self.__lines)): - emsg = 'Attempt to write non-existent line, {}' - raise CCPPError(emsg.format(line_num)) - # end if - self.__lines[line_num] = line - - def __del__(self): - """Attempt to cleanup memory used by this object""" - try: - del self.__lines - del self.regions - except Exception: - pass # Python does not guarantee much about __del__ conditions - # end try - -######################################################################## diff --git a/capgen-ng/metadata/parse_tools/parse_source.py b/capgen-ng/metadata/parse_tools/parse_source.py index 53e5fb9c..6648f59d 100644 --- a/capgen-ng/metadata/parse_tools/parse_source.py +++ b/capgen-ng/metadata/parse_tools/parse_source.py @@ -1,52 +1,7 @@ -#!/usr/bin/env python3 +"""Parsing primitives: parse context and exception types.""" -"""Parsing primitives: context tracking, exception types, and source tagging. - -Copied from scripts/parse_tools/parse_source.py and adapted for the -capgen-ng package structure. -""" - -from collections.abc import Iterable -import copy -import os.path import logging - - -class _StdNameCounter: - """Global counter for generating unique placeholder standard names.""" - - __SNAME_NUM = 0 - - @classmethod - def new_stdname_number(cls): - """Increment and return the counter.""" - _StdNameCounter.__SNAME_NUM += 1 - return _StdNameCounter.__SNAME_NUM - - @classmethod - def reset_stdname_counter(cls, reset_val=0): - """Reset the counter to *reset_val*.""" - _StdNameCounter.__SNAME_NUM = reset_val - - -def unique_standard_name(): - """Return a unique placeholder standard name. - - Used during parsing when a real standard name is not yet known. - - >>> n1 = unique_standard_name() - >>> n2 = unique_standard_name() - >>> n1 != n2 - True - >>> n1.startswith('enter_standard_name_') - True - """ - return 'enter_standard_name_{}'.format(_StdNameCounter.new_stdname_number()) - - -def reset_standard_name_counter(): - """Reset the unique_standard_name counter to zero.""" - _StdNameCounter.reset_stdname_counter() +import os.path def context_string(context=None, with_comma=True, nodir=False): @@ -61,10 +16,6 @@ def context_string(context=None, with_comma=True, nodir=False): nodir : bool Strip the directory portion of the filename. - Returns - ------- - str - >>> context_string() '' >>> context_string(context=ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False) @@ -73,8 +24,6 @@ def context_string(context=None, with_comma=True, nodir=False): ', at dir/source.F90:33' >>> context_string(context=ParseContext(filename="dir/source.F90"), with_comma=False) 'dir/source.F90' - >>> context_string(context=ParseContext(filename="dir/source.F90"), with_comma=True) - ', in dir/source.F90' >>> context_string(context=ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False, nodir=True) 'source.F90:33' """ @@ -91,17 +40,6 @@ def context_string(context=None, with_comma=True, nodir=False): return ('{comma}{where_str}' + spec).format(comma=comma, where_str=where_str, ctx=context) -def type_name(obj): - """Return the class name of *obj*. - - >>> type_name(42) - 'int' - >>> type_name("hello") - 'str' - """ - return type(obj).__name__ - - class CCPPError(ValueError): """User-facing error with a plain message and no traceback noise.""" @@ -132,86 +70,29 @@ def __init__(self, errmsg, context=None): super().__init__(message) -class ParseContextError(CCPPError): - """Error arising from mis-use of ParseContext.""" - - def __init__(self, errmsg, context): - logging.shutdown() - message = "{}{}".format(errmsg, context_string(context)) - super().__init__(message) - - -class ContextRegion(Iterable): - """LIFO stack of (region_type, region_name) pairs.""" - - def __init__(self): - self._lifo = [] - - def push(self, rtype, rname): - """Push a new region onto the stack.""" - self._lifo.append([rtype, rname]) - - def pop(self): - """Remove and return the top item.""" - return self._lifo.pop() - - def type_list(self): - """Return a list of just the region types.""" - return [x[0] for x in self._lifo] - - def __iter__(self): - for item in self._lifo: - yield item[0] - - def __len__(self): - return len(self._lifo) - - def __getitem__(self, index): - return self._lifo[index] - - class ParseContext: - """Tracks a parser's current position inside a source file. + """File-position record used as the location anchor for parse errors. - Parameters - ---------- - linenum : int, optional - Zero-based line number. Negative means «file level, no specific line». - filename : str, optional - Path to the source file. - context : ParseContext, optional - Copy position from an existing context (overrides *linenum*/*filename*). + Holds a filename and a zero-based line number (negative means «file + level, no specific line»); formats as ``filename:line``. - Examples - -------- - >>> ctx = ParseContext(linenum=0, filename="foo.F90") - >>> str(ctx) + >>> str(ParseContext(linenum=0, filename="foo.F90")) 'foo.F90:1' - >>> ctx.increment(4) - >>> str(ctx) - 'foo.F90:5' - >>> ParseContext(linenum="bad", filename="f.F90") + >>> str(ParseContext(filename="foo.F90")) + 'foo.F90' + >>> ParseContext(linenum="bad", filename="f.F90") #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... - metadata.parse_tools.parse_source.CCPPError: ParseContext linenum must be an int + CCPPError: ParseContext linenum must be an int """ - def __init__(self, linenum=None, filename=None, context=None): - if context is not None: - self.__regions = copy.deepcopy(context.regions) - else: - self.__regions = ContextRegion() - - if context is not None: - linenum = context.line_num - elif linenum is None: + def __init__(self, linenum=None, filename=None): + if linenum is None: linenum = -1 elif not isinstance(linenum, int): raise CCPPError('ParseContext linenum must be an int') - if context is not None: - filename = context.filename - elif filename is None: + if filename is None: filename = "" elif not isinstance(filename, str): raise CCPPError('ParseContext filename must be a string') @@ -219,129 +100,19 @@ def __init__(self, linenum=None, filename=None, context=None): self.__linenum = linenum self.__filename = filename - def default_module_name(self): - """Return the base name (without extension) of the source file.""" - return os.path.splitext(os.path.basename(self.__filename))[0] - @property def line_num(self): - """Current zero-based line number (negative = file level).""" return self.__linenum - @line_num.setter - def line_num(self, newnum): - self.__linenum = newnum - @property def filename(self): - """Path to the source file being parsed.""" return self.__filename - @property - def regions(self): - """The nested-region stack.""" - return self.__regions - def __format__(self, spec): - """Format the context as ``filename:line``. - - Supported format specs: ``'nodir'`` strips the directory component. - """ fname = os.path.basename(self.__filename) if spec == 'nodir' else self.__filename if self.__linenum >= 0: return "{}:{}".format(fname, self.__linenum + 1) return fname def __str__(self): - if self.__linenum >= 0: - return "{}:{}".format(self.__filename, self.__linenum + 1) - return self.__filename - - def increment(self, inc=1): - """Advance the line counter by *inc* (default 1).""" - if self.__linenum < 0: - self.__linenum = 0 - self.__linenum += inc - - def enter_region(self, region_type, region_name=None, nested_ok=True): - """Record entering a named region (module, DDT, subroutine, …). - - If *nested_ok* is False, raises :exc:`ParseContextError` when already - inside a region of the same type. - """ - if (region_type not in self.__regions.type_list()) or nested_ok: - self.__regions.push(region_type, region_name) - else: - raise ParseContextError( - "Cannot enter a nested {} region".format(region_type), self - ) - - def leave_region(self, region_type, region_name=None): - """Record leaving a region, with optional name verification.""" - if self.__regions: - curr_type, curr_name = self.__regions.pop() - if curr_type != region_type: - raise ParseContextError( - "Trying to exit {} region while currently in {} region".format( - region_type, curr_type - ), - self, - ) - if region_name is not None and curr_name is not None: - if region_name != curr_name: - raise ParseContextError( - "Trying to exit {} {} while currently in {} {}".format( - region_type, region_name, curr_type, curr_name - ), - self, - ) - elif region_name is not None and curr_name is None: - raise ParseContextError( - "Trying to exit {} {} while currently in unnamed {} region".format( - region_type, region_name, curr_type - ), - self, - ) - else: - raise ParseContextError("Cannot exit, not currently in any region", self) - - def curr_region(self): - """Return the innermost (type, name) pair, or None if not in any region.""" - return self.__regions[-1] if self.__regions else None - - def in_region(self, region_type, region_name=None): - """Return True iff currently inside *region_type* (optionally *region_name*).""" - return self.curr_region() == [region_type, region_name] - - -class ParseSource: - """Lightweight tag associating a name and type with a parse context. - - >>> src = ParseSource("my_func", "subroutine", ParseContext(0, "src.F90")) - >>> src.name - 'my_func' - >>> src.ptype - 'subroutine' - >>> str(src.context) - 'src.F90:1' - """ - - def __init__(self, name_in, type_in, context_in): - self.__name = name_in - self.__type = type_in - self.__context = context_in - - @property - def ptype(self): - """The type label (e.g. 'scheme', 'host', 'subroutine').""" - return self.__type - - @property - def name(self): - """The name of the parsed entity.""" - return self.__name - - @property - def context(self): - """The :class:`ParseContext` where this entity was found.""" - return self.__context + return format(self) diff --git a/capgen-ng/metadata/parse_tools/xml_tools.py b/capgen-ng/metadata/parse_tools/xml_tools.py index 35105141..076a47eb 100644 --- a/capgen-ng/metadata/parse_tools/xml_tools.py +++ b/capgen-ng/metadata/parse_tools/xml_tools.py @@ -1,10 +1,9 @@ #!/usr/bin/env python3 -""" -Parse a host-model registry XML file and return the captured variables. +"""XML helpers: schema-version probing, schema validation, nested-suite +expansion, and pretty-printed XML writing. """ -# Python library imports import os import re import shutil @@ -12,33 +11,27 @@ import sys import xml.etree.ElementTree as ET import xml.dom.minidom -# CCPP framework imports + from .parse_source import CCPPError from .parse_log import init_log, set_log_to_null -# Global data _INDENT_STR = " " beg_tag_re = re.compile(r"([<][^/][^<>]*[^/][>])") end_tag_re = re.compile(r"([<][/][^<>/]+[>])") simple_tag_re = re.compile(r"([<][^/][^<>/]+[/][>])") -# Find python version PYSUBVER = sys.version_info[1] _LOGGER = None -############################################################################### + class XMLToolsInternalError(ValueError): -############################################################################### - """Error class for reporting internal errors""" - def __init__(self, message): - """Initialize this exception""" - super().__init__(message) + """Internal error raised by helpers in this module.""" + -############################################################################### def find_schema_version(root): -############################################################################### - """ - Find the version of the host registry file represented by root + """Return the schema version as ``[major, minor]`` from the *root*'s + ``version`` attribute. + >>> find_schema_version(ET.fromstring('')) [1, 0] >>> find_schema_version(ET.fromstring('')) @@ -51,316 +44,263 @@ def find_schema_version(root): Traceback (most recent call last): CCPPError: Illegal version string, '0.0' Major version must be at least 1 - >>> find_schema_version(ET.fromstring('')) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Illegal version string, '0.0' - Minor version must be at least 0 """ - verbits = None if 'version' not in root.attrib: raise CCPPError("Version attribute required") - # end if version = root.attrib['version'] versplit = version.split('.') try: if len(versplit) != 2: raise CCPPError('Major and minor version required') - # end if (no else needed) try: verbits = [int(x) for x in versplit] except ValueError as verr: raise CCPPError(verr) from verr - # end try if verbits[0] < 1: raise CCPPError('Major version must be at least 1') - # end if if verbits[1] < 0: raise CCPPError('Minor version must be non-negative') - # end if except CCPPError as verr: errstr = """Illegal version string, '{}' Format must be .""" ve_str = str(verr) if ve_str: errstr = ve_str + '\n' + errstr - # end if raise CCPPError(errstr.format(version)) from verr - # end try return verbits -############################################################################### -def find_schema_file(schema_root, version, schema_path=None): -############################################################################### - """Find and return the schema file based on and - or return None. - If is present, use that as the directory to find the - appropriate schema file. Otherwise, just look in the current directory.""" +def find_schema_file(schema_root, version, schema_path=None): + """Return ``_v_.xsd`` under *schema_path* + (or the current directory), or ``None`` if no such file exists.""" verstring = '_'.join([str(x) for x in version]) schema_filename = "{}_v{}.xsd".format(schema_root, verstring) if schema_path: schema_file = os.path.join(schema_path, schema_filename) else: schema_file = schema_filename - # end if if os.path.exists(schema_file): return schema_file - # end if return None -############################################################################### + def validate_xml_file(filename, schema_root, version, logger, schema_path=None): -############################################################################### - """ - Find the appropriate schema and validate the XML file, , - against it using xmllint - """ - # Check the filename + """Validate *filename* against the matching schema using xmllint.""" if not os.path.isfile(filename): raise CCPPError("validate_xml_file: Filename, '{}', does not exist".format(filename)) - # end if if not os.access(filename, os.R_OK): raise CCPPError("validate_xml_file: Cannot open '{}'".format(filename)) - # end if if os.path.isfile(schema_root): - # We already have a file, just use it schema_file = schema_root else: if not schema_path: - # Find the schema, based on the model version thispath = os.path.abspath(__file__) pdir = os.path.dirname(os.path.dirname(os.path.dirname(thispath))) schema_path = os.path.join(pdir, 'schema') - # end if schema_file = find_schema_file(schema_root, version, schema_path) if not (schema_file and os.path.isfile(schema_file)): verstring = '.'.join([str(x) for x in version]) - emsg = f"""validate_xml_file: Cannot find schema for version {verstring}, - {schema_file} does not exist""" - raise CCPPError(emsg) - # end if - # end if + raise CCPPError( + f"validate_xml_file: Cannot find schema for version {verstring},\n" + f" {schema_file} does not exist" + ) if not os.access(schema_file, os.R_OK): - emsg = "validate_xml_file: Cannot open schema, '{}'" - raise CCPPError(emsg.format(schema_file)) - # end if - - # Find xmllint - xmllint = shutil.which('xmllint') # Blank if not installed + raise CCPPError( + "validate_xml_file: Cannot open schema, '{}'".format(schema_file)) + + xmllint = shutil.which('xmllint') if not xmllint: - msg = "xmllint not found, could not validate file {}" - raise CCPPError("validate_xml_file: " + msg.format(filename)) - # end if - - # Validate XML file against schema - logger.debug("Checking file {} against schema {}".format(filename, - schema_file)) - cmd = [xmllint, '--noout', '--schema', schema_file, filename] + raise CCPPError( + "validate_xml_file: xmllint not found, could not validate file {}".format(filename)) + + logger.debug("Checking file {} against schema {}".format(filename, schema_file)) + cmd = [xmllint, '--noout', '--schema', schema_file, filename] cproc = subprocess.run(cmd, check=False, capture_output=True) if cproc.returncode == 0: - # We got a pass return code but some versions of xmllint do not - # correctly return an error code on non-validation so double check - # the result + # Some xmllint builds return 0 even when validation fails; double + # check by looking for the literal 'validates' marker in output. result = b'validates' in cproc.stdout or b'validates' in cproc.stderr else: result = False - # end if if result: logger.debug(cproc.stdout) logger.debug(cproc.stderr) return result - else: - cmd = ' '.join(cmd) - outstr = f"Execution of '{cmd}' failed with code: {cproc.returncode}\n" - if cproc.stdout: - outstr += f"{cproc.stdout.decode('utf-8', errors='replace').strip()}\n" - if cproc.stderr: - outstr += f"{cproc.stderr.decode('utf-8', errors='replace').strip()}\n" - raise CCPPError(outstr) - -############################################################################### -def read_xml_file(filename, logger=None): -############################################################################### - """Read the XML file, , and return its tree and root + cmd_str = ' '.join(cmd) + outstr = f"Execution of '{cmd_str}' failed with code: {cproc.returncode}\n" + if cproc.stdout: + outstr += f"{cproc.stdout.decode('utf-8', errors='replace').strip()}\n" + if cproc.stderr: + outstr += f"{cproc.stderr.decode('utf-8', errors='replace').strip()}\n" + raise CCPPError(outstr) - Parameters: - filename (str): The path to an XML file to read and search. - logger (logging.Logger, optional): Logger for warnings/errors. - Returns: - tree (xml.etree.ElementTree): The element tree from the input file. - root (xml.etree.ElementTree.Element): The root element of tree. +def read_xml_file(filename, logger=None): + """Read *filename* and return ``(tree, root)``. - Raises: - CCPPError: If the file cannot be found or read. + Raises ``CCPPError`` if the file is missing or unreadable. """ if os.path.isfile(filename) and os.access(filename, os.R_OK): - file_open = (lambda x: open(x, 'r', encoding='utf-8')) - with file_open(filename) as file_: + with open(filename, 'r', encoding='utf-8') as fh: try: - tree = ET.parse(file_) + tree = ET.parse(fh) root = tree.getroot() except ET.ParseError as perr: - emsg = "read_xml_file: Cannot read {}, {}" - raise CCPPError(emsg.format(filename, perr)) from perr + raise CCPPError( + "read_xml_file: Cannot read {}, {}".format(filename, perr) + ) from perr elif not os.access(filename, os.R_OK): raise CCPPError("read_xml_file: Cannot open '{}'".format(filename)) else: - emsg = "read_xml_file: Filename, '{}', does not exist" - raise CCPPError(emsg.format(filename)) - # end if + raise CCPPError( + "read_xml_file: Filename, '{}', does not exist".format(filename)) if logger: logger.debug(f"Reading XML file {filename}") - # end if return tree, root -############################################################################### + def load_suite_by_name(suite_name, group_name, file, logger=None): -############################################################################### - """ - Load a suite by its name, or a group of a suite by the suite and group names. - - Parameters: - suite_name (str): The name of the suite to find. - group_name (str or None): The name of the group to find within the suite. - file (str): The path to an XML file to read and search. - logger (logging.Logger, optional): Logger for warnings/errors. - - Returns: - xml.etree.ElementTree.Element: The matching suite or group element. - - Raises: - CCPPError: If the suite or group is not found, or if the schema is invalid. - - Examples: - >>> import tempfile - >>> import xml.etree.ElementTree as ET - >>> logger = init_log('xml_tools') - >>> set_log_to_null(logger) - >>> # Create temporary files for the nested suites - >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> # Write XML contents to temporary file - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... - ... - ... ''') - >>> load_suite_by_name("physics_suite", None, file1_path, logger).tag - 'suite' - >>> load_suite_by_name("physics_suite", "dynamics", file1_path, logger).attrib['name'] - 'dynamics' - >>> load_suite_by_name("physics_suite", "missing_group", file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Nested suite physics_suite, group missing_group, not found - >>> load_suite_by_name("missing_suite", None, file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Nested suite missing_suite not found - >>> tmpdir.cleanup() + """Load and return a suite or group element from a SDF file. + + Parameters + ---------- + suite_name : str + Name of the suite element to find. + group_name : str or None + Name of the group within the suite to find; ``None`` returns the + whole suite. + file : str + Path to the XML file. + logger : logging.Logger, optional + + Returns + ------- + xml.etree.ElementTree.Element + The matching suite or group element. + + Examples + -------- + >>> import tempfile + >>> import xml.etree.ElementTree as ET + >>> logger = init_log('xml_tools') + >>> set_log_to_null(logger) + >>> tmpdir = tempfile.TemporaryDirectory() + >>> file1_path = os.path.join(tmpdir.name, "file1.xml") + >>> with open(file1_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... + ... + ... ''') + >>> load_suite_by_name("physics_suite", None, file1_path, logger).tag + 'suite' + >>> load_suite_by_name("physics_suite", "dynamics", file1_path, logger).attrib['name'] + 'dynamics' + >>> load_suite_by_name("physics_suite", "missing_group", file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + CCPPError: Nested suite physics_suite, group missing_group, not found + >>> load_suite_by_name("missing_suite", None, file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + CCPPError: Nested suite missing_suite not found + >>> tmpdir.cleanup() """ _, root = read_xml_file(file, logger) schema_version = find_schema_version(root) - res = validate_xml_file(file, 'suite', schema_version, logger) - if not res: + if not validate_xml_file(file, 'suite', schema_version, logger): raise CCPPError(f"Invalid suite definition file, '{file}'") - suite = root - if suite.attrib.get("name") == suite_name: + if root.attrib.get("name") == suite_name: if group_name: - for group in suite.findall("group"): + for group in root.findall("group"): if group.attrib.get("name") == group_name: return group else: - return suite + return root emsg = f"Nested suite {suite_name}" \ + (f", group {group_name}," if group_name else "") \ + " not found" + (f" in file {file}" if file else "") raise CCPPError(emsg) -############################################################################### + def replace_nested_suite(element, nested_suite, default_path, logger): -############################################################################### - """ - Replace a tag with the actual suite or group it references. - - This function looks up a referenced suite or suite group from an external - file, deep copies its children, and replaces the element - in the parent `element` with the copied contents. - - Parameters: - element (xml.etree.ElementTree.Element): The parent element containing the nested suite. - nested_suite (xml.etree.ElementTree.Element): The element to be replaced. - default_path (str): The default path to look for nested SDFs if file is not a absolute path. - logger (logging.Logger or None): Logger to record debug information. - - Returns: - str: The name of the suite that was replaced - - Example: - >>> import tempfile - >>> import xml.etree.ElementTree as ET - >>> from types import SimpleNamespace - >>> logger = init_log('xml_tools') - >>> set_log_to_null(logger) - >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... my_scheme - ... - ... - ... ''') - >>> # Import nested suite at suite level - >>> xml = f''' - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> nested = top_suite.find("nested_suite") - >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> # Import group from nested suite at group level - >>> xml = f''' - ... - ... - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> top_group = top_suite.find("group") - >>> nested = top_group.find("nested_suite") - >>> replace_nested_suite(top_group, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> # Import group from nested suite at suite level - >>> xml = f''' - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> nested = top_suite.find("nested_suite") - >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> tmpdir.cleanup() + """Replace a ```` element with the suite/group it references. + + Parameters + ---------- + element : Element + Parent of *nested_suite*. + nested_suite : Element + The ```` element to replace. + default_path : str + Directory to resolve a relative ``file=`` attribute against. + logger : logging.Logger or None + + Returns + ------- + str + Name of the suite that was substituted in. + + Examples + -------- + >>> import tempfile + >>> import xml.etree.ElementTree as ET + >>> logger = init_log('xml_tools') + >>> set_log_to_null(logger) + >>> tmpdir = tempfile.TemporaryDirectory() + >>> file1_path = os.path.join(tmpdir.name, "file1.xml") + >>> with open(file1_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... my_scheme + ... + ... + ... ''') + >>> xml = f''' + ... + ... + ... + ... ''' + >>> top_suite = ET.fromstring(xml) + >>> nested = top_suite.find("nested_suite") + >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) + 'my_suite' + >>> [child.tag for child in top_suite] + ['group'] + >>> top_suite.find("group").find("scheme").text + 'my_scheme' + >>> xml = f''' + ... + ... + ... + ... + ... + ... ''' + >>> top_suite = ET.fromstring(xml) + >>> top_group = top_suite.find("group") + >>> nested = top_group.find("nested_suite") + >>> replace_nested_suite(top_group, nested, tmpdir.name, logger) + 'my_suite' + >>> [child.tag for child in top_suite] + ['group'] + >>> top_suite.find("group").find("scheme").text + 'my_scheme' + >>> xml = f''' + ... + ... + ... + ... ''' + >>> top_suite = ET.fromstring(xml) + >>> nested = top_suite.find("nested_suite") + >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) + 'my_suite' + >>> [child.tag for child in top_suite] + ['group'] + >>> top_suite.find("group").find("scheme").text + 'my_scheme' + >>> tmpdir.cleanup() """ suite_name = nested_suite.attrib.get("name") group_name = nested_suite.attrib.get("group") @@ -371,13 +311,9 @@ def replace_nested_suite(element, nested_suite, default_path, logger): logger=logger) imported_content = [ET.fromstring(ET.tostring(child)) for child in referenced_suite] - # Swap nested suite with imported content for item in imported_content: - # If we are inserting a nested suite at the suite level (element.tag is suite), - # but we only want one group (group_name is not none), then we need to wrap - # the item in a group element. If on the other hand we insert an entire suite - # (all groups) at the suite level, or a specific group at the group level, - # then we can insert the item as is. + # When importing a single group at the suite level, wrap the item + # in a fresh so the parent's level isn't changed. if element.tag == 'suite' and group_name: item_to_insert = ET.Element("group", attrib={"name": group_name}) item_to_insert.append(item) @@ -390,189 +326,142 @@ def replace_nested_suite(element, nested_suite, default_path, logger): + (f", group '{group_name}'," if group_name else "") \ + (f" in file '{file}'" if file else "") logger.debug(msg.rstrip(',')) - # Return the name of the suite that we just replaced return suite_name -############################################################################### + def expand_nested_suites(suite, default_path, logger=None): -############################################################################### - """ - Recursively expand all elements within the XML element. - - This function finds elements within or elements, - and replaces them with the corresponding content from another suite. - - This operation is recursive and will continue expanding until no - elements remain. - - Parameters: - suite (xml.etree.ElementTree.Element): The root element. - logger (logging.Logger, optional): Logger for debug messages. - - Returns: - None. The XML tree is modified in place. - - Example: - >>> import tempfile - >>> import xml.etree.ElementTree as ET - >>> logger = init_log('xml_tools') - >>> set_log_to_null(logger) - >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> file2_path = os.path.join(tmpdir.name, "file2.xml") - >>> file3_path = os.path.join(tmpdir.name, "file3.xml") - >>> file4_path = os.path.join(tmpdir.name, "file4.xml") - >>> file5_path = os.path.join(tmpdir.name, "file5.xml") - >>> # Write mock XML contents for the nested suites - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... cloud_scheme - ... - ... - ... ''') - >>> with open(file2_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... pbl_scheme - ... - ... - ... ''') - >>> with open(file3_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... rrtmg_lw_scheme - ... - ... - ... rrtmg_sw_scheme - ... - ... - ... ''') - >>> with open(file4_path, "w") as f: - ... _ = f.write(f''' - ... - ... - ... - ... ''') - >>> with open(file5_path, "w") as f: - ... _ = f.write(f''' - ... - ... - ... - ... ''') - >>> # Parent suite - >>> xml_content = f''' - ... - ... - ... - ... - ... - ... - ... - ... ''' - >>> suite = ET.fromstring(xml_content) - >>> expand_nested_suites(suite, tmpdir.name, logger) - >>> ET.dump(suite) - - - cloud_scheme - - pbl_scheme - - rrtmg_lw_scheme - - rrtmg_sw_scheme - - >>> # Test infite recursion - >>> xml_content = f''' - ... - ... - ... - ... - ... - ... - ... ''' - >>> suite = ET.fromstring(xml_content) - >>> expand_nested_suites(suite, tmpdir.name, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Exceeded number of iterations while expanding nested suites - >>> tmpdir.cleanup() + """Recursively expand every ```` element inside *suite*. + + Iterative bound caps the recursion at ``max_iterations`` passes to + keep mutually-referential SDFs from looping forever. + + Examples + -------- + >>> import tempfile + >>> import xml.etree.ElementTree as ET + >>> logger = init_log('xml_tools') + >>> set_log_to_null(logger) + >>> tmpdir = tempfile.TemporaryDirectory() + >>> file1_path = os.path.join(tmpdir.name, "file1.xml") + >>> file2_path = os.path.join(tmpdir.name, "file2.xml") + >>> file3_path = os.path.join(tmpdir.name, "file3.xml") + >>> file4_path = os.path.join(tmpdir.name, "file4.xml") + >>> file5_path = os.path.join(tmpdir.name, "file5.xml") + >>> with open(file1_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... cloud_scheme + ... + ... + ... ''') + >>> with open(file2_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... pbl_scheme + ... + ... + ... ''') + >>> with open(file3_path, "w") as f: + ... _ = f.write(''' + ... + ... + ... rrtmg_lw_scheme + ... + ... + ... rrtmg_sw_scheme + ... + ... + ... ''') + >>> with open(file4_path, "w") as f: + ... _ = f.write(f''' + ... + ... + ... + ... ''') + >>> with open(file5_path, "w") as f: + ... _ = f.write(f''' + ... + ... + ... + ... ''') + >>> xml_content = f''' + ... + ... + ... + ... + ... + ... + ... + ... ''' + >>> suite = ET.fromstring(xml_content) + >>> expand_nested_suites(suite, tmpdir.name, logger) + >>> ET.dump(suite) + + + cloud_scheme + + pbl_scheme + + rrtmg_lw_scheme + + rrtmg_sw_scheme + + >>> xml_content = f''' + ... + ... + ... + ... + ... + ... + ... ''' + >>> suite = ET.fromstring(xml_content) + >>> expand_nested_suites(suite, tmpdir.name, logger) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + CCPPError: Exceeded number of iterations while expanding nested suites + >>> tmpdir.cleanup() """ - # To avoid infinite recursion, we simply count the number - # of iterations and stop at a certain limit. If someone is - # smart enough to come up with nested suite constructs that - # require more iterations, than he/she should be able to - # track down this variable and adjust it! max_iterations = 10 - # Collect the names of the expanded suites suite_names = [] - # Iteratively expand nested suites until they are all gone - keep_expanding = True - for num_iterations in range(max_iterations): + for _ in range(max_iterations): keep_expanding = False - # First, search all groups for nested_suite elements - groups = suite.findall("group") - for group in groups: - nested_suites = group.findall("nested_suite") - for nested in nested_suites: - suite_names.append(replace_nested_suite(group, nested, default_path, logger)) - # Trigger another pass over the root element + for group in suite.findall("group"): + for nested in group.findall("nested_suite"): + suite_names.append( + replace_nested_suite(group, nested, default_path, logger)) keep_expanding = True - # Second, search all suites for nested_suite elements - nested_suites = suite.findall("nested_suite") - for nested in nested_suites: - suite_names.append(replace_nested_suite(suite, nested, default_path, logger)) - # Trigger another pass over the root element + for nested in suite.findall("nested_suite"): + suite_names.append( + replace_nested_suite(suite, nested, default_path, logger)) keep_expanding = True if not keep_expanding: return - raise CCPPError("Exceeded number of iterations while expanding nested suites. " + \ - "Check for infinite recursion or adjust limit max_iterations. " + \ - f"Suites expanded so far: {suite_names}") - -############################################################################### + raise CCPPError( + "Exceeded number of iterations while expanding nested suites. " + "Check for infinite recursion or adjust limit max_iterations. " + f"Suites expanded so far: {suite_names}") + + def write_xml_file(root, file_path, logger=None): -############################################################################### - """Pretty-prints element root to an ASCII file using xml.dom.minidom. - - Routes the serialised XML through :func:`io_helpers.write_if_changed` - so an unchanged regeneration leaves the on-disk mtime alone. When - *logger* is supplied, the helper logs ``Wrote `` or - ``Unchanged: `` so the user can see at a glance whether the - file actually changed. + """Pretty-print *root* to *file_path*, routed through write_if_changed. + + Unchanged regenerations preserve the on-disk mtime so downstream + build tools don't rebuild. """ def remove_whitespace_nodes(node): - """Helper function to recursively remove all text nodes that contain - only whitespace, which eliminates blank lines in the output.""" for child in list(node.childNodes): if child.nodeType == child.TEXT_NODE and not child.data.strip(): node.removeChild(child) elif child.hasChildNodes(): remove_whitespace_nodes(child) - # Convert ElementTree to a byte string byte_string = ET.tostring(root, 'us-ascii') - - # Parse string using minidom for pretty printing reparsed = xml.dom.minidom.parseString(byte_string) - - # Clean whitespace-only text nodes remove_whitespace_nodes(reparsed) - - # Generate pretty-printed XML string pretty_xml = reparsed.toprettyxml(indent=" ") - # Route through write_if_changed so identical regenerated XML doesn't - # touch the mtime (downstream build tools rely on this to skip - # recompilation). Passing *logger* lets the helper emit the - # "Wrote / Unchanged" line directly, replacing the old debug-only - # write notice. from .io_helpers import write_if_changed write_if_changed(file_path, pretty_xml, logger=logger) - -############################################################################## From 4d39dade54e2870870802fe14ee22697f9c71d21 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 May 2026 07:35:59 -0600 Subject: [PATCH 28/74] Manual update of doc/briefing_pm.md --- doc/briefing_pm.md | 56 +++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md index 030e2a68..d284e7ae 100644 --- a/doc/briefing_pm.md +++ b/doc/briefing_pm.md @@ -63,13 +63,13 @@ can extend it safely (if at all - primary developer gone). **`capgen-ng`** (new, 2026-05). Procedural Python (a few thousand lines; flat data classes); reads the same metadata format; passes arguments like prebuild; supports the features capgen pioneered -(constituents, suite-owned variables, introspection); adds +(constituents, suite-owned variables, introspection); supports multi-instance, an integer state machine, six explicit scheme phases, vertical-flip / unit / kind transforms, registered scalar-index dimensions for threading and ensembles, write-if-changed build integration, and a separate Fortran-vs-metadata validator tool. Designed so the same generator works for prebuild-style hosts -(UFS / NEPTUNE / SCM) and capgen-style hosts (CAM-SIMA, future). +(UFS / NEPTUNE / SCM) and capgen-style hosts (CAM-SIMA). --- @@ -97,7 +97,7 @@ Three pressures converged in 2025/26: reading several modules together. Realistically, only one or two people on the framework team can change capgen without breaking something downstream. One of them now lives overseas and rejects - simplification attempts from the others. This is a + simplification attempts from the others. This is an inacceptable **bus-factor risk** that the redesign retires. --- @@ -117,14 +117,15 @@ CAM-SIMA's group caps pass **every individual variable as a separate argument** to the scheme dispatch routine. At CAM-SIMA's roughly two-hundred-variable scale this works. At UFS scale (~1200 variables per group), the generated Fortran exceeds compiler limits -under strict error-checking flags (`-check all`, `-fcheck=all`), -fails to compile, and even when it does compile produces -unmaintainably large source files. **This is the technical reason -capgen cannot drive UFS today**, independent of any other concern. +prevents the use of strict error-checking flags (`-check all`, +`-fcheck=all`) required for operational implementation, and even +when it does compile produces unmaintainably large source files. +**This is the technical reason capgen cannot drive UFS today**, +independent of any other concern. `capgen-ng` reverts to prebuild's DDT-argument convention. Host authors pass their physics DDTs by reference (one or a few arguments -per scheme call); component access happens **inside the scheme**. +per scheme call); component access happens **at the scheme call level**. This works at every scale we've measured. ### 3.2 Single-instance constituents are baked into the generated code @@ -179,18 +180,20 @@ agenda for one of the next framework-team meetings. ### 3.5 Synthetic variable-resolution scopes are hard to extend -capgen introduces a synthetic dictionary (`ConstituentVarDict`) -between the suite and host scopes during variable matching. The -mechanism works for capgen's use cases but is a code path most -contributors don't read. Extending the resolver to handle -multi-instance dimensions, scalar-index substitution, or -constituent host-wins semantics required undoing parts of the -synthetic scope. +capgen introduces a five-layer deep synthetic dictionary +(`ConstituentVarDict`) between the suite and host scopes during +variable matching. The mechanism works for capgen's use cases +but is a code path most contributors don't read. Extending the +resolver to handle multi-instance dimensions, scalar-index +substitution, or constituent host-wins semantics required +undoing parts of the synthetic scope. `capgen-ng`'s resolver is flat: each scheme arg is classified into exactly one source (control / host / suite / constituent), recorded on a small data class (`ResolvedArg`), and used directly by the -emitter. No synthetic dictionary. +emitter. No synthetic dictionary. **This design inherits from +`prebuild` and is the primary reason `capgen-ng` is comparable +in performance to `prebuild`. ### 3.6 Code volume and team coverage @@ -205,7 +208,9 @@ context". capgen-ng comes with over a thousand docstring tests and unit tests, as well as a comprehensive end-to-end test suite that covers all of prebuild's and capgen's existing end to end tests. capgen-ng adds additional end-to-end tests for new -features such as the multi-instance constituents test. +features such as the multi-instance constituents test. Including +these tests and the rich inline comments makes capgen-ng comparable +in size to capgen. --- @@ -239,7 +244,8 @@ Features that exist only in capgen-ng (some exist in prebuild): | **Registered scalar-index dimensions** | When metadata says a variable is dimensioned by `number_of_threads` or `number_of_instances`, capgen-ng injects the right per-call subscript automatically; the host's OpenMP-thread-private DDT layout works unchanged | | **Subcycle loop-counter automation** | Schemes inside a `` element can access `ccpp_loop_counter` / `ccpp_loop_extent` directly; the generator emits the Fortran `do` loop and binds the locals | | **`--legacy-mode` migration shim** | One CLI flag enables silent rewrite of two known-good deprecated standard names (`horizontal_loop_extent` → `horizontal_dimension`, `number_of_openmp_threads` → `number_of_threads`) with a loud warning — buys time for host metadata to migrate | -| **`--no-host-introspection` flag** | The five runtime introspection routines (`ccpp_physics_suite_list`, etc.) emit large `select case` blocks at SCM scale; this flag stubs the bodies, dropping the generated static API from ~33,000 lines to ~800 for the SCM build (the introspection routines were making `-O3` compilation effectively hang) | +| **`--no-host-introspection` flag** | The five runtime introspection routines (`ccpp_physics_suite_list`, etc.) emit large `select case` blocks at SCM scale; this flag stubs the bodies, dropping the generated static API from ~33,000 lines to ~800 for the SCM build (the introspection routines were making `-O1` compilation effectively hang) | +| **Consistent handling of external types** (MPI f08 communicator, ESMF clock) | Tabled in capgen because of the complexity of the solution | --- @@ -260,10 +266,14 @@ Features that exist only in capgen-ng (some exist in prebuild): per-instance. No coordination with CAM-SIMA / UFS / NEPTUNE required (host-facing API unchanged). - **NEPTUNE**: cleanup and acceptance testing in progress this week. + Regular/lower atmosphere physics builds and runs, and produces + results within the tolerance (i.e. similar to compiler changes). + High altitude atmosphere testing is next. - **UFS Weather Model**: not yet attempted; SCM is the proving - ground first. + ground first. Expecting updates due to the "fast physics" + called directly from the FV3 dynamical core as separate group. - **CAM-SIMA**: not yet re-connected; the constituent overhaul - decision (see §7) and the availability of CAM-SIMA developer + decision (see §7) and the availability of CAM-SIMA developers are the gating items. --- @@ -284,8 +294,8 @@ the table: - **Proposal B** (recommended for the next 4–6 weeks): relax the identity-equality check, formally classify properties as "scheme-intrinsic" (immutable) vs "host-configuration" (mutable - after registration). Physics schemes become genuinely portable - across hosts. + after registration). Physics schemes using constituents become + genuinely portable across hosts. - **Proposal C** (tabled): drop scheme-side constituent registration entirely; only the host registers. Cleaner but requires coordinated PRs across the framework, both generators, @@ -331,7 +341,7 @@ Three points worth raising explicitly: into capgen-ng. 3. **The team owning capgen-ng can be larger than the team owning capgen.** This is the most important practical point for - long-term programme health. A framework that two organisations + long-term programme health. A framework that three organisations can maintain is more resilient than a framework that one organisation (or one individual in that organisation) can maintain. From f4c0ef17f14994c2307aaf77595abdc1ffdb4271 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 May 2026 07:55:21 -0600 Subject: [PATCH 29/74] Update spelling in docs/* --- doc/briefing.md | 38 +++++++++++++++++++++++------------- doc/briefing_pm.md | 10 +++++----- doc/constituents_overhaul.md | 2 +- doc/migration.md | 10 +++++----- doc/redesign_prompt.md | 2 +- 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/doc/briefing.md b/doc/briefing.md index 5a05e070..69f60336 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -16,9 +16,11 @@ The CCPP Framework runs two code generators today: variables. - **`ccpp-capgen`** — complex, deeply object-oriented Python; flat-field argument passing; in use by NCAR CAM-SIMA. Many advanced features - designed but never implemented; flat-field passing infeasible at - UFS/NEPTUNE scale (1200+ variables, breaks under `-check all`); - nobody on the team fully understands it. + designed but never implemented; at UFS/NEPTUNE scale (1200+ variables) + flat-field passing prevents the use of strict error-checking flags + (`-check all`, `-fcheck=all`) required for operational implementation, + and even when it does compile it produces unmaintainably large source + files; nobody on the team fully understands it. **`capgen-ng`** starts fresh, drawing lessons from both. Guiding principle: **simplicity of prebuild, feature set of capgen**. @@ -118,7 +120,7 @@ Both share the same metadata-parsing library (`metadata/`). |-----------------------------|-----------------------------------|---------------------------------------------------| | Host metadata mechanism | Hard-coded Python dict (`TYPEDEFS_NEW_METADATA`) | Regular `type = ddt` + `type = host` tables | | Framework-owned variables | Not supported | First-class (suite-owned interstitial via Case 2) | -| Constituents | Hand-rolled, host-specific glue | Standardised opt-in mechanism with auto-provision | +| Constituents | Hand-rolled, host-specific glue | Standardized opt-in mechanism with auto-provision | | `register` phase | Doesn't exist | First phase; schemes declare dynamic constituents | | Multi-instance API | Implicit, ad-hoc | Paired-opt-in (`instance_number` / `number_of_instances`) | | Subcycle loop counter | Host plumbs it manually | Registered std names `ccpp_loop_counter` / `ccpp_loop_extent` resolve to the do-loop locals automatically inside `` | @@ -131,7 +133,8 @@ Both share the same metadata-parsing library (`metadata/`). | Topic | capgen | capgen-ng | |-----------------------------|-----------------------------------|---------------------------------------------------| | Group-cap arguments | Flat fields (1200+ at UFS scale) | DDT arguments (as in prebuild) | -| Variable matching algorithm | Scope-chain promotion | Flat host+control dict + suite-owned discovery | +| Variable matching algorithm | Five-layer scope-chain promotion | Flat host+control dict + suite-owned discovery (inherited from prebuild — primary reason runtime is comparable to prebuild) | +| External types (MPI f08 comm, ESMF clock) | Tabled (solution complexity) | First-class via `type = external::` | | `type = module` in metadata | Yes | Renamed `type = host` | | `is_constituent` scheme args | Auto-cloned by generator | Schemes register constituents explicitly in the `register` phase via `ccpp_constituent_properties_t(:)` | | `ConstituentVarDict` | Synthetic scope between suite + host | Removed; constituents are one of four sources (`control`/`host`/`suite`/`constituent`) on `ResolvedArg` | @@ -249,7 +252,7 @@ control-variable arguments to the public entry points. we error at parse time with a remediation pointing at `character(len=:)` deferred-length). - **Multiple registration sources for the same constituent** with - silent dedup. Today's behaviour is to error on conflict; the + silent dedup. Today's behavior is to error on conflict; the proposed reform sets a clear precedence rule (host-set Class B properties win) — pending the constituents-overhaul decision. - **`ConstituentVarDict`** synthetic scope between suite and host. @@ -329,9 +332,10 @@ don't rebuild downstream objects unless something actually moved. ## 10. Where things stand right now -- **Unit tests**: 1229 passing on `main`. +- **Unit tests**: 1316 passing on `feature/capgen-ng` (as of 2026-05-19). - **End-to-end tests passing**: `advection`, `unit_conv`, - `nested_suite`, `variable_transform`, `instances`, `ddt`. + `nested_suite`, `variable_transform`, `instances`, + `instances_advection`, `ddt`. - **CCPP-SCM**: actively driving development — every build / runtime failure surfaced this week landed as a fix in capgen-ng (rather than being patched around in the host). Most of the `phys_ps` group now @@ -339,13 +343,19 @@ don't rebuild downstream objects unless something actually moved. - **`--no-host-introspection`** (new, 2026-05-14): stubs the bodies of the five suite-introspection routines in `ccpp_static_api.F90`, shrinking the file from ~33k lines to ~800 for the 10-suite SCM - build (the introspection case-blocks were making `-O3` compilation - effectively hang). Signatures stay so existing host callers still - link; stubbed bodies return `errflg = 1` with a clear `errmsg`. + build (the introspection case-blocks were making even `-O1` + compilation effectively hang). Signatures stay so existing host + callers still link; stubbed bodies return `errflg = 1` with a clear + `errmsg`. +- **NEPTUNE**: cleanup and acceptance testing in progress. + Regular/lower-atmosphere physics builds and runs and produces + results within tolerance (deviations similar to compiler changes). + High-altitude physics testing is next. +- **UFS Weather Model**: not yet attempted; SCM is the proving + ground first. An anticipated complication is the "fast physics" + called directly from the FV3 dynamical core as a separate group. - **CAM-SIMA**: not yet reconnected; pending the constituents - overhaul decision. -- **UFS Weather Model / NEPTUNE**: not yet attempted; SCM is the - proving ground first. + overhaul decision and CAM-SIMA developer availability. --- diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md index d284e7ae..ccbbcd2f 100644 --- a/doc/briefing_pm.md +++ b/doc/briefing_pm.md @@ -4,7 +4,7 @@ `doc/redesign_analysis.md` (the deep-dive technical comparison of prebuild and capgen). This document targets project leadership and program managers; it summarises the case for `capgen-ng` in terms of -product risk, schedule, and cross-organisation impact rather than +product risk, schedule, and cross-organization impact rather than implementation detail.* *Last revised: 2026-05-18.* @@ -97,7 +97,7 @@ Three pressures converged in 2025/26: reading several modules together. Realistically, only one or two people on the framework team can change capgen without breaking something downstream. One of them now lives overseas and rejects - simplification attempts from the others. This is an inacceptable + simplification attempts from the others. This is an unacceptable **bus-factor risk** that the redesign retires. --- @@ -107,7 +107,7 @@ Three pressures converged in 2025/26: This section is for the project lead who came from the capgen side: none of these are critiques of capgen as a *product*. They are specific architectural choices that worked for CAM-SIMA's -single-instance design and don't generalise. Each is sourced from +single-instance design and don't generalize. Each is sourced from the technical analysis in `doc/redesign_analysis.md` and validated by the SCM / multi-instance test work this month. @@ -341,9 +341,9 @@ Three points worth raising explicitly: into capgen-ng. 3. **The team owning capgen-ng can be larger than the team owning capgen.** This is the most important practical point for - long-term programme health. A framework that three organisations + long-term program health. A framework that three organizations can maintain is more resilient than a framework that one - organisation (or one individual in that organisation) can maintain. + organization (or one individual in that organization) can maintain. --- diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index 834e07c0..2bf6fad8 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -508,7 +508,7 @@ attributes: `datatable.xml`; the `diagnostic_name` slot stays empty (no auto-default to `local_name`). -The behavioural difference is purely *which attribute name* host +The behavioral difference is purely *which attribute name* host tooling sees in `datatable.xml` — both attributes carry the same kind of value (a Fortran-identifier-shaped string), and both are passed through unmodified. `_fixed` is a signal to the host "use verbatim, do diff --git a/doc/migration.md b/doc/migration.md index 2bb77ac4..d69f41c3 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -99,7 +99,7 @@ names resolved at codegen time and routinely exceed 63 chars). ### 1.5 Unit strings: bare vs explicit positive exponent `m2` and `m+2` (or any `` vs `+` -combo) are normalised internally and treated as equivalent. Pre-existing +combo) are normalized internally and treated as equivalent. Pre-existing unit-conversion entries don't need to be duplicated; either spelling matches. @@ -508,7 +508,7 @@ is staged to a sibling temp file under the output root and atomically replaces the target only when the bytes actually differ. Reruns with identical inputs therefore leave on-disk mtimes untouched, so CMake / Make / Ninja do not trigger a downstream rebuild cascade. Matches the -behaviour of legacy `ccpp-prebuild` / `ccpp-capgen`. The staging temp +behavior of legacy `ccpp-prebuild` / `ccpp-capgen`. The staging temp file lives in the target's parent directory (always under `--output-root`), so no `/tmp` access is required. @@ -571,7 +571,7 @@ host_var(lb:ub, nlev:1:-1) = 1.0E+3_kind_phys*temp_l ! ... reverse ... ``` Identity unit conversions (registered for dimensionally-equivalent -spellings like `J kg-1 ↔ m2 s-2`, formula `'{var}'`) are not labelled +spellings like `J kg-1 ↔ m2 s-2`, formula `'{var}'`) are not labeled "unit conversion" in the comment. ### 5.4 Subcycle emission @@ -702,7 +702,7 @@ complete). See `project_validator_host_check_deferred.md` (memory). | Item | Status | |--------------------------------------------|-----------------------------------------------| | `ccpp_loop_counter` standard name inside nested subcycles | Maps to OUTERMOST loop var. None of cam-sima uses this; revisit if a scheme needs the innermost value. | -| Validator host-metadata check | Deferred; revisit after e2e tests stabilise. | +| Validator host-metadata check | Deferred; revisit after e2e tests stabilize. | | Constituents overhaul (Class A/B + setters) | Discussion doc at `doc/constituents_overhaul.md`. | | Framework setters: `set_advected`, `set_diagnostic_name`, `set_default_value` | Deferred; depends on constituents-overhaul decision. | | Codegen-time scheme-registration cross-check | Deferred; would require new `registers_std_names` metadata attr. | @@ -713,7 +713,7 @@ complete). See `project_validator_host_check_deferred.md` (memory). | `fortran_to_metadata` developer utility | Deferred; bootstraps a `.meta` skeleton from an existing `.F90` subroutine. | | `--legacy-mode` shim removal | Transient; remove `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and every `# legacy-compat:` touchpoint when scheme metadata has migrated. | | `ccpp_datafile.py` query CLI rework | Deferred (2026-05-13); collapse `--host-files` / `--suite-files` / `--utility-files` into `--capgen-files`, then repurpose `--host-files` as a filtered list of **input** host metadata files (parallel to `--scheme-files`). Most hosts pack all host data into a handful of shared files, so the filtering pay-off is small — the draw is API symmetry. | -| Original capgen auto-clone path | Intentionally dropped in favour of explicit registration; kept in memory as "Option B" fallback. | +| Original capgen auto-clone path | Intentionally dropped in favor of explicit registration; kept in memory as "Option B" fallback. | --- diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index 8135ae92..2831871c 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -1164,7 +1164,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` - **Multiple `dependencies = …` lines** per `[ccpp-table-properties]`. - **Sliced local names** with long subscript-token CCPP standard names no longer trip the 63-char Fortran-id limit. -- **Unit normalisation** — `m2` ≡ `m+2` (and friends). +- **Unit normalization** — `m2` ≡ `m+2` (and friends). - **Subcycle bound = CCPP std name** — including DDT-component access paths (`phys_state%num_subcycles`). - **Nested ``** — preserved end-to-end as nested `do` loops. From 819164e064d6fc1a220b317ad332db1ce8ecef18 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 May 2026 10:35:11 -0600 Subject: [PATCH 30/74] Update all documents --- doc/briefing_20260519T0905.md | 381 ++++++++ doc/briefing_pm_20260519T0905.md | 360 +++++++ doc/constituents_20260519T0905.md | 1029 ++++++++++++++++++++ doc/constituents_overhaul_20260519T0905.md | 947 ++++++++++++++++++ doc/migration_20260519T0905.md | 729 ++++++++++++++ 5 files changed, 3446 insertions(+) create mode 100644 doc/briefing_20260519T0905.md create mode 100644 doc/briefing_pm_20260519T0905.md create mode 100644 doc/constituents_20260519T0905.md create mode 100644 doc/constituents_overhaul_20260519T0905.md create mode 100644 doc/migration_20260519T0905.md diff --git a/doc/briefing_20260519T0905.md b/doc/briefing_20260519T0905.md new file mode 100644 index 00000000..69f60336 --- /dev/null +++ b/doc/briefing_20260519T0905.md @@ -0,0 +1,381 @@ +# capgen-ng — Briefing for CCPP Framework Developers & Power Users + +*Prepared for the 2026-05-14 walk-through. Companion document to +`doc/migration.md` (the detailed migration guide) and +`doc/redesign_prompt.md` (the implementation spec).* + +--- + +## 1. Why a new generator? + +The CCPP Framework runs two code generators today: + +- **`ccpp-prebuild`** — simple, procedural Python; fast; DDT-argument + passing; in production use by NOAA UFS Weather Model, Navy NEPTUNE, + and CCPP-SCM. Reliable but feature-light. No framework-owned + variables. +- **`ccpp-capgen`** — complex, deeply object-oriented Python; flat-field + argument passing; in use by NCAR CAM-SIMA. Many advanced features + designed but never implemented; at UFS/NEPTUNE scale (1200+ variables) + flat-field passing prevents the use of strict error-checking flags + (`-check all`, `-fcheck=all`) required for operational implementation, + and even when it does compile it produces unmaintainably large source + files; nobody on the team fully understands it. + +**`capgen-ng`** starts fresh, drawing lessons from both. Guiding +principle: **simplicity of prebuild, feature set of capgen**. + +What we wanted to fix: + +1. **Flat fields → DDT arguments at all scales.** No "flat-field + group cap" failure mode. +2. **No scope-chain variable promotion.** Variables flow through + metadata, not through a runtime synthetic dictionary stacking. +3. **Code anyone can read and extend.** No 10-deep class hierarchy. +4. **One generator, one CLI, one query tool** for both prebuild-style + and capgen-style hosts. + +--- + +## 2. What capgen-ng is (in one paragraph) + +capgen-ng reads metadata for the **host model**, the **physics +schemes**, and the **suite definition files** (SDFs), produces a +small set of Fortran cap modules that bridge them, and writes a +`datatable.xml` describing the result for CMake / Make to consume. +At runtime the host calls a small set of public entry points +(`ccpp_register`, `ccpp_init`, `ccpp_physics_init`, +`ccpp_physics_run`, `ccpp_physics_*_init`/`_final`, `ccpp_final`); the +generated caps dispatch by `suite_name` (and optionally `group_name`) +to the right scheme. + +--- + +## 3. Core concepts + +### 3.1 Five metadata table types + +| `type = ` | Owner | How it reaches the cap | +|-------------|------------------|----------------------------| +| `scheme` | Physics scheme | Intent args on scheme subs | +| `host` | Host model | Module USE (direct / DDT) | +| `control` | Framework runtime layer | `ccpp_physics_*` args | +| `ddt` | Type definition | Structural — fields only | +| `suite` | Generated suite cap | Module USE | + +### 3.2 Three layers of generated cap + +- **Static API** (`ccpp_static_api.F90`) — public entry points; one per + build. Dispatches by `suite_name` → suite cap. +- **Suite cap** (`ccpp__cap.F90`) — per-suite state machine, plus + dispatch by `group_name` → group cap. Suite-owned interstitial data + lives in a sibling `ccpp__data.F90`. +- **Group cap** (`ccpp___cap.F90`) — scheme call sites + with full argument lists, unit/kind/vertical-flip transforms, + optional-arg pointer wrappers, subcycle `do` loops. + +### 3.3 Two-level integer state machine + +Replaces both the boolean `initialized(:)` array from prebuild and +the string-based `ccpp_suite_state` from CAM-SIMA capgen. Per +instance: + +- **Suite-level**: `UNREGISTERED → REGISTERED → FRAMEWORK_INITIALIZED`. +- **Group-level**: `UNINITIALIZED → INITIALIZED → IN_TIMESTEP`. + +### 3.4 Six scheme phases + +`register`, `init`, `timestep_init`, `run`, `timestep_final`, `final`. +`register` is new — schemes that contribute to the constituent table +do so here. `final` replaces the older `finalize` (breaking change, +intentional). + +### 3.5 Variable resolution + +For each scheme arg: + +1. Found in host+control metadata → use the access path. If units / + kind / vertical orientation differ, generate a transform. +2. Not found, first use is `intent(out)` → **suite-owned** variable + (interstitial); add to `ccpp__data.F90`. +3. Not found, first use is `intent(in/inout)` → **error**. +4. Found in suite data (a prior scheme provided it) → use suite data + access path. + +### 3.6 Two tools, one parser + +- `ccpp_capgen_ng.py` — the code generator. Trusts metadata; no + Fortran parsing. +- `ccpp_validator.py` — the standalone Fortran-vs-metadata checker. + The ONE place capgen-ng parses Fortran. Run by developers / + CMake before generation. + +Both share the same metadata-parsing library (`metadata/`). + +--- + +## 4. How capgen-ng differs from `ccpp-prebuild` + +| Topic | prebuild | capgen-ng | +|-----------------------------|-----------------------------------|---------------------------------------------------| +| Host metadata mechanism | Hard-coded Python dict (`TYPEDEFS_NEW_METADATA`) | Regular `type = ddt` + `type = host` tables | +| Framework-owned variables | Not supported | First-class (suite-owned interstitial via Case 2) | +| Constituents | Hand-rolled, host-specific glue | Standardized opt-in mechanism with auto-provision | +| `register` phase | Doesn't exist | First phase; schemes declare dynamic constituents | +| Multi-instance API | Implicit, ad-hoc | Paired-opt-in (`instance_number` / `number_of_instances`) | +| Subcycle loop counter | Host plumbs it manually | Registered std names `ccpp_loop_counter` / `ccpp_loop_extent` resolve to the do-loop locals automatically inside `` | +| Suite introspection | Limited | Five runtime queries (`ccpp_physics_suite_list`, `_part_list`, `_schemes`, `_variables`, `_host_data`) | + +--- + +## 5. How capgen-ng differs from `ccpp-capgen` + +| Topic | capgen | capgen-ng | +|-----------------------------|-----------------------------------|---------------------------------------------------| +| Group-cap arguments | Flat fields (1200+ at UFS scale) | DDT arguments (as in prebuild) | +| Variable matching algorithm | Five-layer scope-chain promotion | Flat host+control dict + suite-owned discovery (inherited from prebuild — primary reason runtime is comparable to prebuild) | +| External types (MPI f08 comm, ESMF clock) | Tabled (solution complexity) | First-class via `type = external::` | +| `type = module` in metadata | Yes | Renamed `type = host` | +| `is_constituent` scheme args | Auto-cloned by generator | Schemes register constituents explicitly in the `register` phase via `ccpp_constituent_properties_t(:)` | +| `ConstituentVarDict` | Synthetic scope between suite + host | Removed; constituents are one of four sources (`control`/`host`/`suite`/`constituent`) on `ResolvedArg` | +| `_state` runtime check | String | Integer (named parameters) | +| Fortran-vs-metadata check | Inside the generator | Separate tool (`ccpp_validator.py`) | +| Code complexity | Deep OO hierarchy | Flat data classes + procedural resolver | + +--- + +## 6. Breaking metadata changes hosts must make + +Comprehensive list — see `doc/migration.md` for full detail. + +### 6.1 Table types + +- `type = module` → **`type = host`**. + +### 6.2 Phase names + +- `_finalize` → **`_final`** in both metadata and + Fortran source. + +### 6.3 Standard names + +- `horizontal_loop_extent` → **`horizontal_dimension`** uniformly in + scheme metadata. (The chunk-vs-full-domain distinction is driven + by what the host passes for `horizontal_loop_begin` / + `horizontal_loop_end`.) +- `number_of_openmp_threads` → **`number_of_threads`** (matches the + `thread_number` control variable convention). + +Both are rewritten on the fly by **`--legacy-mode`** for a transition +period; the shim prints a banner listing every rewrite it performs +and is marked for clean removal. + +### 6.4 Required host `type = control` table + +Every host MUST declare scalar integers (and one character) with +these CCPP standard names: + +- `suite_name`, `horizontal_loop_begin`, `horizontal_loop_end`, + `thread_number`, `number_of_threads`, `number_of_physics_threads`, + `ccpp_error_code`, `ccpp_error_message`. + +Optional (paired): `instance_number` (control) + +`number_of_instances` (host). + +### 6.5 DDT-instance variables with scalar-index dims + +Container DDT-instance variables (`physics%Interstitial`, +`physics%Coupling`, ...) dimensioned by a count standard name +(`number_of_threads`, `number_of_instances`) get their scalar index +inserted **automatically** by capgen-ng. The host metadata declares +the dim; the generator emits +`physics%Interstitial(thread_number)%alpha(...)` at every call site. + +The host's Fortran can keep its existing OpenMP-thread-private DDT +layout — no glue code needed on the host side. + +### 6.6 Leaf variables MUST NOT carry registered scalar-index dims + +Rule 2 of the registered-scalar-index-dimension contract: scalar +variables (real / integer / character / DDT-typed leaves the scheme +binds to) cannot declare `number_of_threads` or `number_of_instances` +as a dimension. Wrap them in a container DDT instead. This is +enforced at parse time with an explicit remediation message; existing +CCPP-physics, UFS-WM, and CAM-SIMA host metadata is already +compliant. + +### 6.7 No more `cdata` / `ccpp_t` struct passing + +The framework-owned bag-of-state struct is replaced by explicit +control-variable arguments to the public entry points. + +--- + +## 7. What capgen-ng does NOT support (yet) + +### 7.1 Deferred — to be resolved in upcoming work + +- **Constituents overhaul.** Three reform proposals on the table + (`doc/constituents_overhaul.md`); decision pending an upcoming + meeting. Pieces involved: framework setter additions + (`set_advected`, `set_diagnostic_name`, `set_default_value`), + `is_match` relaxation, Class A vs Class B property classification. +- **Validator host-metadata check.** `ccpp_validator.py` currently + validates scheme metadata only; host-metadata-vs-Fortran is on + hold until the e2e test suite settles. +- **Codegen-time scheme-registration cross-check.** Today's + registration check is at runtime + (`ccpp_initialize_constituents`). Stronger options: new metadata + attribute `registers_std_names = a, b, c` on register-phase + tables; cross-check at codegen. +- **Nested-subcycle `ccpp_loop_counter` semantics.** When a scheme + inside a deeply nested subcycle asks for `ccpp_loop_counter`, it + currently resolves to the **outermost** loop's counter. None of + the in-tree physics catalogs uses the inner-counter case. +- **`ccpp_datafile.py --host-files` repurpose.** The current + `--host-files` returns the generated host-API file; should be a + filtered list of *input* host metadata files (parallel to the new + `--scheme-files`). Deferred. +- **`ccpp_host_constituents.F90` suppression** when no suite touches + constituents (file is correct-but-empty under host-wins; should + not be emitted at all). +- **Python linter / formatter pass.** Pick `ruff`, apply across + `capgen-ng/`. + +### 7.2 Intentionally NOT supported + +- **`_finalize` phase spelling.** Use `_final`. No legacy-mode + shim — rename in metadata + Fortran. +- **`type = module`.** Use `type = host`. +- **Flat-field scheme call arguments** (capgen's failure mode). +- **`character(len=*)` as a DDT component** (Fortran disallows it; + we error at parse time with a remediation pointing at + `character(len=:)` deferred-length). +- **Multiple registration sources for the same constituent** with + silent dedup. Today's behavior is to error on conflict; the + proposed reform sets a clear precedence rule (host-set Class B + properties win) — pending the constituents-overhaul decision. +- **`ConstituentVarDict`** synthetic scope between suite and host. + Gone for good. + +--- + +## 8. Validation and error reporting + +A deliberate design choice across capgen-ng: **errors are loud, +specific, and actionable**. Examples surfaced during the SCM +shake-down: + +- Empty `units =` line → error names file, line, variable, + attribute, raw value, AND inner reason. +- Scheme metadata file passed via `--scheme-files` but missing from + the SDF → silently ignored (and dropped from `` so + CMake doesn't compile orphan code). +- Scheme listed in the SDF but its metadata not supplied → single + CCPPError listing every missing scheme + pointer to + `--scheme-files`. Replaces silent empty-cap emission. +- DDT-instance variable with a non-registered scalar-index dim AND + flattenable fields → error shows the broken access pattern + capgen-ng WOULD have emitted and quotes the Fortran compiler + error verbatim ("Component to the right of a part reference with + nonzero rank must not have the POINTER attribute"). +- Generated `case default` on `select case(suite_name)` / + `select case(group_name)` → unknown suite or group at runtime + produces a clear errflg + errmsg, not silent fall-through. + +--- + +## 9. Build-system integration (capsule view) + +```cmake +# In your CMakeLists.txt +set(SCHEME_METADATA_FILES …list of .meta paths…) +set(HOST_METADATA_FILES …list of host .meta paths…) +set(SUITE_FILES …list of suite XML paths…) + +# Validate before generation (developer step, optional in CI) +ccpp_validator(SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES}) + +# Run the code generator +ccpp_capgen(HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT ${OUTPUT_ROOT}) + +# Pull the manifest from the datatable +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES ${CCPP_FILES}) + +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) + +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +add_library(scm-ccpp STATIC + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES}) +``` + +Regenerating on every CMake configure is cheap — `write_if_changed` +preserves mtimes when content hasn't changed, so `make` / `ninja` +don't rebuild downstream objects unless something actually moved. + +--- + +## 10. Where things stand right now + +- **Unit tests**: 1316 passing on `feature/capgen-ng` (as of 2026-05-19). +- **End-to-end tests passing**: `advection`, `unit_conv`, + `nested_suite`, `variable_transform`, `instances`, + `instances_advection`, `ddt`. +- **CCPP-SCM**: actively driving development — every build / runtime + failure surfaced this week landed as a fix in capgen-ng (rather than + being patched around in the host). Most of the `phys_ps` group now + builds end-to-end via `--legacy-mode`. +- **`--no-host-introspection`** (new, 2026-05-14): stubs the bodies of + the five suite-introspection routines in `ccpp_static_api.F90`, + shrinking the file from ~33k lines to ~800 for the 10-suite SCM + build (the introspection case-blocks were making even `-O1` + compilation effectively hang). Signatures stay so existing host + callers still link; stubbed bodies return `errflg = 1` with a clear + `errmsg`. +- **NEPTUNE**: cleanup and acceptance testing in progress. + Regular/lower-atmosphere physics builds and runs and produces + results within tolerance (deviations similar to compiler changes). + High-altitude physics testing is next. +- **UFS Weather Model**: not yet attempted; SCM is the proving + ground first. An anticipated complication is the "fast physics" + called directly from the FV3 dynamical core as a separate group. +- **CAM-SIMA**: not yet reconnected; pending the constituents + overhaul decision and CAM-SIMA developer availability. + +--- + +## 11. Walk-through outline (suggested order for the meeting) + +1. Live `ccpp_capgen_ng.py --help` (CLI shape). +2. Show one scheme's `.meta` + its generated group-cap fragment. +3. Run the generator twice — note the `Unchanged: …` messages on the + second pass (write-if-changed in action). +4. Run `ccpp_datafile.py --scheme-files datatable.xml` to show the + filtered manifest. +5. Demonstrate a deliberately-broken metadata (`units =` empty, or + missing scheme, or invalid `case default` group) to show the + error UX. +6. Walk through the registered scalar-index dimension table and the + two rules. +7. Open the floor — focus areas for the audience: + - **Host metadata maintainers**: anything in §6 that surprises + you for your model? + - **Scheme metadata maintainers**: anything in §6.2 / §6.3 that + can't be migrated cleanly? + - **Framework devs**: §7.1 — which deferred items block your + downstream work? diff --git a/doc/briefing_pm_20260519T0905.md b/doc/briefing_pm_20260519T0905.md new file mode 100644 index 00000000..ccbbcd2f --- /dev/null +++ b/doc/briefing_pm_20260519T0905.md @@ -0,0 +1,360 @@ +# capgen-ng — Briefing for Project Management + +*Companion to `doc/briefing.md` (the developer walk-through) and +`doc/redesign_analysis.md` (the deep-dive technical comparison of +prebuild and capgen). This document targets project leadership and +program managers; it summarises the case for `capgen-ng` in terms of +product risk, schedule, and cross-organization impact rather than +implementation detail.* + +*Last revised: 2026-05-18.* + +--- + +## TL;DR + +The CCPP Framework today ships **two** code generators that solve the +same problem differently: + +- **`ccpp-prebuild`** powers NOAA UFS, Navy NEPTUNE, and CCPP-SCM. + Simple and reliable, but feature-light — does not support features + CAM-SIMA needs (constituents, framework-owned variables, + introspection). +- **`ccpp-capgen`** powers NCAR CAM-SIMA. Feature-rich, but built on + technical choices that **do not scale** to UFS or NEPTUNE and that + **do not support multi-instance hosts** at all. + +Neither generator can be the basis for a single shared toolchain. +**`capgen-ng`** is a third generator, started in early May 2026, +designed to do everything both other generators do, in code small +enough for a few people to own, with the architectural choices that +make it work at UFS/NEPTUNE scale and beyond. The redesign is +running on the SCM as proving ground; UFS / NEPTUNE / CAM-SIMA +re-integration is sequenced behind that. + +This document explains, in plain language, **why we did not extend +capgen instead**, what risks the redesign retires, and where things +stand. + +--- + +## 1. The three generators in one paragraph each + +**`ccpp-prebuild`** (NOAA, in production for UFS, NEPTUNE, SCM). +Procedural Python; reads metadata; emits Fortran caps; passes +host-defined derived-type (DDT) arguments to scheme call sites. In +production for several years; bug rate is low; the team understands +it (those who worked with it). What it doesn't do: framework-owned +variables, the constituent mechanism CAM-SIMA needs, and runtime +introspection. Treated as the **baseline for simplicity and +reliability**. + +**`ccpp-capgen`** (NCAR, in production for CAM-SIMA). Heavy +object-oriented Python (deep class hierarchy, ~tens of thousands of +lines); reads metadata; emits Fortran caps that pass **flat scalar +fields** to scheme call sites instead of DDTs; supports the +constituent mechanism, suite-owned variables, introspection, and a +few other features prebuild lacks. **It is the only existing +generator with those features.** But — see §3 — it has structural +limits that make it impractical for UFS, NEPTUNE, or multi-instance +hosts, and the implementation is concentrated enough that few people +can extend it safely (if at all - primary developer gone). + +**`capgen-ng`** (new, 2026-05). Procedural Python (a few thousand +lines; flat data classes); reads the same metadata format; passes +arguments like prebuild; supports the features capgen pioneered +(constituents, suite-owned variables, introspection); supports +multi-instance, an integer state machine, six explicit scheme +phases, vertical-flip / unit / kind transforms, registered +scalar-index dimensions for threading and ensembles, write-if-changed +build integration, and a separate Fortran-vs-metadata validator +tool. Designed so the same generator works for prebuild-style hosts +(UFS / NEPTUNE / SCM) and capgen-style hosts (CAM-SIMA). + +--- + +## 2. Why this matters now + +Three pressures converged in 2025/26: + +1. **Framework unification heavily delayed.** A fully-functional + capgen that supports UFS / NEPTUNE / SCM and replaces prebuild + was promised for years, and never delivered. Pressure from + project management and sponsors is building. +2. **UFS / NEPTUNE want the capgen feature set.** Constituents + in particular are increasingly central to atmospheric physics + (chemistry, aerosols, deep atmosphere), and re-implementing the + prebuild-side glue per host is duplicated effort. Extending + capgen to UFS-scale runs into the flat-field problem (§3.1) — + not a small refactor, a fundamental data-shape change. The + performance of capgen generating multi-suite caps is up to + 20 times slower than that of prebuild for the CCPP SCM. This + is caused by fundamental design choices (five layers of + classes inheriting from each other) that are integral to capgen. +3. **The team owning capgen has limited bandwidth to extend it.** + The class hierarchy is intricate; understanding the + `ConstituentVarDict` scope-chain or the auto-clone path requires + reading several modules together. Realistically, only one or two + people on the framework team can change capgen without breaking + something downstream. One of them now lives overseas and rejects + simplification attempts from the others. This is an unacceptable + **bus-factor risk** that the redesign retires. + +--- + +## 3. What capgen does that does not extend to UFS / NEPTUNE / multi-instance + +This section is for the project lead who came from the capgen side: +none of these are critiques of capgen as a *product*. They are +specific architectural choices that worked for CAM-SIMA's +single-instance design and don't generalize. Each is sourced from +the technical analysis in `doc/redesign_analysis.md` and validated +by the SCM / multi-instance test work this month. + +### 3.1 Flat-field argument passing fails at UFS / NEPTUNE scale + +CAM-SIMA's group caps pass **every individual variable as a separate +argument** to the scheme dispatch routine. At CAM-SIMA's roughly +two-hundred-variable scale this works. At UFS scale (~1200 +variables per group), the generated Fortran exceeds compiler limits +prevents the use of strict error-checking flags (`-check all`, +`-fcheck=all`) required for operational implementation, and even +when it does compile produces unmaintainably large source files. +**This is the technical reason capgen cannot drive UFS today**, +independent of any other concern. + +`capgen-ng` reverts to prebuild's DDT-argument convention. Host +authors pass their physics DDTs by reference (one or a few arguments +per scheme call); component access happens **at the scheme call level**. +This works at every scale we've measured. + +### 3.2 Single-instance constituents are baked into the generated code + +CAM-SIMA runs one host per executable, so capgen generates a single +module-level `ccpp_model_constituents_obj`. The constituent +mechanism — the central feature capgen-ng inherited from capgen — +references that global directly. Re-targeting capgen to +multi-instance is not a configuration toggle; it requires +re-emitting the constituent module per-instance throughout the +generator, plus refactoring the framework setters. + +`capgen-ng` was multi-instance **from day one**: every constituent +entry point takes an `instance_number` argument; the property +storage, the state machine, the dynamic-constituent buffers are all +per-instance. As of 2026-05-18, the per-suite dynamic-constituents +buffer was also moved per-instance after the new combined +multi-instance + constituents end-to-end test surfaced a latent bug +that capgen would never have hit (because capgen never supported +multi-instance). **The redesign is finding bugs the legacy +toolchain hid.** + +### 3.3 Constituent registration has three competing paths in capgen + +capgen accepts constituent declarations from (a) host-supplied +arrays, (b) scheme `register`-phase Fortran subroutines, and (c) an +**auto-clone path** that scans scheme metadata for the +`is_constituent` attribute and silently generates a registration in +the host cap. The auto-clone path is invisible from the scheme +Fortran — to know whether a scheme registers a constituent you have +to know the generator semantics. This makes scheme code harder to +read, harder to port between hosts, and harder to debug when +registrations collide. + +`capgen-ng` keeps only the first two (explicit) paths. Auto-clone +is deliberately gone — see `doc/constituents_overhaul.md` §2.3. + +### 3.4 Host-specific values baked into scheme metadata + +capgen requires `diagnostic_name` (host's diagnostic-output label, +e.g. `CLDLIQ` for CAM-SIMA but something else for UFS) at +constituent instantiation time. Schemes therefore embed +host-specific strings into their own metadata. Porting a scheme +between hosts requires either editing the scheme or maintaining a +fork. + +`capgen-ng` is moving `diagnostic_name` (and a handful of other +host-configuration properties) to a host-side override mechanism; +schemes carry physics-portable defaults only. The reform is +documented in `doc/constituents_overhaul.md`; the decision is on the +agenda for one of the next framework-team meetings. + +### 3.5 Synthetic variable-resolution scopes are hard to extend + +capgen introduces a five-layer deep synthetic dictionary +(`ConstituentVarDict`) between the suite and host scopes during +variable matching. The mechanism works for capgen's use cases +but is a code path most contributors don't read. Extending the +resolver to handle multi-instance dimensions, scalar-index +substitution, or constituent host-wins semantics required +undoing parts of the synthetic scope. + +`capgen-ng`'s resolver is flat: each scheme arg is classified into +exactly one source (control / host / suite / constituent), recorded +on a small data class (`ResolvedArg`), and used directly by the +emitter. No synthetic dictionary. **This design inherits from +`prebuild` and is the primary reason `capgen-ng` is comparable +in performance to `prebuild`. + +### 3.6 Code volume and team coverage + +capgen is roughly an order of magnitude larger than prebuild, with a +deeply layered class hierarchy. This is not a moral failing — it +reflects the feature set — but the practical consequence is that +the maintenance burden falls on a small subset of the framework +team. capgen-ng is comparable to prebuild in size (a few thousand +lines of mostly procedural Python with small data classes), so the +"who can fix this" pool is closer to "anyone with framework +context". capgen-ng comes with over a thousand docstring tests +and unit tests, as well as a comprehensive end-to-end test suite +that covers all of prebuild's and capgen's existing end to end +tests. capgen-ng adds additional end-to-end tests for new +features such as the multi-instance constituents test. Including +these tests and the rich inline comments makes capgen-ng comparable +in size to capgen. + +--- + +## 4. What `capgen-ng` does better than capgen — at any scale + +For audiences who already accept the multi-instance and UFS-scale +arguments, the day-to-day quality-of-life improvements that apply +even to CAM-SIMA-shape problems: + +| Topic | capgen | capgen-ng | +|---|---|---| +| Scheme call argument shape | Flat fields | DDT references | +| Variable resolution | Scope-chain promotion via synthetic dict | Flat 4-source classification on `ResolvedArg` | +| Suite state runtime check | String comparison | Integer-named-parameter state machine | +| Fortran-vs-metadata validation | Embedded in generator | Standalone tool (`ccpp_validator.py`) — run by developers or CMake before generation | +| Generator code style | Deep class hierarchy | Flat data classes + procedural resolver | +| Error reporting | Variable amount of context | "Loud, specific, actionable" enforced — every parse-time error names file, line, variable, attribute, value, and reason | +| Constituent registration | Three sources (one invisible) | Two sources, both explicit | +| `is_constituent` auto-clone | Yes (host-specific values baked into scheme metadata) | Removed | +| `_finalize` vs `_final` phase name | `_finalize` | `_final` (renamed to keep symmetry with init/timestep_init/timestep_final) | + +--- + +## 5. Additional features of `capgen-ng` compared to `capgen` + +Features that exist only in capgen-ng (some exist in prebuild): + +| Capability | Why it matters | +|---|---| +| **Multi-instance host support** (per-instance state machine, per-instance constituent objects, per-instance dynamic-constituents buffers as of 2026-05-18) | Required by NEPTUNE (prebuild has basic solution) | +| **Registered scalar-index dimensions** | When metadata says a variable is dimensioned by `number_of_threads` or `number_of_instances`, capgen-ng injects the right per-call subscript automatically; the host's OpenMP-thread-private DDT layout works unchanged | +| **Subcycle loop-counter automation** | Schemes inside a `` element can access `ccpp_loop_counter` / `ccpp_loop_extent` directly; the generator emits the Fortran `do` loop and binds the locals | +| **`--legacy-mode` migration shim** | One CLI flag enables silent rewrite of two known-good deprecated standard names (`horizontal_loop_extent` → `horizontal_dimension`, `number_of_openmp_threads` → `number_of_threads`) with a loud warning — buys time for host metadata to migrate | +| **`--no-host-introspection` flag** | The five runtime introspection routines (`ccpp_physics_suite_list`, etc.) emit large `select case` blocks at SCM scale; this flag stubs the bodies, dropping the generated static API from ~33,000 lines to ~800 for the SCM build (the introspection routines were making `-O1` compilation effectively hang) | +| **Consistent handling of external types** (MPI f08 communicator, ESMF clock) | Tabled in capgen because of the complexity of the solution | + +--- + +## 6. Where things stand right now (2026-05-18) + +- **Unit tests**: 1319 passing. No known failures. +- **End-to-end tests**: 10 passing — `chunked_data`, `opt_arg`, + `nested_suite`, `ddthost`, `instances`, `capgen_ng`, + `var_compat`, `advection`, and the new `instances_advection` + (multi-instance + constituents). +- **CCPP-SCM**: actively driving development. Each build / runtime + issue surfaced this week landed as a fix in capgen-ng rather than + a host-side workaround. All available suites in CCPP-SCM now + build and run end-to-end via `--legacy-mode`. +- **Multi-instance + constituents fix landed 2026-05-18**. The new + combined end-to-end test surfaced a latent shared-buffer mutation + bug; the fix moves the per-suite dynamic-constituents buffer + per-instance. No coordination with CAM-SIMA / UFS / NEPTUNE + required (host-facing API unchanged). +- **NEPTUNE**: cleanup and acceptance testing in progress this week. + Regular/lower atmosphere physics builds and runs, and produces + results within the tolerance (i.e. similar to compiler changes). + High altitude atmosphere testing is next. +- **UFS Weather Model**: not yet attempted; SCM is the proving + ground first. Expecting updates due to the "fast physics" + called directly from the FV3 dynamical core as separate group. +- **CAM-SIMA**: not yet re-connected; the constituent overhaul + decision (see §7) and the availability of CAM-SIMA developers + are the gating items. + +--- + +## 7. What is intentionally NOT decided yet + +The redesign is opinionated about the architectural choices (DDT +arguments, per-instance everything, integer state machine, two-tool +split). It is **not** opinionated about the framework-level +constituent reform. + +`doc/constituents_overhaul.md` lays out three reform proposals on +the table: + +- **Proposal A** (mostly landed): bug-fix on the deallocate path + + add missing host setters for properties the host wants to + override. Conservative. +- **Proposal B** (recommended for the next 4–6 weeks): relax the + identity-equality check, formally classify properties as + "scheme-intrinsic" (immutable) vs "host-configuration" (mutable + after registration). Physics schemes using constituents become + genuinely portable across hosts. +- **Proposal C** (tabled): drop scheme-side constituent + registration entirely; only the host registers. Cleaner but + requires coordinated PRs across the framework, both generators, + the CAM-SIMA atmospheric_physics tree, and CAM-SIMA itself. + +These are open questions for the framework-team meeting, not +capgen-ng decisions. capgen-ng is structured so all three +proposals are implementable on top of it. + +--- + +## 8. Risk register (project-management view) + +| Risk | Status | Mitigation | +|---|---|---| +| capgen-ng diverges from capgen feature set | LOW | Cross-checked by `doc/redesign_analysis.md`; the feature comparison table in §4 / §5 is exhaustive | +| Host metadata break for UFS / NEPTUNE / CAM-SIMA | MEDIUM | `--legacy-mode` shim covers the known incompatible standard-name pair; remaining required changes (e.g., `_finalize` → `_final`) are mechanical and listed in `doc/migration.md` §3 | +| Constituent overhaul stalls | MEDIUM | Proposal A unblocks the immediate bug; capgen-ng works with the current framework today; the overhaul is a separate decision track | +| Bus-factor on capgen-ng itself | MEDIUM | Procedural code style + flat data classes + 1319-test safety net; significantly lower than capgen's bus factor | +| Two host call-shape conventions (prebuild-style vs capgen-style) coexist forever | LOW | capgen-ng emits one shape; downstream host conversions are tracked in `doc/migration.md` | +| Regression discovered during NEPTUNE / UFS testing | EXPECTED | SCM proving ground catches most; remaining issues become capgen-ng tickets, not host-side patches | +| ccpp-prebuild end-of-life requires a sunset plan | OPEN | Not yet scoped; both generators currently coexist in the framework repo | + +--- + +## 9. The pragmatic case (for the meeting) + +Three points worth raising explicitly: + +1. **Extending capgen to UFS/NEPTUNE scale is not a configuration + change — it is a refactor of the same magnitude as a redesign.** + The flat-field convention is load-bearing throughout capgen's + variable-matching, resolution, and emission code. Once that + change is made, the resulting generator looks substantially + like capgen-ng anyway. +2. **The features capgen pioneered (constituents, suite-owned + variables, introspection) are kept and improved — not + discarded.** capgen-ng is genuinely the successor, not a + parallel project. The contributions made on the capgen side are + what made the capgen-ng feature set possible. A significant + portion of capgen's code, in particular metadata parsine, + Fortran-metadata validation, and constituents, were imported + into capgen-ng. +3. **The team owning capgen-ng can be larger than the team owning + capgen.** This is the most important practical point for + long-term program health. A framework that three organizations + can maintain is more resilient than a framework that one + organization (or one individual in that organization) can maintain. + +--- + +## 10. References + +- `doc/briefing.md` — developer walk-through; same outline, more + technical detail. +- `doc/redesign_analysis.md` — deep-dive technical comparison of + prebuild and capgen with named-product examples. +- `doc/migration.md` — host-author migration guide. +- `doc/constituents_overhaul.md` — the constituent-reform discussion + document. +- `end-to-end-tests/` — the working examples (`instances_advection` + is the newest, exercises everything end-to-end). diff --git a/doc/constituents_20260519T0905.md b/doc/constituents_20260519T0905.md new file mode 100644 index 00000000..c7581801 --- /dev/null +++ b/doc/constituents_20260519T0905.md @@ -0,0 +1,1029 @@ +# CCPP capgen-ng — Constituents Reference + +*Last revised: 2026-05-13.* + +This document is the authoritative reference for **constituent variables** in +capgen-ng — what they are, how scheme authors declare them in metadata, what +the host model has to do to plumb them through, what the generator emits, and +how the per-instance lifecycle works. + +> If you are migrating a host or scheme from the original capgen, jump to +> [§9 Differences from original capgen](#9-differences-from-original-capgen) +> first. + +--- + +## Table of Contents + +1. [What is a constituent?](#1-what-is-a-constituent) +2. [The four rules (scheme-author conventions)](#2-the-four-rules-scheme-author-conventions) +3. [Required host metadata + Fortran](#3-required-host-metadata--fortran) +4. [Host-side lifecycle (call sequence)](#4-host-side-lifecycle-call-sequence) +5. [Public API reference](#5-public-api-reference) +6. [Generated code structure](#6-generated-code-structure) +7. [Multi-instance design](#7-multi-instance-design) +8. [Limitations and gotchas](#8-limitations-and-gotchas) +9. [Differences from original capgen](#9-differences-from-original-capgen) +10. [Worked example](#10-worked-example) + +--- + +## 1. What is a constituent? + +A **constituent** is a model variable owned by the host's dynamical core (or +its constituent infrastructure) that is read and updated by physics schemes — +typically a tracer / mass mixing ratio (water vapor, cloud liquid, ozone, +chemistry species) — together with its **tendency**, the rate of change that +physics writes back so the dycore can advect/integrate it forward. + +In capgen-ng, the constituent layer has three concerns: + +1. **Registration** — declaring at model startup which constituents exist + (their standard name, units, vertical layout, advection flag, …). +2. **Storage** — the framework owns one `ccpp_model_constituents_t` DDT per + host instance (see [§7](#7-multi-instance-design)) which holds the + constituent values (`%vars_layer`), tendencies (`%vars_layer_tend`), and + metadata (`%const_metadata`). +3. **Access** — schemes reference constituents by standard name in their + metadata; the resolver translates those references to + `ccpp_model_constituents_obj(inst)%vars_layer(slice, index_of_)` + subscripts at code-gen time. + +All constituent state lives in **one generated module**: +`ccpp_host_constituents.F90` (one per generator run, emitted only when at +least one suite touches constituent state). Public symbols from this module +are also re-exported by `ccpp_static_api`, so most host code only needs + +```fortran +use ccpp_static_api, only: ccpp_register_constituents, ccpp_initialize_constituents, & + ccpp_constituents_array, ccpp_const_get_index, ... +``` + +--- + +## 2. The four rules (scheme-author conventions) + +These four rules govern every scheme-arg metadata pattern related to +constituents. They derive from a 2026-05-11 audit of all 12 cam-sima scheme +metadata files that touch constituent attributes. + +### Rule 1 — Register a new constituent (register phase) + +A scheme that creates a new constituent declares it in the **register** +phase via an `intent=out, allocatable` array of +`ccpp_constituent_properties_t`: + +``` +[ccpp-arg-table] + name = my_scheme_register + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_my_scheme + long_name = per-scheme constituent array + units = none + dimensions = (:) + type = ccpp_constituent_properties_t + allocatable = True + intent = out +[ errmsg ] + ... +[ errflg ] + ... +``` + +The scheme's Fortran register routine `allocate`s this array, populates +each entry via `%instantiate(std_name=..., long_name=..., units=..., +vertical_dim=..., advected=..., ...)` and returns it. The framework +captures every register-phase scheme's array, packs them into a per-suite +buffer (`_dynamic_constituents`), and merges them into each +host-instance's constituent object during `ccpp_register_constituents`. + +This is the **only path** for declaring a new constituent. + +### Rule 2 — Consume a base constituent (any physics phase) + +A scheme that reads (or reads + writes) an existing base constituent +declares the variable with `is_constituent` set (any of `advected`, +`constituent`, or `molar_mass` non-default) and `intent=in` or `intent=inout`: + +``` +[ cldliq ] + standard_name = cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in ! or inout + advected = true +``` + +The resolver translates this scheme arg to +`ccpp_model_constituents_obj()%vars_layer(, index_of_)` +in the generated group cap. No host metadata declaration is needed for +the variable. + +### Rule 3 — Produce a tendency (any physics phase) + +A scheme that writes a constituent tendency declares the variable with +`is_constituent` set, `intent=out`, and a standard name that **starts +with `tendency_of_`**: + +``` +[ tend_cldliq ] + standard_name = tendency_of_cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = true +``` + +The resolver translates this scheme arg to +`ccpp_model_constituents_obj()%vars_layer_tend(, index_of_)` +where `` is the std_name with the `tendency_of_` prefix stripped. +The tendency variable is implicitly tied to the base constituent of the +same name. + +### Rule 4 — Mismatched combinations are hard errors + +Two combinations are explicitly rejected by the resolver at code-gen time: + +| Mismatch | Error | +|---|---| +| `is_constituent=True` + `intent=out` + std_name does NOT start with `tendency_of_` | *"Physics phases may only produce constituent tendencies; new base constituents must be declared via a `ccpp_constituent_properties_t` argument in a register-phase scheme."* | +| `is_constituent=True` + `intent in (in, inout)` + std_name starts with `tendency_of_` | *"Constituent tendency arg must be declared with intent=out; physics phases only produce tendencies, never consume them."* | + +### Direct framework-array access + +A scheme may also access the framework's bulk arrays directly by +declaring an arg with one of these standard names: + +| Standard name | Maps to | +|---|---| +| `ccpp_constituents` | `ccpp_model_constituents_obj()%vars_layer` (3D) | +| `ccpp_constituent_tendencies` | `ccpp_model_constituents_obj()%vars_layer_tend` (3D) | +| `ccpp_constituent_properties` | `ccpp_model_constituents_obj()%const_metadata` (1D of `ccpp_constituent_prop_ptr_t`) | +| `number_of_ccpp_constituents` | `ccpp_model_constituents_obj()%num_layer_vars` (scalar integer) | +| `index_of_` | module-level `integer :: index_of_` (no per-instance access — the index is identical for every instance) | + +The trailing dimension `number_of_ccpp_constituents` in a 3D scheme arg +is emitted as `:` (whole-axis slice). + +--- + +## 3. Required host metadata + Fortran + +### Host metadata (`type=host` table) + +The host **must** declare: + +``` +[ ] + standard_name = number_of_instances + units = count + dimensions = () + type = integer +``` + +… **only when the host actually wants multi-instance support**. When +absent, every per-instance allocation falls back to size `1` and the +host effectively runs single-instance. + +The host **does not** need to declare: + +- `ccpp_model_constituents_object` — the constituent object is owned + by the generator (in `ccpp_host_constituents`); the host doesn't + declare it in metadata. +- `ccpp_constituents`, `ccpp_constituent_tendencies`, + `ccpp_constituent_properties`, `number_of_ccpp_constituents`, + `index_of_` — all auto-provided by the generator. + +#### Host metadata wins over auto-provisioning + +If the host **does** declare any of the framework-named standard +names above as a regular host variable, the resolver uses the host's +declaration instead of auto-provisioning. This matters for legacy +hosts (GFS / SCM) that own their own tracer indices: + +```meta +[ ntcw ] + standard_name = index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array + units = index + type = integer + protected = True + dimensions = () +``` + +A scheme arg requesting the same standard name resolves to the host's +short local name (`ntcw`), not a parallel module-level integer in +`ccpp_host_constituents` named after the full standard name (which +would blow the Fortran 63-character identifier limit). Auto-provisioning +only fires for framework-named standard names the host has **not** +claimed. + +### Host control-table requirements + +The host's `type=control` table must declare: + +``` +[ ] + standard_name = instance_number + units = 1 + dimensions = () + type = integer +``` + +… so the framework signature knows the index for per-instance state. +Same caveat as `number_of_instances` — required only when multi-instance +is wanted. + +### Host Fortran code + +The host's Fortran code only needs to: + +1. Maintain its own `integer :: ` for `number_of_instances` + in a module that's USE'd by the generator. (Same module that owns + the metadata.) +2. Build its **host constituents** array (water vapor, ozone, etc. — + the constituents that the host model owns directly, separately from + any scheme-registered ones). Pass this to + `ccpp_register_constituents`. + +The host does **not** need to allocate or own a +`type(ccpp_model_constituents_t)` variable. + +--- + +## 4. Host-side lifecycle (call sequence) + +``` + ┌─ host startup ─┐ + │ + ▼ + ┌──────────────────────────────────────┐ + │ for each instance: │ + │ ccpp_register(suite_name, │ + │ errmsg, errflg, │ + │ instance_number) │ ─── per-instance ───┐ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ allocate host_constituents(:) │ │ + │ host_constituents(1)%instantiate( │ ─── once ─────────┘ + │ std_name='water_vapor_specific_humidity', ...) │ + │ ... │ │ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_register_constituents( │ │ + │ host_constituents, │ │ + │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_initialize_constituents( │ │ + │ ncols, num_layers, │ │ + │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_init(suite_name, │ │ + │ errmsg, errflg, │ │ + │ instance_number) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ (model time-stepping) │ + ┌──────────────────────────────────────┐ │ + │ ccpp_physics_*(...) │ ─── per-instance ──┤ + └──────────────────────────────────────┘ │ + │ │ + ▼ (host shutdown) │ + ┌──────────────────────────────────────┐ │ + │ for each instance: │ │ + │ ccpp_deallocate_dynamic_constituents( │ + │ instance_number) │ ─── per-instance ──┤ + │ ccpp_final(suite_name, │ │ + │ errmsg, errflg, │ │ + │ instance_number) │ │ + └──────────────────────────────────────┘ │ + │ + ┌────────────────────────────────────────┘ + │ ◀── last-to-leave dealloc fires + │ automatically inside the per-instance + │ calls when the final instance finishes. + ▼ +``` + +### Important ordering rules + +- `ccpp_register_constituents` **must** be called *after* `ccpp_register` + (per instance). The latter populates the per-suite dynamic-constituent + buffers via `_register`; the former merges them into the + per-instance constituent object. +- `ccpp_initialize_constituents` **must** be called *after* + `ccpp_register_constituents` (per instance). It calls `%lock_data` + on the per-instance object — which can only happen once + `%lock_table` has fired (which `ccpp_register_constituents` does). +- Physics phases (`ccpp_init`, `ccpp_physics_run`, etc.) require + the constituent state to be locked + bound (i.e., + `ccpp_initialize_constituents` already called). +- `ccpp_deallocate_dynamic_constituents` is per-instance with + last-to-leave teardown. Once the last instance calls it, the shared + per-suite buffers and the constituent object array are deallocated + automatically. + +### Built-in constituents vs scheme-registered constituents + +`ccpp_register_constituents` takes one explicit argument: an array of +`ccpp_constituent_properties_t` describing the **host's own constituents** +(typically water vapor and any other tracers the dycore carries +intrinsically). The framework then merges those entries with every +suite's per-suite dynamic-constituent buffer (populated during +`ccpp_register` from each register-phase scheme's output). + +Pass an empty (zero-size) array if the host has no built-in constituents +of its own. + +--- + +## 5. Public API reference + +All routines below live in `ccpp_host_constituents` and are also +re-exported from `ccpp_static_api` for convenience. The dummy-argument +name `instance_number` is the **standard name**; the actual emitted +dummy uses the host's local name for it (typically also +`instance_number` or `inst_num`). + +### `ccpp_register_constituents(host_constituents, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `host_constituents` | `type(ccpp_constituent_properties_t), target, intent(in) :: (:)` | Host-owned constituent declarations (water vapor, etc.). May be zero-size. | +| `instance_number` | `integer, intent(in)` | Per-instance index. | +| `errflg` | `integer, intent(out)` | Error flag (0 = success). | +| `errmsg` | `character(len=*), intent(out)` | Error message. | + +**Effect**: +- On the first call across instances, allocates + `ccpp_model_constituents_obj(number_of_instances)`. +- Calls `obj(instance_number)%initialize_table(num_consts)` where + `num_consts = size(host_constituents) + sum(size(_dynamic_constituents))`. +- Iterates `host_constituents` first, then every suite's + `_dynamic_constituents` buffer, calling + `obj(instance_number)%new_field(const_prop, errcode=errflg, errmsg=errmsg)` + for each entry. +- Calls `obj(instance_number)%lock_table(...)`. + +**Preconditions**: every `_register` call (across all suites) for +this instance has already happened (so the per-suite buffers are +populated). + +### `ccpp_initialize_constituents(ncols, num_layers, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `ncols` | `integer, intent(in)` | Horizontal extent for this instance's chunk. | +| `num_layers` | `integer, intent(in)` | Vertical layer count. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +**Effect**: +- Calls `obj(instance_number)%lock_data(ncols, num_layers, ...)` — + allocates `obj(inst)%vars_layer` and `%vars_layer_tend` arrays. +- Registers a singleton pointer with + `ccpp_scheme_utils.ccpp_initialize_constituent_ptr(obj(inst))` so + cam-sima schemes that call `ccpp_constituent_index` see the + constituent table. **First instance wins** — see + [§8 Limitations](#8-limitations-and-gotchas). +- Queries `obj(instance_number)%const_index(index_of_, '', ...)` for + every constituent `` known at code-gen time; populates the + module-level integer `index_of_`. These integers are identical + across instances; the last call to set them wins (benign — the + constituent table is the same per instance). + +**Preconditions**: `ccpp_register_constituents` has been called for this +instance. + +### `ccpp_is_scheme_constituent(var_name, constituent_exists, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `var_name` | `character(len=*), intent(in)` | Standard name to query. | +| `constituent_exists` | `logical, intent(out)` | True iff *var_name* matches one of the constituent std names known to capgen at code-gen time. | +| `errflg` / `errmsg` | `intent(out)` | | + +**No `instance_number`** — the data lookup is against the module-level +`character(len=N), parameter :: ccpp_model_const_stdnames(K)` array +(compile-time constant, identical across instances). + +### `ccpp_number_constituents(num_flds, advected, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `num_flds` | `integer, intent(out)` | Constituent count returned. | +| `advected` | `logical, optional, intent(in)` | If `.true.`, count advected only. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%num_constituents(num_flds, advected=advected, ...)`. + +> Even though every `obj(i)` returns the same count (registration is +> identical across instances), `instance_number` is part of the +> signature so the caller can guarantee they're querying an +> already-locked instance. Useful for hosts that lifecycle one +> instance at a time. + +### `ccpp_gather_constituents(const_array, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `const_array` | `real(kind=kind_phys), intent(out) :: (:,:,:)` | Destination buffer for constituent values. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%copy_in(const_array, ...)`. Use this to +pull the per-instance constituent values into a host-side array. + +### `ccpp_update_constituents(const_array, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `const_array` | `real(kind=kind_phys), intent(in) :: (:,:,:)` | Source buffer with updated constituent values. | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%copy_out(const_array, ...)`. Use this to +push host-side updates back into the per-instance constituent object. + +### `ccpp_const_get_index(stdname, const_index, instance_number, errflg, errmsg)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `stdname` | `character(len=*), intent(in)` | Constituent standard name to look up. | +| `const_index` | `integer, intent(out)` | Returned index into the constituent array (or `int_unassigned` on miss). | +| `instance_number` | `integer, intent(in)` | | +| `errflg` / `errmsg` | `intent(out)` | | + +Wraps `obj(instance_number)%const_index(standard_name=stdname, +index=const_index, ...)`. For constituents whose std names are known +at code-gen time, prefer using the module-level `index_of_` integer +directly (no call needed; it's bound during +`ccpp_initialize_constituents`). + +### `ccpp_constituents_array(instance_number) result(const_ptr)` + +Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → +`obj(instance_number)%field_data_ptr()`. + +### `ccpp_advected_constituents_array(instance_number) result(const_ptr)` + +Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → +`obj(instance_number)%advected_constituents_ptr()`. Subset of the +full constituent array containing only those flagged `advected=.true.`. + +### `ccpp_model_const_properties(instance_number) result(const_ptr)` + +Returns `type(ccpp_constituent_prop_ptr_t), pointer :: const_ptr(:)` → +`obj(instance_number)%constituent_props_ptr()`. + +### `ccpp_deallocate_dynamic_constituents(instance_number)` + +| Arg | Direction / Type | Purpose | +|---|---|---| +| `instance_number` | `integer, intent(in)` | | + +**Per-instance reset + last-to-leave teardown**: +1. `obj(instance_number)%reset()` — unlocks the table for this instance. +2. Iterates every `obj(i)` and checks `%const_props_locked()`. If any + instance is still locked, the routine returns. +3. If **every** instance has been reset (none still locked), the routine + tears down the shared state: + - Deallocates every `_dynamic_constituents` buffer. + - Deallocates `ccpp_model_constituents_obj(:)`. + - Resets every `index_of_` integer to 0. + +The host should call this for every instance that successfully called +`ccpp_register_constituents`. + +--- + +## 6. Generated code structure + +When any suite touches constituent state, capgen-ng emits one extra +module per generator run: **`ccpp_host_constituents.F90`**. + +### Module declarations + +```fortran +module ccpp_host_constituents + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: & + ccpp_model_constituents_t, & + ccpp_constituent_properties_t, & + ccpp_constituent_prop_ptr_t + + implicit none + private + + ! ----- public state ---------------------------------------------------- + public :: ccpp_model_constituents_obj + public :: index_of_ ! one per known constituent std name + public :: index_of_ + public :: ccpp_model_const_stdnames ! parameter array + + ! ----- public routines (also re-exported from ccpp_static_api) -------- + public :: ccpp_register_constituents + public :: ccpp_initialize_constituents + public :: ccpp_is_scheme_constituent + public :: ccpp_number_constituents + public :: ccpp_gather_constituents + public :: ccpp_update_constituents + public :: ccpp_const_get_index + public :: ccpp_constituents_array + public :: ccpp_advected_constituents_array + public :: ccpp_model_const_properties + public :: ccpp_deallocate_dynamic_constituents + public :: _dynamic_constituents ! one per suite with register-phase producers + public :: _dynamic_constituents + + ! ----- module-level state --------------------------------------------- + type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) + type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) + type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) + integer :: index_of_ = 0 + integer :: index_of_ = 0 + character(len=N), parameter :: ccpp_model_const_stdnames(K) = (/ & + ' ', & + ' ' /) + +contains + ! ... routines as documented in §5 ... +end module ccpp_host_constituents +``` + +### Suite-cap responsibilities + +`ccpp__cap.F90` does NOT own constituent state. Its +`_register` routine packs each register-phase scheme's +constituent array into the suite's `_dynamic_constituents` +buffer (USE'd from `ccpp_host_constituents`): + +```fortran +if (.not. allocated(_dynamic_constituents)) then + ! First-instance-only two-pass count + populate. + num_consts = 0 + call _register(scheme_consts=scheme_consts, ...) + num_consts = num_consts + size(scheme_consts, 1) + deallocate(scheme_consts) + ... + allocate(_dynamic_constituents(num_consts)) + num_consts = 0 + call _register(scheme_consts=scheme_consts, ...) + do i = 1, size(scheme_consts, 1) + _dynamic_constituents(num_consts + i) = scheme_consts(i) + end do + num_consts = num_consts + size(scheme_consts, 1) + deallocate(scheme_consts) + ... +end if +``` + +The buffer is **shared across instances** (registration is identical +per instance); only the first instance to call `_register` +populates it. The host-wide merge happens in +`ccpp_register_constituents`. + +### Group-cap call sites + +`ccpp___cap.F90` USE's the constituent symbols it needs +from `ccpp_host_constituents`: + +```fortran +use ccpp_host_constituents, only: ccpp_model_constituents_obj, & + index_of_cloud_liquid_water_mixing_ratio +``` + +… and emits scheme call sites with the per-instance access expression: + +```fortran +call cld_liq_run( & + ... + cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer(lb:ub, 1:nlev, & + index_of_cloud_liquid_water_mixing_ratio), & + tend_cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer_tend(lb:ub, 1:nlev, & + index_of_cloud_liquid_water_mixing_ratio), & + ...) +``` + +The `instance_number` dummy is auto-injected into the group-cap +subroutine signatures by `_extra_dim_ctrl_entries` because the +resolver adds `instance_number` to every constituent arg's +`used_dim_std_names`. + +### Framework F90 dependencies + +`ccpp_host_constituents.F90` and the suite caps depend on these +framework files (listed under `` in `datatable.xml`): + +| File | Why | +|---|---| +| `ccpp_constituent_prop_mod.F90` | Provides `ccpp_model_constituents_t`, `ccpp_constituent_properties_t`, `ccpp_constituent_prop_ptr_t`. | +| `ccpp_hashable.F90` | Transitive dep of `ccpp_constituent_prop_mod`. | +| `ccpp_hash_table.F90` | Transitive dep. | +| `ccpp_scheme_utils.F90` | Provides `ccpp_initialize_constituent_ptr` + `ccpp_constituent_index` (used by cam-sima rrtmgp / mmm schemes). | + +The host's CMake should query `ccpp_datafile.py --utility-files` to +get the absolute paths to these files at the right output location. + +--- + +## 7. Multi-instance design + +In capgen-ng, **per-instance state** means: each "instance" (typically +an OpenMP team / chunk-domain partition) has its own copy of the +state arrays, indexed by `instance_number ∈ [1, number_of_instances]`. + +### What's per-instance + +| State | Storage | +|---|---| +| Constituent values + tendencies | `ccpp_model_constituents_obj(:)` — one DDT per instance | +| Suite-level state machine | `ccpp_suite_state(:)` — declared in each suite cap | +| Suite-owned data | `ccpp_suite_data(:)` — declared in each suite-data module | +| Group-level state machine | `ccpp_group_state(:)` — declared in each group cap | + +### What's shared across instances + +| State | Reason | +|---|---| +| `_dynamic_constituents(:)` per-suite buffers | Registration is identical per instance — populated by the first instance to call `_register` and reused by the rest | +| `index_of_` integers | The constituent table is identical per instance, so the indices are too | +| `ccpp_model_const_stdnames` parameter array | Compile-time constant | + +### Sizing + +`number_of_instances` is the single source of truth. The host declares +it in metadata + Fortran; the generator USE's it from the host module +wherever per-instance allocation happens. See the prior memo +[*Where the total number of instances comes from*](#) for the call +chain (and matching values across all four state arrays: +`ccpp_suite_state`, `ccpp_suite_data`, `ccpp_group_state`, +`ccpp_model_constituents_obj`). + +If the host doesn't declare `number_of_instances`, every per-instance +allocation falls back to `1` and the framework runs single-instance. + +### Two host-side lifecycle patterns + +Both work; pick whichever fits your model. + +**Pattern A: all instances registered first** +``` +do isuite = 1, num_suites + do iinst = 1, num_instances + call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) + end do +end do +do iinst = 1, num_instances + call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) + call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) +end do +do isuite = 1, num_suites + do iinst = 1, num_instances + call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) + end do +end do +! ... time-stepping ... +do iinst = 1, num_instances + call ccpp_deallocate_dynamic_constituents(iinst) + ... +end do +``` + +**Pattern B: serial per instance** +``` +do iinst = 1, num_instances + do isuite = 1, num_suites + call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) + end do + call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) + call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) + do isuite = 1, num_suites + call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) + end do + ! ... per-instance time-stepping ... + call ccpp_deallocate_dynamic_constituents(iinst) +end do +``` + +### Last-to-leave teardown + +`ccpp_deallocate_dynamic_constituents(inst)`: +1. Per-instance `obj(inst)%reset()`. +2. Iterates every `obj(i)`; if any has `%const_props_locked() == .true.`, + returns early. +3. Otherwise (every instance reset): deallocates the shared per-suite + buffers, deallocates `ccpp_model_constituents_obj(:)`, and zeros every + `index_of_` integer. + +This works for both lifecycle patterns above. + +--- + +## 8. Limitations and gotchas + +> **Note (2026-05-12).** Several items in this section are under active +> discussion for an upcoming framework + generator overhaul. See +> `doc/constituents_overhaul.md` for the full architectural review and +> three reform proposals. + +### Framework property ownership (2026-05-12) + +The framework's `ccpp_constituent_properties_t` now carries a private +`framework_owns_me` flag (default `.false.`) with +`is_framework_owned()` getter and `set_framework_owned(value)` setter. +`ccpt_deallocate` only deallocates the underlying prop when the flag +is `.true.`; otherwise it just nullifies its pointer. + +Under capgen-ng's explicit-registration model, all +`ccpp_constituent_properties_t` objects are **target-owned by the +caller** (the host's `host_constituents(:)` array, or the per-suite +`_dynamic_constituents(:)` buffer). We never set the flag, so +the framework correctly skips deallocation. Hosts that hand-allocate +property objects on the heap and want the framework to free them must +call `set_framework_owned(.true.)` before passing to `%new_field`. + +### Missing setters (framework gap) + +The framework lacks setters for `advected`, `diagnostic_name`, +`default_value` (and `mixing_ratio_type`). This means once a +constituent is `%instantiate`d, those properties cannot be changed. +If your host needs to override a scheme-supplied `diagnostic_name` or +`advected` value, you currently cannot — open item in the constituents +overhaul proposal. + +### `ccpp_scheme_utils` singleton + +`ccpp_scheme_utils.ccpp_initialize_constituent_ptr` accepts only one +singleton pointer. It's a framework-level convenience used by cam-sima +schemes that call `ccpp_constituent_index` from `ccpp_scheme_utils`. + +`ccpp_initialize_constituents` calls +`ccpp_initialize_constituent_ptr(obj(inst))` on each instance, but +**only the first call across instances actually sets the pointer** +(the routine is internally guarded by an `initialized` flag). +Subsequent calls are silent no-ops. + +For multi-instance hosts, schemes that use +`ccpp_scheme_utils.ccpp_constituent_index` will see only the first +instance's object — a known limitation inherited from the framework +module's design. Schemes that use the per-instance accessors +(`obj(inst)%const_index(...)` via `ccpp_const_get_index`) are +unaffected. + +### Constituent metadata is identical across instances + +The constituent table (which constituents exist, their properties, the +`index_of_` mapping) is **identical** for every instance. Every +instance's `obj(i)` has the same hash table, populated identically by +its own `ccpp_register_constituents` call. + +This means: + +- `ccpp_number_constituents` returns the same value regardless of + `instance_number`. +- `ccpp_const_get_index` returns the same index regardless of + `instance_number`. +- The `index_of_` integers are populated identically by every + instance's `ccpp_initialize_constituents` (last-write-wins is fine + since every write is the same value). + +`instance_number` is still in the signatures of these routines — see +[§5](#5-public-api-reference) for the rationale. + +### Forbidden patterns recap + +These are rejected at code-gen time (Rule 4 of [§2](#2-the-four-rules-scheme-author-conventions)): + +- `is_constituent + intent=out + non-tendency std_name` — physics phases + may only produce tendencies, not new base constituents. +- `is_constituent + intent=in/inout + tendency_of_*` — tendencies are + write-only. + +### Subscript indices in sliced local_names must be standard names + +If a host metadata variable is declared with a sliced local name +like `q(:,:,index_of_water_vapor_specific_humidity)`, every subscript +token (other than `:` and integer literals) must be a known standard +name. Otherwise the resolver raises a `CCPPError` with a clear +message naming the offending token. + +### Open work items + +- **Unconditional `ccpp_host_constituents.F90` emission.** The + generator currently emits `ccpp_host_constituents.F90` for every + build, even when no scheme or host actually uses the constituent + system (no `ccpp_constituent_properties_t(:)` register-phase arg, + no `is_constituent`-flagged scheme arg, no framework-named + `index_of_` / `ccpp_constituents` / etc. claimed by capgen-ng). + When the host owns its own indices (SCM/GFS) and no scheme exercises + the constituent path, the generated file is dead code that should be + suppressed. Tracked as a deferred item; the `host_dict` precedence + rule above already keeps the file *correct* (empty) in that case. + +--- + +## 9. Differences from original capgen + +| Aspect | Original capgen | capgen-ng | +|---|---|---| +| Constituent object location | Generated `_ccpp_cap.F90` module | `ccpp_host_constituents.F90` (one per generator run) | +| Per-instance | No (single instance) | Yes (`obj(:)` allocatable, sized to `number_of_instances`) | +| Routine name prefix | `_ccpp_register_constituents`, etc. | `ccpp_register_constituents`, etc. (no host prefix; one set per generator run) | +| Routine signatures | No `instance_number` arg | `instance_number` in every per-instance routine | +| Host metadata for constituent obj | None (auto-created by generator) | None (still auto-created by generator) | +| Module-level pointers | `_constituents_array` etc. as functions returning pointers | Same idea, now per-instance via `instance_number` arg | +| Scheme std-name set | `_model_const_stdnames` parameter array | `ccpp_model_const_stdnames` parameter array (no host prefix) | +| Host-facing API surface | `_ccpp_register_constituents`, `_ccpp_initialize_constituents`, `_ccpp_number_constituents`, `_ccpp_is_scheme_constituent`, `_ccpp_gather_constituents`, `_ccpp_update_constituents`, `_ccpp_deallocate_dynamic_constituents`, `_constituents_array`, `_advected_constituents_array`, `_model_const_properties`, `_const_get_index` | Same surface, no `_` prefix | +| Dynamic constituent buffer dimensionality | 1D, per host | 1D, per generator run, **shared across instances** | +| Static suite constituents | Auto-cloned by `ConstituentVarDict.find_variable` and registered via `_constituents_copy_const` accessors | Tracked at code-gen time via `is_constituent` flag; included in the constituent table only if a register-phase scheme produces them (rule 1). Schemes that *consume* a base constituent (rule 2) don't trigger registration — the constituent must be registered by SOMEONE (host or another scheme's register) for the access to work at runtime. | + +### Migration notes for cam-sima hosts + +- **Scheme metadata**: no changes needed. Cam-sima follows rules 2 and + 3 already (audited 2026-05-11). The 4 schemes that register + constituents via `ccpp_constituent_properties_t` (rule 1) work + unchanged. +- **Host metadata**: drop any explicit declaration of + `ccpp_model_constituents_object` if you carried one over from a + previous capgen-ng experiment — the generator owns it now. +- **Host Fortran**: change all `_ccpp_*_constituents` calls to + the unprefixed names (`ccpp_register_constituents` etc.) and add + `instance_number` to every call site. + +--- + +## 10. Worked example + +A minimal cam-sima-style suite with one scheme that consumes a base +constituent and produces its tendency. + +### Scheme metadata (`consume_constituent.meta`) + +``` +[ccpp-table-properties] + name = consume_constituent + type = scheme + +[ccpp-arg-table] + name = consume_constituent_run + type = scheme +[ cldliq ] + standard_name = cloud_liquid_water_mixing_ratio + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in + advected = .true. +[ tend_cldliq ] + standard_name = tendency_of_cloud_liquid_water_mixing_ratio + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = .true. +[ errmsg ] + ... +[ errflg ] + ... +``` + +### Host metadata (`my_host.meta`) + +``` +[ccpp-table-properties] + name = my_host + type = host + +[ccpp-arg-table] + name = my_host + type = host +[ ncols ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer +[ nlev ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer +[ ninstances ] + standard_name = number_of_instances + units = count + dimensions = () + type = integer +``` + +(Plus a `type=control` table declaring `instance_number`, +`horizontal_loop_begin`, `horizontal_loop_end`, `ccpp_error_message`, +`ccpp_error_code`, etc.) + +### Suite XML (`my_suite.xml`) + +```xml + + + + consume_constituent + + +``` + +### Generated `ccpp_host_constituents.F90` (excerpt) + +```fortran +module ccpp_host_constituents + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: & + ccpp_model_constituents_t, ccpp_constituent_properties_t, & + ccpp_constituent_prop_ptr_t + + implicit none + private + + public :: ccpp_model_constituents_obj + public :: index_of_cloud_liquid_water_mixing_ratio + public :: ccpp_register_constituents, ccpp_initialize_constituents + public :: ccpp_is_scheme_constituent, ccpp_number_constituents + public :: ccpp_gather_constituents, ccpp_update_constituents + public :: ccpp_const_get_index, ccpp_constituents_array + public :: ccpp_advected_constituents_array, ccpp_model_const_properties + public :: ccpp_deallocate_dynamic_constituents + public :: ccpp_model_const_stdnames + + type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) + integer :: index_of_cloud_liquid_water_mixing_ratio = 0 + character(len=31), parameter :: ccpp_model_const_stdnames(1) = (/ & + 'cloud_liquid_water_mixing_ratio' /) + +contains + ! ... full subroutine bodies as in §5 ... +end module ccpp_host_constituents +``` + +### Host code skeleton (single-instance illustration) + +```fortran +subroutine my_host_run() + use ccpp_static_api, only: ccpp_register, ccpp_register_constituents, & + ccpp_initialize_constituents, ccpp_init, & + ccpp_physics_run, ccpp_final, & + ccpp_deallocate_dynamic_constituents + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + type(ccpp_constituent_properties_t), allocatable :: host_consts(:) + integer :: errflg + character(len=512) :: errmsg + integer, parameter :: inst = 1 + + ! 1. Run register phase: populates per-suite dynamic-constituent buffers. + call ccpp_register('my_suite', errmsg, errflg, inst) + + ! 2. Build host's own constituent declarations (water vapor, etc.). + allocate(host_consts(1)) + call host_consts(1)%instantiate( & + std_name='water_vapor_specific_humidity', long_name='water vapor', & + units='kg kg-1', vertical_dim='vertical_layer_dimension', & + advected=.true., errcode=errflg, errmsg=errmsg) + + ! 3. Merge host + suite-side constituents into obj(inst). + call ccpp_register_constituents(host_consts, inst, errflg, errmsg) + + ! 4. Allocate vars_layer + bind cached indices. + call ccpp_initialize_constituents(ncols, nlev, inst, errflg, errmsg) + + ! 5. Framework init phase. + call ccpp_init('my_suite', errmsg, errflg, inst) + + ! 6. Time-stepping (omitted). + call ccpp_physics_run('my_suite', 'phys', col_start, col_end, & + thread_num, nthreads, nphys_threads, & + errflg, errmsg, inst) + + ! 7. Shutdown. + call ccpp_final('my_suite', errmsg, errflg, inst) + call ccpp_deallocate_dynamic_constituents(inst) + deallocate(host_consts) +end subroutine my_host_run +``` + +For multi-instance, wrap each per-instance call in +`do iinst = 1, ninstances ... end do` per the patterns in +[§7](#7-multi-instance-design). diff --git a/doc/constituents_overhaul_20260519T0905.md b/doc/constituents_overhaul_20260519T0905.md new file mode 100644 index 00000000..2bf6fad8 --- /dev/null +++ b/doc/constituents_overhaul_20260519T0905.md @@ -0,0 +1,947 @@ +# CCPP Constituents — Architecture Review & Overhaul Discussion + +**Authors:** Dom Heinzeller (lead), Claude (assistant) +**Date drafted:** 2026-05-12 +**Last revised:** 2026-05-18 +**Intended audience:** CCPP framework team, CAM-SIMA team +**Status:** Discussion document — no decisions are final. Proposals +A/B/C below remain pending the upcoming meeting; the bug fix from +Proposal A (the `ccpt_deallocate` ownership flag) and the capgen-ng +internal cleanup from Proposal B (§4.8) have landed; the missing +setters from Proposal A and the `is_match` relaxation from Proposal B +have not. Independent of A/B/C, the per-suite dynamic_constituents +buffer was made per-instance on 2026-05-18 to fix a multi-instance +mutation conflict — see §4.13. + +--- + +## Executive summary + +CCPP's "constituent" mechanism — how schemes declare and how the framework +manages tracer species like water vapor, cloud liquid, prescribed ozone, +etc. — has grown organically over the last few years. The result works, +but it carries: + +- **A latent framework bug** in `ccpp_constituent_prop_mod` that crashes on + teardown of explicitly-registered (target-passed) constituent property + arrays. Fixed in capgen-ng's framework copy 2026-05-12; needs to land + upstream. +- **Architectural confusion** about which properties are *physics-portable* + (the scheme owns them) versus *host-configuration* (the host owns them). + Today schemes are forced to supply host-specific values (`diag_name` is + the worst offender) at `%instantiate` time. +- **Setter API gaps**: properties that the host wants to override after + scheme-side registration (`advected`, `diagnostic_name`, `default_value`) + have no setters; `is_match` is overly strict about properties hosts + should be free to change. +- **Two registration models** coexist — original capgen's auto-clone of + is_constituent scheme args, and capgen's/capgen-ng's explicit register-phase + + host-side declaration. Capgen-ng deliberately dropped auto-clone. + +This document is a structured brief for a discussion this week. It does +NOT pre-commit to any decision; it lays out what exists, what's broken, +what we audited, and what proposals are on the table. + +--- + +## Table of contents + +1. [How original capgen handles constituents](#1-how-original-capgen-handles-constituents) +2. [How capgen-ng handles constituents](#2-how-capgen-ng-handles-constituents) +3. [What CAM-SIMA actually needs (audit)](#3-what-cam-sima-actually-needs-audit) +4. [Bugs and design flaws](#4-bugs-and-design-flaws) +5. [Property classification (Class A vs Class B)](#5-property-classification-class-a-vs-class-b) +6. [What to remove, replace, improve](#6-what-to-remove-replace-improve) +7. [Open design questions](#7-open-design-questions) +8. [Three proposals — minimal / clean / deep](#8-three-proposals--minimal--clean--deep) +9. [Appendix: framework setter inventory](#9-appendix-framework-setter-inventory) + +--- + +## 1. How original capgen handles constituents + +### 1.1 Mental model + +Original capgen treats constituents as a **separate scope** between +suite and host: + +``` +group → suite → ConstituentVarDict → host +``` + +A scheme arg flagged `constituent = True` in metadata is matched first +against group/suite/ConstituentVarDict, and only against host as a last +resort. The ConstituentVarDict is a synthetic dictionary whose entries +are auto-created by `find_variable()` when a scheme metadata declares a +constituent dependency. + +### 1.2 Auto-clone of `is_constituent` scheme args + +Every scheme arg with non-default `advected`, `constituent`, or +`molar_mass` is treated as a *registration*. The generator emits, into +the host cap, a routine `_constituents_ccpp_create_constituent_array` +that: + +1. Allocates a `ccpp_constituent_properties_t` pointer per scheme arg. +2. Calls `%instantiate(...)` populating fields **from the scheme + metadata directly** — `std_name`, `long_name`, `diagnostic_name`, + `units`, `default_value`, `advected`, `vertical_dim`, etc. (See + `scripts/constituents.py:565`.) +3. Adds it to the model constituents object via `%new_field`. + +After this auto-clone runs, the host's hand-written +`host_constituents(:)` array is appended, then `%lock_table` finalizes +the hash table. + +### 1.3 The host-cap-owned `ccpp_model_constituents_obj` + +Original capgen generates **one** `ccpp_model_constituents_obj` per +generator invocation, declared module-level in `_ccpp_cap.F90`. +Single global; not per-instance. (CAM-SIMA runs one host per +executable, so single-instance is fine for them.) + +### 1.4 Scheme-side `%instantiate` registration (the other path) + +A scheme may also register constituents via a register-phase argument: + +```fortran +type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) +``` + +The scheme allocates the array, calls `%instantiate` per entry, and +returns it. Original capgen wires this through a per-suite +"dynamic constituents" buffer and merges it during host-cap setup, +alongside the auto-cloned set. + +So original capgen really supports **three** registration sources: + +- Host: hand-written `host_constituents(:)` arg. +- Suite-dynamic: register-phase scheme args. +- Suite-static: auto-cloned from any `is_constituent` consumer. + +All three flow into one `%new_field` table. + +### 1.5 Lifecycle + +- `_ccpp_register_constituents(host_constituents, ...)` runs the + three-source merge. +- `ccpp_initialize_constituent_ptr(const_obj, ...)` (in + `ccpp_scheme_utils`) caches a pointer for the `ccpp_constituent_index` + lookup. +- Phase entry points access `vars_layer` / `vars_layer_tend` via cached + `index_of_` integers. + +### 1.6 What's good about original capgen's approach + +- Schemes declare a constituent dependency once in metadata; no manual + Fortran registration ever needed for "static" tracers. +- Host doesn't have to enumerate every species every scheme wants. +- Works for CAM-SIMA's current scheme catalog. + +### 1.7 What's painful about original capgen's approach + +- The auto-clone path is **invisible** to anyone reading the scheme + Fortran — the registration happens in generated code. +- `ConstituentVarDict` is a synthetic scope, conceptually subtle, and + doesn't generalize cleanly to multi-instance. +- The auto-clone path lifts `diagnostic_name` and `default_value` from + scheme metadata, but those values are often host-specific (see §4.4). +- Three sources of registration with overlap mean two registrations of + the same `std_name` may collide; original capgen relies on + `is_match` (units, advected, thermo_active, water_species) to dedup, + which means schemes accidentally diverge on `advected` and trip the + "incompatible constituent" error. + +--- + +## 2. How capgen-ng handles constituents + +### 2.1 Mental model + +No synthetic scope. Constituents are *one of four* sources for any +scheme arg: + +``` +control | host | suite | constituent +``` + +The resolver classifies each scheme arg into exactly one source. A +`constituent` source means the value will be accessed at runtime as +`ccpp_model_constituents_obj()%vars_layer(:, :, index_of_)` +(or `%vars_layer_tend(...)` for `tendency_of_` outputs). + +### 2.2 The four scheme-author rules + +(See `doc/constituents.md` for full details; this is the summary.) + +1. **Register** — register-phase scheme args of type + `ccpp_constituent_properties_t(:), intent=out, allocatable` declare + new constituents the scheme contributes. +2. **Consume** — physics-phase scheme args with `advected=true` (or + `molar_mass=...` or `constituent=true`) and `intent=in/inout`, with + the constituent's standard name, read the base species. +3. **Produce a tendency** — physics-phase scheme args with + `constituent=true`, `intent=out`, and standard name + `tendency_of_`, write the tendency. +4. **Mismatched combinations are errors** — `intent=out` on a base + constituent, or `intent=in` on a tendency, are codegen-time errors. + +### 2.3 Two registration sources (no auto-clone) + +- **Host**: hand-written `host_constituents(:)`, passed into + `ccpp_register_constituents(host_constituents, instance_number, ...)`. +- **Suite-dynamic**: register-phase scheme args, accumulated into a + per-suite buffer `_dynamic_constituents(:)` by `_register`, + drained into `ccpp_model_constituents_obj(inst)` by + `ccpp_register_constituents`. + +The auto-clone-from-metadata path is deliberately **gone**. If a scheme +declares `advected=true` on an arg but no source registers that +standard name, capgen-ng now emits a runtime check during +`ccpp_initialize_constituents` that errors with the missing name. + +### 2.4 Per-instance state + +Everything is per-instance: + +```fortran +type(ccpp_model_constituents_t), allocatable :: ccpp_model_constituents_obj(:) + ! indexed by instance_number +``` + +All host-facing entry points take `instance_number`: + +``` +ccpp_register_constituents (host_constituents, instance_number, errflg, errmsg) +ccpp_initialize_constituents (ncols, num_layers, instance_number, errflg, errmsg) +ccpp_number_constituents (num_flds, advected, instance_number, errflg, errmsg) +ccpp_gather_constituents (const_array, instance_number, errflg, errmsg) +ccpp_update_constituents (const_array, instance_number, errflg, errmsg) +ccpp_const_get_index (stdname, const_index, instance_number, errflg, errmsg) +ccpp_constituents_array (instance_number) => pointer +ccpp_advected_constituents_array (instance_number) => pointer +ccpp_model_const_properties (instance_number) => pointer +ccpp_deallocate_dynamic_constituents (instance_number, ...) +``` + +`ccpp_is_scheme_constituent(var_name, ...)` and the +`ccpp_model_const_stdnames(:)` parameter array are NOT per-instance — +the standard-name catalog is identical across instances. + +### 2.5 Lifecycle + +``` +ccpp_register(suite_name, instance_number, ...) + └─ _register → packs scheme-dynamic constituents into + _dynamic_constituents(instance)%items + (per-instance wrapper-DDT array; each instance + allocates and fills its own slot — see §4.13) + ↓ +ccpp_register_constituents(host_constituents, instance_number, ...) + └─ initialize_table(num_host_consts + num_suite_consts) + └─ new_field(host_consts ...) + └─ new_field(_dynamic_constituents ...) + └─ lock_table + ↓ +ccpp_initialize_constituents(ncols, num_layers, instance_number, ...) + └─ lock_data (allocates vars_layer, vars_layer_tend, vars_minvalue) + └─ ccpp_initialize_constituent_ptr (first instance wins; documented limit) + └─ %const_index('') for each enumerated constituent + └─ post-lookup int_unassigned check → clear error message + ↓ +ccpp_init(suite_name, instance_number, ...) + └─ _init → binds module-level pointers + ↓ +... physics phases ... + ↓ +ccpp_final(suite_name, instance_number, ...) + └─ _final → nullifies + last-to-leave deallocates + ↓ +ccpp_deallocate_dynamic_constituents(instance_number, ...) + └─ ccp_model_constituents_obj(inst)%reset + ↓ (in _final, last-to-leave) + deallocate(_dynamic_constituents) +``` + +### 2.6 What's good + +- Explicit. Every constituent registration is visible in someone's + Fortran source. +- Multi-instance from day one. +- The "four rules" are small enough to fit on a slide. +- Resolver-time + codegen-time + runtime checks catch the most common + mistakes. + +### 2.7 What's still painful + +Covered in §4. + +--- + +## 3. What CAM-SIMA actually needs (audit) + +### 3.1 Scheme-side registration usage + +We audited `EXT/cam-sima/atmospheric_physics/schemes/` for use of the +register-phase `ccpp_constituent_properties_t(:)` pattern: + +| Scheme | File | Registers | +|---|---|---| +| RRTMGP constituents | `schemes/rrtmgp/rrtmgp_constituents.meta` | radiative-active species | +| MUSICA chemistry | `schemes/musica/musica_ccpp.meta` | chemical species from MUSICA | +| Prescribed aerosols | `schemes/chemistry/prescribed_aerosols.meta` | aerosol species | +| Prescribed ozone | `schemes/chemistry/prescribed_ozone.meta` | ozone | + +**Total: 4 of 128 schemes** in the atmospheric_physics tree use +scheme-side registration. The other 124 only **consume** constituents +(`advected=true` + `intent=in/inout` in metadata, accessed via the +framework's `vars_layer`). + +This is a small enough number that an alternative "host-only +registration" model is feasible: move those 4 register calls into the +host (or into helper modules the host calls), and the rest of the +catalog only consumes. + +### 3.2 Host-side patterns + +`EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` wraps +the framework setters and exposes: + +- `const_set_thermo_active(const_obj | const_ind, value)` +- `const_set_water_species(const_obj | const_ind, value)` +- `const_set_minimum(...)` + +CAM-SIMA actively **calls these setters at runtime** — schemes don't +supply `thermo_active` at instantiate time; the host configures it +afterwards. This is direct evidence that the "post-instantiation +override" pattern is real and used today, and that the framework's +setter API is load-bearing. + +### 3.3 What CAM-SIMA does **not** do + +- It does not rely on auto-clone for `diag_name`. The scheme-side + register calls in the 4 schemes do supply `diag_name`, but those + values are CAM-SIMA's; a different host would need different ones. +- It does not use `ccpp_constituent_index` (the + `ccpp_scheme_utils`-singleton-based lookup) extensively — most + access goes through the framework's `index_of_` integers. + +### 3.4 What CAM-SIMA's host-cap-owned constituent object looks like + +Because original capgen generates **one** `ccpp_model_constituents_obj` +per generator invocation, and CAM-SIMA uses one generator invocation per +executable, CAM-SIMA effectively runs single-instance today. A +multi-instance CAM-SIMA (sub-columns, ensembles) would expose the +single-global limitation immediately. + +--- + +## 4. Bugs and design flaws + +This section lists known issues across the three layers (framework, +original capgen, capgen-ng). Items marked **(FIXED)** were resolved +2026-05-12 and either are or will be PRs; items marked **(OPEN)** are +intentionally left for this discussion. + +### 4.1 Framework: `ccpt_deallocate` ownership bug (FIXED in capgen-ng tree, needs upstream PR) + +- **Location**: `src/ccpp_constituent_prop_mod.F90`, `ccpt_deallocate` + + `ccpt_set`. +- **Symptom**: `free(): invalid size` crash when + `ccp_model_const_reset` is called on a properly-locked table whose + entries came from pointer-assigned targets (the common pattern + under capgen-ng's explicit registration; also potentially under + original capgen's `host_constituents` path). +- **Root cause**: `ccpt_set` does pointer assignment (`this%prop => + const_ptr`); `ccpt_deallocate` does an unconditional + `deallocate(this%prop)`. The deallocate is correct only when the + caller allocated `const_ptr` on the heap and transferred ownership. +- **Why it didn't surface earlier**: original capgen's advection test + only calls `deallocate` once between a *failing* register and a + *successful* one — at that point `lock_table` has not populated + `const_metadata`, so the broken inner loop is skipped. Capgen-ng + triggers it because its teardown calls `reset` after a successful + lock. +- **Fix landed 2026-05-12**: added `framework_owns_me` private flag on + `ccpp_constituent_properties_t` (default `.false.`) with + `is_framework_owned()` getter and `set_framework_owned(value)` + setter; `ccpt_deallocate` now only deallocates when the flag is set. + Original capgen's auto-clone path in `scripts/constituents.py` + updated to call `set_framework_owned(.true.)` after `allocate`. + Diffs in `src/ccpp_constituent_prop_mod.F90` (and capgen-ng's + parallel copy) + `scripts/constituents.py`. +- **Status**: framework tests pass; capgen-ng unit-test suite (1127 passing + as of 2026-05-13) is green. Still needs upstream PR to ccpp-framework + + original ccpp-capgen. + +### 4.2 Framework: missing setters (OPEN) + +| Property | Optional in `%instantiate`? | Has setter? | `is_match`-checked? | +|---|---|---|---| +| `std_name` | required | — | (lookup key) | +| `long_name` | required | — | no | +| `diag_name` | required | **NO** | no | +| `units` | required | — | **yes** | +| `vertical_dim` | required | — | no | +| `advected` | optional (default .false.) | **NO** | **yes** | +| `default_value` | optional | **NO** | no | +| `min_value` | optional | `set_minimum` | no | +| `molar_mass` | optional | `set_molar_mass` | no | +| `water_species` | optional (default .false.) | `set_water_species` | **yes** | +| `mixing_ratio_type`| optional | **NO** | no | +| `thermo_active` | not in instantiate | `set_thermo_active` | **yes** | +| `const_index` | internal | `set_const_index` | no | + +**Pain points**: + +- `advected` is `is_match`-checked AND has no setter. Once registered, + immutable. If a scheme and the host disagree, you get the + "incompatible constituent" error and you cannot reconcile from + Fortran. +- `diag_name` is required (cannot be omitted at instantiate) AND has + no setter. A scheme must pick a value at registration time; that + value is then frozen. +- `default_value` is silently optional. If omitted, the constituent + array initializes to `huge(real)` and downstream comparisons fail + in surprising ways (we burnt half a day on this 2026-05-12). +- `thermo_active` is the only property in the "post-instantiate-only" + shape: it has a setter but isn't a `%instantiate` arg. The + asymmetry is confusing. + +### 4.3 Framework: `is_match` is too strict (OPEN) + +`is_match` (in `ccp_is_match`) checks `units`, `advected`, +`thermo_active`, `water_species`. Three of those four (`advected`, +`thermo_active`, `water_species`) are properties the host legitimately +overrides post-registration. Two registrations of the same `std_name` +with the same `units` but different `advected` should be a +duplicate-dedup (host wins), not a hard error. + +### 4.4 Framework: `diag_name` portability problem (OPEN) + +Diagnostic output names are host-specific. CAM-SIMA names cloud +liquid mixing ratio `CLDLIQ`; UFS would call it something else. Yet +`%instantiate` makes `diag_name` a *required* arg, forcing schemes to +either: + +- Pick a host-specific value (couples the scheme to a host), or +- Pick a "neutral" default that no host's diagnostic tooling + recognizes. + +The current de-facto pattern in CAM-SIMA scheme code is to pick a +CAM-SIMA-flavoured value and ship it. Any port to UFS would need to +either monkey-patch or fork the scheme. + +A clean fix: +1. Make `diag_name` optional at `%instantiate` (default to empty + string or `std_name`). +2. Add `set_diagnostic_name(value)` setter. +3. Host overrides per-registration after `ccpp_register_constituents`. + +### 4.5 Original capgen: implicit registration (OPEN — observation) + +The auto-clone path is generator magic. Reading scheme metadata +doesn't tell you whether the scheme's args result in registration; you +have to know that `advected=true` triggers it. This is a documentation ++ comprehension problem more than a bug. + +### 4.6 Original capgen: single-instance `ccpp_model_constituents_obj` (OPEN — limitation) + +The host cap declares one global. Multi-instance hosts would need to +either generate one cap per instance or restructure. + +### 4.7 Original capgen: `ConstituentVarDict` complexity (OPEN — observation) + +The synthetic scope between suite and host serves correctness but +adds a code path that most contributors don't read. If we drop it +(capgen-ng has), the variable-matching algorithm shrinks. + +### 4.8 Capgen-ng: `_FRAMEWORK_CONST_DIM_INPUTS` cleanup (LANDED 2026-05-13) + +`generator/static_api.py` no longer carries the hand-curated frozenset of +standard names; framework-constituent dimension references now ride on a +dedicated `used_const_dim_std_names` field on `ResolvedArg`. Closes the +"hand-curated → structured field" REVISIT note that was in the code. + +### 4.9 Capgen-ng: no codegen-time cross-check of scheme registration (OPEN) + +The resolver knows every `is_constituent` arg's standard name (in +`SuiteResolution.constituent_index_names`) but doesn't know what each +scheme's `_register` subroutine actually `%instantiate`s. Today's +guarantee is a runtime check (the `int_unassigned` validation we +added 2026-05-12). Stronger options: + +- (a) New metadata attribute `registers_std_names = a, b, c` on + register-phase tables; codegen errors at generation time. +- (b) Parse scheme `_register` Fortran for `%instantiate(std_name=…)` + calls and cross-check. +- (c) Keep runtime check as authoritative, document the gap. + +### 4.10 Capgen-ng: scheme-metadata `diagnostic_name` for is_constituent args is host-specific (OPEN) + +Same issue as §4.4 but in capgen-ng's metadata layer. Today's +`diagnostic_name` attribute on a scheme metadata arg flows into +`datatable.xml` and is then trusted as "the" diagnostic name. If we +adopt setter-based class-B overrides, this attribute should either be +dropped for constituent args or marked as a default-only hint. + +### 4.11 Capgen-ng: `ccpp_scheme_utils` singleton (OPEN — documented limit) + +`ccpp_initialize_constituent_ptr(const_obj)` stores a single module-level +pointer. Schemes that use `ccpp_constituent_index(stdname)` get that +pointer back. Under multi-instance, only the first instance's +pointer is retained — `ccpp_constituent_index` queries from +within a scheme will always reflect instance 1. CAM-SIMA's 4 +scheme-registering schemes don't rely on this; documented in +`doc/constituents.md` §8. Real fix requires either threading +`instance_number` through `ccpp_constituent_index` (interface +change) or maintaining a per-instance pointer table. + +### 4.12 Capgen-ng: drop `diagnostic_name_fixed`, keep only `diagnostic_name` (OPEN — proposed simplification) + +Today the metadata layer carries two mutually-exclusive scheme-arg +attributes: + +- `diagnostic_name = X` — emits `diagnostic_name="X"` in `datatable.xml`; + defaults to `local_name` when absent. +- `diagnostic_name_fixed = Y` — emits `diagnostic_name_fixed="Y"` in + `datatable.xml`; the `diagnostic_name` slot stays empty (no + auto-default to `local_name`). + +The behavioral difference is purely *which attribute name* host +tooling sees in `datatable.xml` — both attributes carry the same kind +of value (a Fortran-identifier-shaped string), and both are passed +through unmodified. `_fixed` is a signal to the host "use verbatim, do +not decorate or transform"; but `diagnostic_name = X` already means +exactly that — the cap code never decorates the value, and any host +tooling that wants to decorate would have to opt in by parsing a +separate attribute (or by syntactic convention on the value itself). + +**Proposal:** Remove `diagnostic_name_fixed` from the metadata layer +and the parser. Keep `diagnostic_name` with the existing defaulting +rule (explicit → use it; absent → fall back to `local_name`). Hosts +that today rely on the `_fixed` semantic ("don't auto-default to +`local_name`") get the same outcome by simply *setting* +`diagnostic_name` to the desired exact value. + +Touchpoints to retire: + +- `metadata/parse_tools/parse_checkers.py::check_diagnostic_fixed` and + the mutual-exclusion block at the top of `check_diagnostic_id`. +- `metadata_table.py::MetaVar._KNOWN_ATTRS` entry and the + `@property diagnostic_name` fallback that returns `''` when + `_diagnostic_name_fixed` is set. +- `generator/datatable.py:267-269` emission of the + `diagnostic_name_fixed` XML attribute. +- Existing unit-test coverage for `diagnostic_name_fixed` becomes + obsolete and is removed (not migrated). + +**Why it's worth doing as part of the overhaul:** the attribute has no +unique semantics that `diagnostic_name` can't express, and dropping it +shrinks the metadata-layer surface area at the same time the +`set_diagnostic_name(value)` framework setter (§4.4 / §4.10) is being +added on the framework side. Hosts that want runtime override get +`set_diagnostic_name`; hosts that want metadata-declared values get +`diagnostic_name`. There is no third use case that needs `_fixed`. + +**Risk:** non-CCPP-ng metadata in the wild may carry +`diagnostic_name_fixed`. Mitigation: a one-line legacy-mode rewrite +(`metadata/legacy_compat.py`) translates the deprecated attribute to +`diagnostic_name` at parse time with a loud warning, identical in +spirit to the existing `horizontal_loop_extent → horizontal_dimension` +shim. Remove the rewrite once known consumers are migrated. + +### 4.13 Capgen-ng: per-suite `dynamic_constituents` buffer was shared across instances (FIXED 2026-05-18) + +- **Location**: `capgen-ng/generator/host_constituents.py` (buffer + declaration + `ccpp_register_constituents` iteration); + `capgen-ng/generator/suite_cap.py::_register_lines` (the two-pass + count→allocate→pack inside `_register`). +- **Symptom**: with two or more instances and any register-phase + scheme that produces constituents, the second per-instance + `ccpp_register_constituents` call fails with `ccp_set_const_index + ccpp_constituent_properties_t const index is already set`. +- **Root cause**: the per-suite buffer + `_dynamic_constituents(:)` was declared as a single shared + 1-D array of `ccpp_constituent_properties_t`, filled exactly once on + first instance entry (`.not. allocated(buf)` gate). + `ccpp_register_constituents` then iterates that shared buffer per + instance and calls `%new_field(const_prop)` on each property + object. `%new_field` calls `ccp_set_const_index`, which **mutates + the property object** by writing `const_ind`. Instance 1 set + `const_ind` on every shared object; instance 2's call tripped the + "set exactly once" guard. +- **Latent companion bug**: the same shared-mutation pattern means + that once Proposal B's class-B setters (`set_advected`, + `set_diagnostic_name`, `set_water_species` per-instance, etc.) are + exercised, instance 1's setter call would silently corrupt instance + 2's view of the property. No "already set" guard exists on those + setters today. +- **Why it didn't surface earlier**: the advection end-to-end test is + single-instance; the instances end-to-end test has no constituents. + Surfaced by the new `instances_advection` combined test + (`end-to-end-tests/instances_advection/`) on first run. +- **Fix landed 2026-05-18**: the per-suite buffer is now a wrapper-DDT + array indexed by `instance_number`: + ```fortran + type :: ccpp_dyn_const_buffer_t + type(ccpp_constituent_properties_t), allocatable :: items(:) + end type + type(ccpp_dyn_const_buffer_t), allocatable, target :: _dynamic_constituents(:) + ``` + The outer array is allocated to `number_of_instances` on first call; + each instance independently runs the two-pass count+pack into its + own `%items` slot. `ccpp_register_constituents` iterates + `_dynamic_constituents(instance)%items` so each instance's + `new_field` calls operate on **distinct** property objects. + Scheme `_register` routines are now called N times instead of once + (negligible cost — typical register bodies are a few `%instantiate` + calls), in exchange for clean per-instance isolation. +- **Cost**: ~50 lines across the two generator emitters, plus updates + to six pinned unit tests. No CAM-SIMA / NEPTUNE / SCM coordination + needed (host-facing API unchanged). +- **Status**: framework tests pass; full unit-test suite (1319 tests) + is green; all 10 end-to-end tests pass. +- **Position relative to Proposals A/B/C**: orthogonal — none of the + three proposed touching the buffer. Independently adopted. + +--- + +## 5. Property classification (Class A vs Class B) + +Proposed in `design_constituents_mutability.md` 2026-05-12. Each +constituent property is conceptually owned by either the scheme +(physics-portable, immutable once instantiated) or the host +(host-configuration, mutable post-instantiation). + +### Class A — scheme-intrinsic (immutable) + +| Property | Why class A | +|---|---| +| `std_name` | Identity. Cannot change. | +| `long_name` | Human-readable name of the *species*. Not host-specific. | +| `units` | Physics correctness. `is_match`-checked. | +| `vertical_dim` | Scheme's structural expectation (interface vs layer). | +| `molar_mass` | Physical constant of the species. | +| `default_value` | (Debatable — see §7) Scheme-appropriate initial value. | + +### Class B — host-configuration (mutable post-instantiation) + +| Property | Why class B | +|---|---| +| `advected` | Whether the host's dycore advects this — host decision. | +| `diag_name` | Host-specific diagnostic system name. | +| `thermo_active` | Host model configuration. | +| `min_value` | Host runtime guardrail. | +| `water_species` | (Borderline — see §7) Physical classification but also host-config. | +| `mixing_ratio_type` | (Borderline — see §7) Depends on dycore convention. | + +### Consequences if adopted + +- `is_match` should check **only class A**. Today it checks 3 of 4 + class-B properties. +- Class B properties need setters. Today + `advected`, `diag_name`, (and `mixing_ratio_type` if it stays + class B) have none. +- `%instantiate` can demote class B from "required + optional" to + "all optional with sane defaults" — `diag_name=''`, + `advected=.false.`, etc. Schemes wouldn't need to set them at all. + +--- + +## 6. What to remove, replace, improve + +### Remove (or stop requiring) + +- **Scheme-metadata `diagnostic_name` on is_constituent args** — host + will override. Keep the attribute valid on non-constituent args + (where it's host tooling documentation, no portability issue). +- **`is_match` checks on advected / water_species / thermo_active** — + class B should not block dedup. +- **The `diag_name` requirement at `%instantiate`** — demote to + optional with `''` default. +- **(Not adopting)** Original capgen's auto-clone path. Already gone + in capgen-ng; this discussion does not propose bringing it back. + Listed for completeness because the option is in memory. + +### Replace + +- **`ConstituentVarDict`** as a concept — capgen-ng already runs + without it. If the framework or future generator code references + it, dropping is fine. +- **Single-global `ccpp_model_constituents_obj`** — capgen-ng's + per-instance array is the replacement. Original capgen could be + retrofitted, but the priority depends on whether multi-instance + enters the original capgen's roadmap. + +### Improve + +- **Add the missing setters**: `set_advected`, `set_diagnostic_name`, + `set_default_value` (if `default_value` becomes class B), + `set_mixing_ratio_type` (if class B). +- **Add a convenience routine** like + `ccpp_get_constituent_props_by_std_name(stdname, instance_number, prop_ptr, errflg, errmsg)` + so hosts can lookup a single constituent's property wrapper by + name without iterating. +- **Codegen-time cross-check** of scheme `_register` calls vs + metadata declarations (preferred: §4.9 option (a) — new + `registers_std_names` attr). +- **Document the lifecycle** clearly. `doc/constituents.md` is + ~960 lines; targeted additions for "register-then-override" + workflow once the new setters land. +- **Capgen-ng-internal cleanup** (LANDED 2026-05-13): replaced + `_FRAMEWORK_CONST_DIM_INPUTS` with a `used_const_dim_std_names` + field on `ResolvedArg`. + +--- + +## 7. Open design questions + +These are the calls we need to make in the meeting. + +### Q1. `default_value` — class A or class B? + +- **Class A argument**: the scheme knows what the species + should be initialized to (zero for "starts empty"; small positive + for "starts at background"); the host doesn't typically override. +- **Class B argument**: hosts may want non-default starting values + (chemistry runs with prescribed initial profiles). +- **Today's reality**: framework has no setter, so it's de-facto + class A. The advection-test issue 2026-05-12 surfaced because we + removed the `default_value=0._kind_phys` from cld_liq.F90's + scheme-side register and had no way to put it back; restoring it + in the scheme fixed the test but cements the class-A treatment. +- **Recommendation**: leave class A for now. Revisit when a real + host-override use case appears. + +### Q2. `water_species` — class A or class B? + +- The current `is_match` check on `water_species` treats it as + identity-defining (class A semantics). But the actual *meaning* of + the bit is mostly host bookkeeping ("does the dycore treat this as + water?"). CAM-SIMA has a `set_water_species` wrapper and uses it. +- **Recommendation**: class B, with the caveat that schemes whose + numerics depend on a constituent *being* water should declare that + in metadata as a hard requirement (different mechanism — not the + `is_match` machinery). + +### Q3. `mixing_ratio_type` — class A or class B? + +- The scheme's calculations assume `wrt_dry` or `wrt_moist`; this + feels class A. +- But hosts using different dycores might want to interpret the + same `std_name` differently — feels class B. +- **Recommendation**: class A. The mismatch should manifest as + different `std_name`s (`cloud_liquid_dry_mixing_ratio` vs + `cloud_liquid_wet_mixing_ratio`), not the same name with a runtime + override. Need cam-sima input. + +### Q4. After `is_match` relaxation: what happens on disagreement? + +- If two registrations of the same std_name agree on class A but + disagree on class B (e.g., `advected=.false.` from a scheme, + `advected=.true.` from the host), the second registration's class + B values should win without error. Effectively: the host overrides + the scheme. +- Order matters: today the host appends *after* the dynamic + constituents. Should we reverse so the host appends *first*? + Probably not — the "first registration wins on class A; host + setters override class B" model is conceptually clearer. +- **Recommendation**: silently dedup on matching class A; for class + B disagreements, the *later* registration's class B values are + ignored. Hosts use setters to override after registration + finalizes. + +### Q5. Should `%instantiate` accept class-B args at all? + +- **Option Y**: keep `%instantiate` accepting class B args (with + defaults). Schemes can supply them as hints; hosts can override. + Backward-compatible. +- **Option N**: remove class-B args from `%instantiate`. Schemes + *must* leave them to the host. Breaks the 4 cam-sima + scheme-registering schemes. +- **Recommendation**: option Y. The cost of breaking 4 schemes for + marginal clarity isn't worth it. + +### Q6. `ccpp_scheme_utils` singleton + +- Today: `ccpp_initialize_constituent_ptr(const_obj)` stores one + pointer module-wide. First instance wins. +- Fix options: + - (a) Maintain a per-instance pointer table; threading + `instance_number` through `ccpp_constituent_index`. + - (b) Document the limitation, route around it (no scheme uses + `ccpp_constituent_index` under multi-instance — capgen-ng + already enforces `index_of_` everywhere). +- **Recommendation**: (b). It's a one-line doc note and zero code + change. + +--- + +## 8. Three proposals — minimal / clean / deep + +### Proposal A — bugfix only + +**Scope**: +- Land the `ccpt_deallocate` ownership fix (done 2026-05-12). +- Update `scripts/constituents.py` for original capgen's auto-clone + path to pass `owned=.true.` (done). +- Add the three missing setters (`set_advected`, + `set_diagnostic_name`, `set_default_value`) without changing + semantics. Doesn't touch `is_match` or `%instantiate`. +- Document the gaps in `doc/constituents.md`. + +**Cost**: ~50 lines framework code + tests. No cam-sima changes +required. + +**Benefit**: closes the immediate bug, gives hosts the override +mechanism they need today (specifically for `diag_name`), unblocks +the advection test's deferred-property pattern. + +**Limit**: leaves `is_match` strict — hosts that disagree with a +scheme on `advected` still hit the "incompatible constituent" error. + +### Proposal B — class A/B split + setters + +**Scope** (in addition to A): +- Relax `is_match` to check only class A (`units` and possibly + `mixing_ratio_type`). +- Make all class-B properties optional in `%instantiate` with sane + defaults; deprecate (but keep accepting) class-B kwargs. +- Adopt the recommendation in Q4: silently dedup; host setters + override. +- Update `doc/constituents.md` with the register-then-override + workflow. +- (capgen-ng) Reject `diagnostic_name` on `is_constituent=True` + scheme args at parse time, or downgrade it to a default-only hint. +- (capgen-ng) **DONE 2026-05-13**: replaced `_FRAMEWORK_CONST_DIM_INPUTS` + with a `ResolvedArg.used_const_dim_std_names` field. + +**Cost**: ~150 lines framework + ~50 lines capgen-ng + tests. +CAM-SIMA host code can stay as-is (the 4 scheme-side registrations +continue to work with their existing class-B values; they're just +not enforced anymore). Optional: tidy the 4 schemes to pass class-A +only. + +**Benefit**: physics schemes become genuinely portable across +hosts. The class-B override pattern that CAM-SIMA already uses for +`thermo_active` and `water_species` generalizes. + +**Limit**: does not change the registration model (still +explicit-only in capgen-ng, still auto-clone in original capgen). + +### Proposal C — host-only registration + +**Scope** (in addition to B): +- Move the 4 cam-sima scheme-side register calls into a CAM-SIMA + helper module called from `cam_comp.F90`'s initialization. +- Drop register-phase `ccpp_constituent_properties_t(:)` support + from capgen-ng (and possibly original capgen). Schemes only + consume constituents; only the host registers. +- Codegen-time enforcement: any `advected=true` scheme arg whose + std_name is not in the host's enumeration → codegen error. +- Eliminates the `_dynamic_constituents` per-suite buffer + entirely. + +**Cost**: ~300 lines code total; requires coordinated PRs across +ccpp-framework, ccpp-capgen, ccpp-capgen-ng, atmospheric_physics, and +CAM-SIMA. The 4 schemes need their `_register` routines deleted (or +made no-ops); the host needs a new helper. + +**Benefit**: one source of truth for what constituents exist +(the host). Removes the auto-clone / scheme-register conceptual +overlap. Simplifies generator and runtime. + +**Limit**: changes the contract for the 4 scheme authors. Risk of +breaking yet-undiscovered downstream users of the scheme-side +registration model. + +### Comparison + +| Aspect | A | B | C | +|---|---|---|---| +| Lines changed | ~50 | ~200 | ~500+ | +| Coordination needed | framework only | framework + capgen-ng | framework + both generators + cam-sima | +| Fixes the crash | yes | yes | yes | +| Fixes `diag_name` portability | yes (host overrides) | yes | yes | +| Relaxes `is_match` | no | yes | yes | +| Removes scheme-side register | no | no | yes | +| Risk to existing CAM-SIMA workflows | none | low | medium | + +### Recommendation + +**Adopt A immediately (mostly done), aim for B over the next 4–6 +weeks, table C until the framework PR for B is in and we have a +clearer signal on whether the scheme-side register pattern is worth +keeping.** + +--- + +## 9. Appendix: framework setter inventory + +(For reference during the meeting. Reproduced from +`design_constituents_mutability.md`.) + +`ccpp_constituent_properties_t` methods (`src/ccpp_constituent_prop_mod.F90`): + +``` +Instantiation + procedure :: instantiate ! takes std_name, long_name, diag_name (REQUIRED), + ! units, vertical_dim, plus optional + ! advected, default_value, min_value, molar_mass, + ! water_species, mixing_ratio_type + procedure :: deallocate + +Getters (subset) + procedure :: standard_name + procedure :: long_name + procedure :: diagnostic_name + procedure :: units + procedure :: vertical_dimension + procedure :: is_advected + procedure :: is_thermo_active + procedure :: is_water_species + procedure :: is_mass_mixing_ratio + procedure :: is_volume_mixing_ratio + procedure :: is_number_concentration + procedure :: is_dry / is_moist / is_wet + procedure :: minimum + procedure :: molar_mass + procedure :: default_value + procedure :: has_default + procedure :: is_framework_owned ! NEW 2026-05-12 + +Setters (changes after instantiate) + procedure :: set_const_index + procedure :: set_thermo_active + procedure :: set_water_species + procedure :: set_minimum + procedure :: set_molar_mass + procedure :: set_framework_owned ! NEW 2026-05-12 + procedure :: set_advected ! GAP + procedure :: set_diagnostic_name ! GAP + procedure :: set_default_value ! GAP (or keep class A) + procedure :: set_mixing_ratio_type ! GAP (if class B) + +Identity / equality + procedure :: equivalent ! full equality + procedure :: is_match ! checks units + (class-B props ← too strict) +``` + +`ccpp_constituent_prop_ptr_t` is the pointer wrapper. Has parallel +setters that delegate to the underlying `ccpp_constituent_properties_t`. + +--- + +## Cross-references + +- `doc/constituents.md` — capgen-ng's user-facing constituents reference. +- `design_constituent_api.md` (memory) — capgen-ng's per-instance option-A design. +- `design_constituents_mutability.md` (memory) — extended design notes incl. class A/B classification. +- `project_implementation_status.md` (memory) — current implementation state and deferred items. +- `scripts/constituents.py` — original capgen's host-cap generator. +- `src/ccpp_constituent_prop_mod.F90` — framework. +- `capgen-ng/generator/host_constituents.py` — capgen-ng's host-side module emitter. +- `capgen-ng/generator/suite_resolver.py` (`_resolve_constituent_arg`) — capgen-ng's resolver routing. +- `EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` — CAM-SIMA's host-side wrappers around framework setters. + diff --git a/doc/migration_20260519T0905.md b/doc/migration_20260519T0905.md new file mode 100644 index 00000000..d69f41c3 --- /dev/null +++ b/doc/migration_20260519T0905.md @@ -0,0 +1,729 @@ +# Migrating from ccpp-prebuild / ccpp-capgen to capgen-ng + +This document captures the **user-facing differences** a host model author +or scheme author needs to know when moving metadata, suite XML, and host +Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to +**capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and +`doc/redesign_analysis.md` (analysis of the old systems). + +*Last revised: 2026-05-14 (end-of-day).* Current unit-test suite: 1229 passing. + +**Repository layout** (post-2026-05-13 cleanup): tooling lives under +`capgen-ng/` (top-level of this repo). Unit tests live at the top +level in `unit-tests/`; end-to-end tests in `end-to-end-tests/`. Run +the unit suite from the repo root with `python -m pytest unit-tests/`. + +## Table of contents + +1. [Metadata format changes](#1-metadata-format-changes) + 1. [1.8 `horizontal_loop_extent` → `horizontal_dimension`](#18-horizontal_loop_extent--horizontal_dimension) +2. [Suite definition file (SDF) changes](#2-suite-definition-file-sdf-changes) +3. [Host Fortran requirements](#3-host-fortran-requirements) +4. [Generator CLI and build integration](#4-generator-cli-and-build-integration) +5. [Generated cap layout — what's new and what changed](#5-generated-cap-layout--whats-new-and-what-changed) +6. [Framework changes (constituents)](#6-framework-changes-constituents) + 1. [6.3 Host metadata wins over auto-provisioning](#63-host-metadata-wins-over-auto-provisioning-2026-05-12) +7. [Validator (`ccpp_validator.py`)](#7-validator) +8. [Known gaps and deferred items](#8-known-gaps-and-deferred-items) + +--- + +## 1. Metadata format changes + +### 1.1 Table types + +Four `type =` values in `[ccpp-table-properties]`: + +| Type | Contents | +|---------|---------------------------------------------------------| +| `control` | Control variables passed as ``ccpp_physics_*`` args. | +| `host` | Host-model variables imported via `use`. | +| `ddt` | Derived-type definitions. | +| `scheme` | Scheme metadata. | + +The legacy `type = module` (capgen) becomes `type = host`. The legacy +`TYPEDEFS_NEW_METADATA` Python dict (prebuild) is replaced by `type = ddt` +tables. See `doc/redesign_prompt.md` §3.2. + +### 1.2 New table-property attributes + +All optional inside the `[ccpp-table-properties]` block: + +| Attribute | Applies to | Description | +|-----------------------|-----------------------|-------------| +| `module_name` | scheme, host, ddt | Fortran module name; overrides "module name = table name" when they differ. | +| `dependencies` | any | Comma-separated list of file paths to compile. **May appear multiple times** in one block (new this session); single occurrence still accepted. | +| `dependencies_path` | any | Relative base for `dependencies` entries. Single-valued. | +| `source_path` | any | Relative path to the Fortran source. Single-valued. | +| `kind_spec` | any | `:=>spec` (or shorthand). May appear multiple times. | + +Example with multi-line dependencies (real CCPP physics pattern): + +``` +[ccpp-table-properties] + name = GFS_rrtmg_setup + type = scheme + module_name = GFS_rrtmg_setup # optional when names match + dependencies_path = ../../ + dependencies = tools/mpiutil.F90 + dependencies = hooks/machine.F + dependencies = Radiation/RRTMG/radlw_main.F90,Radiation/RRTMG/radsw_main.F90 +``` + +### 1.3 New per-variable attributes + +Inside a `[ var_name ]` section. All optional. + +| Attribute | Type | Default | Notes | +|------------------|------|---------|-------| +| `top_at_one` | bool | `False` | When host and scheme disagree, generator emits a vertical-flip transform with reverse-stride subscript on the host side. Meaningless on variables without a vertical dimension. | +| `constituent` | bool | `False` | Scheme metadata only. Marks the var as a constituent reference. | +| `advected` | bool | `False` | Scheme metadata only. | +| `molar_mass` | float | `0.0` | Scheme metadata only. | +| `diagnostic_name` | str | (defaults to `local_name`) | Host-tooling hint; mutually exclusive with `diagnostic_name_fixed`. | + +### 1.4 Sliced local names with long subscript indices + +Local names with array slices may carry CCPP standard names as subscript +tokens: + +``` +[ dqdt(:,:,index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array) ] + standard_name = ... +``` + +The 63-char Fortran-identifier limit is enforced only on the base +identifier (`dqdt`), not on subscript tokens (which are CCPP standard +names resolved at codegen time and routinely exceed 63 chars). + +### 1.5 Unit strings: bare vs explicit positive exponent + +`m2` and `m+2` (or any `` vs `+` +combo) are normalized internally and treated as equivalent. Pre-existing +unit-conversion entries don't need to be duplicated; either spelling +matches. + +### 1.6 Improved error messages + +- **Duplicate standard name**: error message now lists both colliding + access paths and hints at the "sibling DDT instance" pattern (when + applicable). +- **Subcycle bound unresolved**: error names the std_name and points + at the control/host metadata as the fix. +- **Instance-dim used without `instance_number`**: error explains the + paired-opt-in requirement (see §1.7). + +### 1.7 Optional `instance_number` / `number_of_instances` pair + +These two control variables are now **paired optional** and both live +in the host's `type=control` table (symmetric with the +`thread_number` / `number_of_threads` pair): + +- Declare **both** in `type=control` → multi-instance API. Both flow + as control dummies through every lifecycle and physics-phase + signature. +- Declare **neither** → single-instance API. Public entry points drop + both args; internal per-instance arrays size to length 1. +- Declare exactly one → hard error from the validator. +- Declare `number_of_instances` in `type=host` → hard error + (must be `type=control`). + +Hosts that don't need multi-instance bookkeeping can drop both declarations. + +### 1.8 Deprecated standard names rewritten by `--legacy-mode` + +`--legacy-mode` is a transient migration shim that rewrites a small +set of deprecated standard names to their canonical capgen-ng +equivalents at parse time. The full table currently covers: + +| Deprecated (legacy) | Canonical (capgen-ng) | +|--------------------------------|--------------------------| +| `horizontal_loop_extent` | `horizontal_dimension` | +| `number_of_openmp_threads` | `number_of_threads` | + +Why each entry: + +* `horizontal_loop_extent` — ccpp-prebuild / original ccpp-capgen used + this for the horizontal-axis std name in scheme metadata. capgen-ng + uses `horizontal_dimension` uniformly; the run-vs-non-run distinction + isn't expressed in scheme metadata anymore (host passes + `horizontal_loop_begin` / `horizontal_loop_end` as control vars and + the generated cap slices accordingly). +* `number_of_openmp_threads` — legacy CCPP-physics hosts (CCPP-SCM + 17p8 in particular) size per-thread DDT containers by + `number_of_openmp_threads` (e.g. `physics%Interstitial`). capgen-ng + uses `number_of_threads`, which matches the `thread_number` control + variable, so the registered scalar-index dim table can substitute + `physics%Interstitial(thread_number)%…` automatically (see §3.4). + +The rewrite fires for both standard-name attributes AND dimension +tokens (so a host's `dimensions = (number_of_openmp_threads)` becomes +`dimensions = (number_of_threads)` before any further processing). + +Migration paths: + +1. **Edit the metadata** (recommended) — search-and-replace the + legacy names in every host / scheme `.meta` you maintain. +2. **Use `--legacy-mode`** (transient) — pass `--legacy-mode` to both + `ccpp_capgen_ng.py` and `ccpp_validator.py` and the renames happen + at parse time. A loud warning banner prints at startup, listing + every pair the shim is rewriting, so the substitution is never + invisible. This shim *will be removed*; treat it as a runway, + not a destination. + +--- + +## 2. Suite definition file (SDF) changes + +### 2.1 Schema v2.0 with nested-suite expansion + +Capgen-ng parses v2.0 SDFs and expands `` references +recursively at parse time. See `doc/redesign_prompt.md` §3 and the +`suite_v2_0.xsd` schema. + +### 2.2 `` with CCPP standard-name loop bound + +```xml + + effr_pre + +``` + +The `loop=` attribute accepts: + +- **Integer literal** (`loop="3"`) — emitted verbatim. +- **CCPP standard name** (`loop="num_subcycles_for_effr"`) — resolved + against host/control metadata; supports DDT-component access paths + (e.g. `phys_state%num_subcycles`). +- **Absent / empty** — treated as `loop="1"`. + +The loop-bound standard name is automatically included in the +introspection inputs list (`ccpp_physics_suite_variables` and +`_suite_host_data`). + +### 2.3 Nested `` elements + +```xml + + + effr_calc + + +``` + +Nested subcycles produce nested `do` loops in the generated cap. Loop +counter variables follow the convention: + +- Outermost / single-level: `ccpp_loop_counter`. +- Each deeper level: `ccpp_loop_counter_2`, `ccpp_loop_counter_3`, ... + +Effective iteration count = product of every level's `loop=` value. +`effr_calc` in the example runs 3·2 = 6 times. + +### 2.3.1 Passing the loop counter / extent to a scheme + +A scheme inside a `` block may consume the current iteration +counter and the total iteration count via two CCPP standard names: + +| Standard name | Fortran type | Meaning | +|----------------------|--------------|----------------------------------------------------------| +| `ccpp_loop_counter` | integer | Current subcycle iteration (1 … `ccpp_loop_extent`) | +| `ccpp_loop_extent` | integer | Total iterations — the `loop=` value on the `` | + +These are **loop-context control variables**: the host model does **not** +declare them. capgen-ng emits them automatically as locals in the +generated group cap (the `do` loop's induction variable for the counter, +the loop bound for the extent), and resolves any scheme arg requesting +them against those locals. + +Example scheme metadata fragment: + +``` +[iter] + standard_name = ccpp_loop_counter + units = index + dimensions = () + type = integer + intent = in +[niter] + standard_name = ccpp_loop_extent + units = index + dimensions = () + type = integer + intent = in +``` + +Place the scheme in a `` in the SDF: + +```xml + + sfc_diff + GFS_surface_loop_control_part1 + sfc_nst + +``` + +The generated group cap will emit `do ccpp_loop_counter = 1, 2` and call +the scheme with `iter = ccpp_loop_counter, niter = 2` (or the loop's +resolved local name when `loop=` is used). + +**Scope is the subcycle body.** A scheme that requests +`ccpp_loop_counter` / `ccpp_loop_extent` but is NOT inside a +`` block raises a clear parse-time error pointing at this +contract. + +**Nested-subcycle nuance** (see §8): nested-subcycle schemes that ask +for `ccpp_loop_counter` currently get the **outermost** loop's counter, +not the innermost. None of the in-tree physics catalogs use the +inner-counter case yet; revisit when one needs it. + +### 2.4 Suite-level `` and `` schemes + +```xml + + my_init_scheme + ... + my_final_scheme + +``` + +- Each element contains a **single** scheme name as text content. + Multiple `` children inside ``/`` is a schema + violation. (Group-shaped lists belong inside ``.) +- The named scheme's `init` / `final` phase metadata is resolved like + any other scheme phase; missing-phase metadata is a generator error. +- The scheme call is emitted inside `_init` / `_final` + with USE for the scheme module + per-arg host modules, and the + standard errflg check. +- Call ordering: + - `_init`: after all group `state_alloc` and + `suite_data_init_fields`, **before** the `CCPP_SUITE_FRAMEWORK_INITIALIZED` + state transition. + - `_final`: before the `CCPP_SUITE_UNREGISTERED` transition. + +**Accepted spellings**: `` and `` only. Legacy spellings +**``** (typo), **``** (correct long form), and +**``** are rejected with a clear error pointing at the +canonical short form. + +To exercise: + +1. Declare a scheme with `init` and/or `final` phases in its metadata + (minimal sig — just `errmsg` + `errflg` — is fine). +2. Reference it in the SDF as shown above. +3. Add the scheme's `.F90` to your build's source list. + +--- + +## 3. Host Fortran requirements + +### 3.1 Required control variables + +Every host's `type=control` table must declare: + +| Standard name | Fortran type | Purpose | +|-----------------------------------|--------------|-----------------------------------| +| `suite_name` | character | Drives suite dispatch | +| `horizontal_loop_begin` | integer | Lower chunk-bound | +| `horizontal_loop_end` | integer | Upper chunk-bound | +| `thread_number` | integer | Current thread | +| `number_of_threads` | integer | Total threads | +| `number_of_physics_threads` | integer | Physics-internal budget | +| `ccpp_error_code` | integer | Error flag | +| `ccpp_error_message` | character | Error message | + +Optional (paired — see §1.7): + +| Standard name | Fortran type | Table type | Purpose | +|-------------------------|--------------|------------|--------------------------------| +| `instance_number` | integer | control | Current instance index | +| `number_of_instances` | integer | control | Total instance count | + +### 3.2 Required entry-point call sequence + +``` +ccpp_register(suite_name, errflg, errmsg, [instance_number, number_of_instances]) + └── per scheme that declares a register phase +ccpp_init(suite_name, errflg, errmsg, [instance_number, number_of_instances]) + └── per scheme that declares an init phase +ccpp_physics_init(...) + └── physics phase routines per group: + ccpp_physics_init + ccpp_physics_timestep_init + ccpp_physics_run ← run-loop phase + ccpp_physics_timestep_final + ccpp_physics_final +ccpp_final(suite_name, errflg, errmsg, [instance_number, number_of_instances]) +``` + +The `(instance_number, number_of_instances)` pair appears in every +signature only when the host declares it (§1.7). Both flow uniformly +through lifecycle and physics-phase calls; the framework consumes +`number_of_instances` only at register/init time but carries it +elsewhere for API symmetry with `(thread_number, number_of_threads)`. + +### 3.3 Host module convention + +The Fortran module that exports a host metadata table's variables is +typically named after the table. When that's not the case, use the +`module_name` table-property override (§1.2): + +``` +[ccpp-table-properties] + name = test_host_data + type = host + module_name = mod_test_host_data +``` + +### 3.4 Registered scalar-index dimensions + +A small set of CCPP standard-name dimensions are *registered*: each +one is a count that capgen-ng auto-collapses to a paired scalar index +variable at every access site. + +| Count dim (in `dimensions = (...)`) | Index var (capgen-ng substitutes) | +|---|---| +| `number_of_instances` | `instance_number` | +| `number_of_threads` | `thread_number` | + +**Where these may appear**: ONLY on container DDT-instance variables in +the access path. Example: + +``` +[Interstitial] + standard_name = GFS_interstitial_type_instance + type = GFS_interstitial_type + dimensions = (number_of_threads) +``` + +Every scheme that reaches into `Interstitial%` will see the +generator emit `physics%Interstitial(thread_number)%` at the +call site — no metadata work required on the scheme side. + +**Two rules govern this:** + +1. *(generalized)* A container DDT-instance variable may carry any + registered scalar-index dim — single (`(number_of_threads)`) or + paired (`(number_of_instances, number_of_threads)`). Dims that + AREN'T registered flow through the normal slice machinery + (`horizontal_loop_begin:horizontal_loop_end`, `1:vertical_*`, …) + just like flat-array dims. +2. *(enforced — hard parse-time error)* A **leaf** variable + (intrinsic-typed or `external:` — the kind a scheme binds to) + **MUST NOT** declare a registered scalar-index dim. If you write:: + + [my_array] + type = real | kind = kind_phys + dimensions = (number_of_threads, horizontal_dimension) # ILLEGAL + + capgen-ng will reject it at parse time with a message pointing + at the wrap-in-DDT remediation pattern. Wrap the leaf in a + container DDT instead. + +The registered table lives in +[`capgen-ng/metadata/registered_dimensions.py`](../capgen-ng/metadata/registered_dimensions.py). +It carries a four-step recipe at the top of the file for adding new +pairings. + +--- + +## 4. Generator CLI and build integration + +### 4.1 `ccpp_capgen_ng.py` invocation + +``` +python ccpp_capgen_ng.py \ + --host-files [,,...] \ + --scheme-files [,,...] \ + --suites [,,...] \ + --host-name \ + --output-root /ccpp \ + [--kind-type =[:]] \ + [--legacy-mode] \ + [--verbose] [--verbose] +``` + +`--kind-type` syntax: `=[:]`. When `:` is +omitted, `` must be an ISO_FORTRAN_ENV constant (REAL32/REAL64/...) +and the module defaults to `iso_fortran_env`. `kind_phys` is +auto-defaulted to `iso_fortran_env:REAL64` when not supplied. + +`--legacy-mode` (transient migration shim, will be removed): silently +rewrites a small set of deprecated CCPP standard names to their +capgen-ng equivalents at parse time — see §1.8 for the full table +(`horizontal_loop_extent` → `horizontal_dimension`, +`number_of_openmp_threads` → `number_of_threads`). The rewrite fires +for both standard-name attributes AND dimension tokens. Prints a +loud warning banner at startup, enumerating every pair the shim is +rewriting, so the substitution is never invisible. Available on both +`ccpp_capgen_ng.py` and `ccpp_validator.py` (keep the flag consistent +between the two when both are invoked from CMake). All translation +logic is isolated in `metadata/legacy_compat.py` and tagged with +`# legacy-compat:` comments at every touchpoint, so the shim can be +cleanly removed when migration is complete. + +### 4.2 `ccpp_datafile.py` query CLI + +Generated `datatable.xml` carries: + +- `` — generated outputs (utilities/host_files/suite_files). +- `` — `.meta` and expanded SDF. +- `` — per-scheme call lists, **scoped to schemes that are + actually referenced by the loaded suites** (group phase calls + the + suite-level ``/`` hooks). Scheme metadata files passed + on the CLI but never referenced are silently dropped. +- `` — `dependencies = …` from host/control/ddt tables + (always) plus the same per-scheme list as `` (filtered to + the used set). Build systems that compile against + `ccpp_datafile.py --dependencies` therefore only pull in scheme deps + for compiled schemes; missing transitive deps in scheme metadata + surface as link errors and should be fixed in the `.meta` file. +- `` — host/api/suite/group dictionaries. + +Query via `ccpp_datafile.py -- `. Flags include +`--dependencies`, `--capgen-files`, `--host-files`, `--utility-files`, +`--suite-files`, `--scheme-files`, `--suite-list`, +`--required-variables `, `--input-variables `, +`--output-variables `, `--host-variables`, `--show`. + +`--suite-files` returns capgen-generated cap files (`ccpp__cap.F90`, +etc.). `--scheme-files` returns the **user-supplied scheme `.F90` sources** +that the loaded suites actually reference — the filtered compile manifest. +Each used scheme's source is resolved as `/.` +(extension preference order: `.F90`, `.f90`, `.F`, `.f`); missing files are +warned about and the canonical `.F90` guess is emitted so the build-system +query stays useful. + +### 4.3 CMake helpers + +`cmake/ccpp_capgen.cmake` and `cmake/ccpp_validator.cmake` provide the +`ccpp_capgen(...)` and `ccpp_validator(...)` macros. `ccpp_datafile(...)` +queries datatable.xml at configure time. + +### 4.4 No-op regeneration preserves mtimes + +Every generated file (caps, `datatable.xml`, `ccpp_kinds.F90`, expanded +SDFs, `.meta` artifacts) goes through `write_if_changed`: the new content +is staged to a sibling temp file under the output root and atomically +replaces the target only when the bytes actually differ. Reruns with +identical inputs therefore leave on-disk mtimes untouched, so CMake / +Make / Ninja do not trigger a downstream rebuild cascade. Matches the +behavior of legacy `ccpp-prebuild` / `ccpp-capgen`. The staging temp +file lives in the target's parent directory (always under +`--output-root`), so no `/tmp` access is required. + +--- + +## 5. Generated cap layout — what's new and what changed + +### 5.1 Output files + +Always generated: + +- `ccpp_kinds.F90` — kind parameters. Listed under ``. +- `_ccpp_cap.F90` — public host-facing entry points + introspection routines. + Filename and emitted `module _ccpp_cap` name are both driven by the + required `--host-name ` CLI argument so multiple host integrations + can co-exist in one executable. The public sub names inside + (`ccpp_register`, `ccpp_init`, `ccpp_physics_*`, `ccpp_final`) are + unchanged regardless of ``. +- `ccpp__cap.F90` — per-suite dispatcher. +- `ccpp___cap.F90` — per-group phase implementations. +- `ccpp__data.F90` — suite-owned interstitial DDT + module-level array. +- `ccpp__types.F90` — pointer-wrapper types for optional args. +- `ccpp__data.meta` — inspection artifact; pairs with `ccpp__data.F90` (`.meta` ↔ `.F90` filename convention). +- `datatable.xml` — build-system + host-introspection metadata. + +When any scheme registers constituents: + +- `ccpp_host_constituents.F90` — owns `ccpp_model_constituents_obj(:)` + and the host-facing constituent API. + +### 5.2 Per-suite data: TARGET on the instance array + +`ccpp_suite_data(:)` carries the `TARGET` attribute: + +```fortran +type(ccpp__data_t), allocatable, target, public :: ccpp_suite_data(:) +``` + +This makes every `ccpp_suite_data(i)%component(...)` subobject a valid +pointer-assignment target — needed for transformation temps and +optional-arg pointer wrappers. + +### 5.3 Variable transformations + +The generator emits three kinds of transform on a per-arg basis: + +| Transform | Trigger | +|------------------|--------------------------------------------------| +| Unit conversion | `host.units != scheme.units` with a registered conversion entry. | +| Kind conversion | `host.kind != scheme.kind` (different strings). | +| Vertical flip | `host.top_at_one != scheme.top_at_one` on a var with a vertical dim. | + +These compose. A scheme arg that needs unit + flip emits a single +combined assignment through a transformation temp: + +```fortran +temp_l = 1.0E-3_kind_phys*host_var(lb:ub, nlev:1:-1) ! unit conversion: kind_phys to kind_phys; vertical flip (top_at_one mismatch) +call scheme_run(temp=temp_l, ...) +host_var(lb:ub, nlev:1:-1) = 1.0E+3_kind_phys*temp_l ! ... reverse ... +``` + +Identity unit conversions (registered for dimensionally-equivalent +spellings like `J kg-1 ↔ m2 s-2`, formula `'{var}'`) are not labeled +"unit conversion" in the comment. + +### 5.4 Subcycle emission + +```fortran +integer :: ccpp_loop_counter +integer :: ccpp_loop_counter_2 +... +do ccpp_loop_counter = 1, phys_state%num_subcycles ! outer + call scheme_pre(...) + do ccpp_loop_counter_2 = 1, 2 ! inner + call scheme_calc(...) + end do +end do +``` + +### 5.5 State machine + +Per-instance integer state arrays: + +- `ccpp_suite_state(:)` — suite-level (UNREGISTERED / REGISTERED / + FRAMEWORK_INITIALIZED). +- `ccpp_group_state(:)` — group-level (UNINITIALIZED / INITIALIZED / + IN_TIMESTEP). + +Single-instance hosts get length-1 arrays indexed with literal `1`. +See `doc/redesign_prompt.md` §7. + +**Idempotent entry points.** `ccpp_physics_init`, `ccpp_physics_final`, and +`ccpp_final` are all silently idempotent — repeat calls return cleanly with +`errflg=0` rather than erroring. `ccpp_physics_final` additionally silent-skips +when issued *after* `ccpp_final` has torn the suite down (state array +deallocated on the last instance, or `== UNREGISTERED` on any other instance). +The other physics phases (`timestep_init`, `run`, `timestep_final`) still +hard-error on a state mismatch. `ccpp_init` does *not* silent-skip when the +state array is unallocated — there, "not allocated" really does mean +"you forgot `ccpp_register`" and continues to be a hard error. + +--- + +## 6. Framework changes (constituents) + +### 6.1 `ccpp_constituent_prop_mod` ownership flag + +(Framework PR — needs upstream merge.) Adds: + +- `framework_owns_me` private flag on `ccpp_constituent_properties_t`, + default `.false.`. +- `set_framework_owned(value)` setter (call before + `obj%new_field(const_prop, ...)` when transferring ownership). +- `is_framework_owned()` getter. +- `ccpt_deallocate` only frees when the flag is set; otherwise just + nullifies. + +Backward-compatible. Original capgen's auto-clone path in +`scripts/constituents.py` has been updated to call the setter. + +### 6.2 capgen-ng constituent API + +(See `doc/constituents.md` for the full reference.) Highlights: + +- One `ccpp_model_constituents_obj(:)` array per generator invocation, + sized to `number_of_instances`. +- Host-facing API: + - `ccpp_register_constituents(host_constituents, instance_number, ...)` + - `ccpp_initialize_constituents(ncols, num_layers, instance_number, ...)` + - `ccpp_const_get_index(stdname, const_index, instance_number, ...)` + - `ccpp_constituents_array(instance_number) → pointer` + - `ccpp_advected_constituents_array(instance_number) → pointer` + - `ccpp_model_const_properties(instance_number) → pointer` + - `ccpp_number_constituents(num_flds, advected, instance_number, ...)` + - `ccpp_gather_constituents`, `ccpp_update_constituents` + - `ccpp_is_scheme_constituent(var_name, ...)` (not per-instance) +- Scheme-side registration: four rules — register-phase + `ccpp_constituent_properties_t(:)` arg, consume base via + `advected=true intent=in/inout`, produce tendency via + `constituent=true intent=out` + `tendency_of_` std name, mismatches + are codegen errors. + +### 6.3 Host metadata wins over auto-provisioning (2026-05-12) + +If the host declares a framework-named standard name +(`ccpp_constituents` / `ccpp_constituent_tendencies` / +`ccpp_constituent_properties` / `number_of_ccpp_constituents` / +`index_of_`) as a regular host variable, the resolver uses the +host's declaration and skips capgen-ng auto-provisioning. Matters +most for legacy hosts (GFS / SCM) that own their own tracer +indices — e.g. `[ntcw]` with `standard_name = +index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array` +resolves to the host's short local name `ntcw`, not a parallel +module-level integer named after the full standard name (which +would also blow Fortran's 63-char identifier limit). See +`doc/constituents.md` §3. + +Active design review for the next constituents iteration: +`doc/constituents_overhaul.md` (Class A vs Class B property +classification, three reform proposals). + +--- + +## 7. Validator + +`capgen-ng/ccpp_validator.py` — standalone Fortran-vs-metadata checker. +Today validates **scheme** metadata against scheme Fortran files +(subroutine signatures, optional args, paren-aware decl splitting). + +Continuation-line handling covers both free-form (`&` at trailing end +of prior line only) and fixed-form / dual-form (`&` at both ends, with +the leading marker at column 6). Comment-only and blank lines +interleaved between continuation lines are skipped as Fortran 90+ +permits. + +When the signature parser finds a subroutine but extracts zero args +while metadata declares many, the "Argument count mismatch" error +appends a HINT pointing at the parser rather than masquerading as a +real mismatch — common cause is an unsupported signature feature. + +**Known gap**: host-metadata validation is not yet implemented. When +invoked with non-scheme `.meta` files, the validator silently filters +to zero schemes and reports "Validation passed." Slated for revisit +after the e2e test suite settles (`unit_conv` + `variable_transform` +complete). See `project_validator_host_check_deferred.md` (memory). + +--- + +## 8. Known gaps and deferred items + +| Item | Status | +|--------------------------------------------|-----------------------------------------------| +| `ccpp_loop_counter` standard name inside nested subcycles | Maps to OUTERMOST loop var. None of cam-sima uses this; revisit if a scheme needs the innermost value. | +| Validator host-metadata check | Deferred; revisit after e2e tests stabilize. | +| Constituents overhaul (Class A/B + setters) | Discussion doc at `doc/constituents_overhaul.md`. | +| Framework setters: `set_advected`, `set_diagnostic_name`, `set_default_value` | Deferred; depends on constituents-overhaul decision. | +| Codegen-time scheme-registration cross-check | Deferred; would require new `registers_std_names` metadata attr. | +| `_FRAMEWORK_CONST_DIM_INPUTS` cleanup | **Done 2026-05-13**: hand-curated frozenset gone; framework-constituent dim refs ride on a dedicated `used_const_dim_std_names` field on `ResolvedArg`. | +| Suppress `ccpp_host_constituents.F90` when unused | Deferred; currently emitted for every build even when no scheme/host actually exercises the constituent system. Now *correct* (empty) for SCM-style hosts thanks to the host-wins rule, but still dead code. See `design_constituent_host_wins.md`. | +| Python linter / formatter pass | Deferred; pick `ruff` and apply across `capgen-ng/`. | +| Generated Fortran ↔ Codee formatter idempotency | Deferred; emitted `.F90` must round-trip cleanly through the project's Codee Fortran formatter. | +| `fortran_to_metadata` developer utility | Deferred; bootstraps a `.meta` skeleton from an existing `.F90` subroutine. | +| `--legacy-mode` shim removal | Transient; remove `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and every `# legacy-compat:` touchpoint when scheme metadata has migrated. | +| `ccpp_datafile.py` query CLI rework | Deferred (2026-05-13); collapse `--host-files` / `--suite-files` / `--utility-files` into `--capgen-files`, then repurpose `--host-files` as a filtered list of **input** host metadata files (parallel to `--scheme-files`). Most hosts pack all host data into a handful of shared files, so the filtering pay-off is small — the draw is API symmetry. | +| Original capgen auto-clone path | Intentionally dropped in favor of explicit registration; kept in memory as "Option B" fallback. | + +--- + +## Cross-references + +- `doc/redesign_prompt.md` — original design specification (sections + marked "historic" where the implementation has evolved). +- `doc/redesign_analysis.md` — analysis of the legacy ccpp-prebuild + + ccpp-capgen toolchains. +- `doc/constituents.md` — full constituents reference for capgen-ng. +- `doc/constituents_overhaul.md` — architecture review and reform + proposals for the next iteration. + From 80b0d170397fa7360f0e12b8a73f0556fbdc54e1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 May 2026 05:35:03 -0600 Subject: [PATCH 31/74] Better error messages from xml suite parser; run doctests in CI --- .github/workflows/unit-tests.yaml | 10 ++++------ capgen-ng/generator/suite_resolver.py | 7 +------ capgen-ng/generator/suite_xml.py | 7 ++++++- capgen-ng/metadata/parse_tools/xml_tools.py | 7 ++++++- unit-tests/test_suite_xml.py | 4 +++- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index eabc01ad..2b99bc13 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -24,11 +24,6 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - #- name: Install dependencies - # run: | - # python -m pip install --upgrade pip - # pip install pytest - # which pytest - name: Update repos and install dependencies run: | sudo apt-get update @@ -47,4 +42,7 @@ jobs: which pytest - name: Run capgen-ng unit tests run: | - pytest -v unit-tests/ \ No newline at end of file + pytest -v unit-tests/ + - name: Run capgen-ng module doctests + run: | + PYTHONPATH=capgen-ng pytest --doctest-modules capgen-ng diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 21cf6487..94bcb44f 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -1910,12 +1910,7 @@ def resolve_suite( "``[ccpp-table-properties] type = scheme`` table is available " "in the metadata files passed via ``--scheme-files``. Add " "the missing scheme ``.meta`` files to the generator's " - "--scheme-files argument (CMake users: add them to the " - "scheme metadata list in the relevant ``CMakeLists.txt``).\n" - "\n" - "Without this check capgen-ng would silently emit an empty " - "group cap and the build would succeed with the wrong " - "runtime behaviour (schemes never run).".format( + "--scheme-files argument.".format( suite=suite.name, n=len(missing), names='\n '.join(missing), diff --git a/capgen-ng/generator/suite_xml.py b/capgen-ng/generator/suite_xml.py index fefbcf8a..d04cbad8 100644 --- a/capgen-ng/generator/suite_xml.py +++ b/capgen-ng/generator/suite_xml.py @@ -573,7 +573,12 @@ def parse_suite_xml( log.info("Reading suite XML: %s", suite_file) _, root = read_xml_file(suite_file, log) - version = find_schema_version(root) + try: + version = find_schema_version(root) + except CCPPError as verr: + raise CCPPError( + f"{verr} in suite XML file '{suite_file}'" + ) from verr log.debug("Suite XML schema version: %d.%d", *version) # ---- schema validation (pre-expansion) -------------------------------- diff --git a/capgen-ng/metadata/parse_tools/xml_tools.py b/capgen-ng/metadata/parse_tools/xml_tools.py index 076a47eb..52f80c12 100644 --- a/capgen-ng/metadata/parse_tools/xml_tools.py +++ b/capgen-ng/metadata/parse_tools/xml_tools.py @@ -208,7 +208,12 @@ def load_suite_by_name(suite_name, group_name, file, logger=None): >>> tmpdir.cleanup() """ _, root = read_xml_file(file, logger) - schema_version = find_schema_version(root) + try: + schema_version = find_schema_version(root) + except CCPPError as verr: + raise CCPPError( + f"{verr} in nested suite XML file '{file}'" + ) from verr if not validate_xml_file(file, 'suite', schema_version, logger): raise CCPPError(f"Invalid suite definition file, '{file}'") if root.attrib.get("name") == suite_name: diff --git a/unit-tests/test_suite_xml.py b/unit-tests/test_suite_xml.py index 8fe3647b..054ed7cf 100644 --- a/unit-tests/test_suite_xml.py +++ b/unit-tests/test_suite_xml.py @@ -594,8 +594,10 @@ def test_bad_schema_version_formats(self): self._parse(fname) def test_missing_version_raises(self): - with self.assertRaises(CCPPError): + with self.assertRaises(CCPPError) as cm: self._parse('suite_missing_version.xml') + self.assertIn('suite_missing_version.xml', str(cm.exception)) + self.assertIn('Version attribute required', str(cm.exception)) def test_infinite_group_recursion_detected(self): """Circular nested-suite references at group level are caught.""" From 5ab30e5c92e773b519a319d2ddc95bc65841a589 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 May 2026 09:21:22 -0600 Subject: [PATCH 32/74] Simplify GitHub actions testing --- .github/workflows/end-to-end-tests.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 3af5eff3..0bdf2924 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -38,16 +38,11 @@ jobs: which xmllint xmllint --version which pytest - - - name: Build the framework + - name: Run end-to-end tests run: | cmake --fresh -S./end-to-end-tests -B./build cd build make - - - name: Run end-to-end tests - run: | - cd build ctest --rerun-failed --output-on-failure . # - name: Run python tests From 9346b75ca75de0380a9784a1fd3d22eda7a282ec Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 May 2026 10:49:40 -0600 Subject: [PATCH 33/74] Enable stricter Fortran <--> Metadata validation --- capgen-ng/ccpp_capgen_ng.py | 11 +- capgen-ng/ccpp_datafile.py | 4 +- capgen-ng/ccpp_validator.py | 464 ++++++++++++++++-- capgen-ng/generator/datatable.py | 6 +- capgen-ng/generator/static_api.py | 2 +- capgen-ng/generator/suite_cap.py | 2 +- doc/briefing.md | 15 +- doc/constituents.md | 19 +- doc/migration.md | 79 ++- doc/redesign_prompt.md | 27 +- .../sample_files/scheme_multipart_correct.F90 | 11 +- unit-tests/test_validator.py | 306 ++++++++++++ 12 files changed, 849 insertions(+), 97 deletions(-) diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index d0946f6d..15dd726f 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -7,7 +7,8 @@ and suite XML definition files, resolves all variable connections, and writes: * ``ccpp_kinds.F90`` — kind parameter definitions -* ``ccpp_static_api.F90`` — static dispatch API +* ``_ccpp_cap.F90`` — static dispatch API (per-host; filename and module + name derived from ``--host-name``) * ``ccpp__cap.F90`` — suite-level cap (state machine, group dispatch) * ``ccpp___cap.F90`` — group-level cap (scheme call sites) * ``ccpp__data.F90`` — suite-owned interstitial data module @@ -259,15 +260,15 @@ def _build_arg_parser() -> argparse.ArgumentParser: action='store_true', help=( "Stub the five suite-introspection routines in " - "ccpp_static_api.F90 (ccpp_physics_suite_list / " + "_ccpp_cap.F90 (ccpp_physics_suite_list / " "suite_part_list / suite_schemes / suite_variables / " "suite_host_data). Signatures remain so callers still " "link, but bodies set errflg=1 with a clear errmsg " "(suite_list, which has no errflg, writes to error_unit " "and returns an empty list). Use this to shrink the " - "generated ccpp_static_api.F90 from ~33000 lines to ~800 " - "for multi-suite builds where -O3 cannot finish compiling " - "the introspection case-blocks." + "generated _ccpp_cap.F90 from ~33000 lines to ~800 " + "for multi-suite builds where even -O1 cannot finish " + "compiling the introspection case-blocks." ), ) parser.add_argument( diff --git a/capgen-ng/ccpp_datafile.py b/capgen-ng/ccpp_datafile.py index dd55d872..8976fedf 100755 --- a/capgen-ng/ccpp_datafile.py +++ b/capgen-ng/ccpp_datafile.py @@ -26,8 +26,8 @@ Notes specific to capgen-ng --------------------------- -* ``--host-files`` returns ``ccpp_static_api.F90`` (capgen-ng emits the - static API in lieu of a per-host cap file). +* ``--host-files`` returns ``_ccpp_cap.F90`` (the per-host static API; + filename and module name derived from ``--host-name`` at generation time). * ``--capgen-files`` enumerates Fortran sources only. Non-Fortran inspection artifacts (``ccpp_.meta``, ``ccpp__expanded.xml``) are reported via ``--inspection-files``. diff --git a/capgen-ng/ccpp_validator.py b/capgen-ng/ccpp_validator.py index 749e1102..88d64d8b 100755 --- a/capgen-ng/ccpp_validator.py +++ b/capgen-ng/ccpp_validator.py @@ -9,10 +9,26 @@ 2. Has the **same number of dummy arguments** as declared in the metadata. 3. The dummy-argument **names match** the ``local_name`` values in the metadata (order-insensitive). - -The tool does *not* parse full Fortran type declarations — that level of -verification is intentionally kept out of the code generator path (see design -doc: toolchain structure). +4. For every dummy argument present in both sides, the **per-arg attributes** + agree: ``intent``, ``type``, ``kind``, and number of dimensions (rank). + ``character`` arguments treat ``len=*`` on either side as a wildcard + against any concrete ``len=N`` / ``len=:``. + +Asymmetric treatment of ``optional``: + +* Fortran-declared optional argument **absent** from metadata → silently + allowed (the cap never passes it); emits a ``logger.warning``. +* Fortran-declared optional argument **present** in metadata as + ``optional=False`` → silently allowed (the cap always passes it, which + is a valid subset of the Fortran contract); emits a ``logger.warning``. +* Metadata declares ``optional=True`` but Fortran does **not** carry the + ``optional`` attribute → **error** (the cap-side ``present()`` check + would be invalid on a Fortran-required dummy). + +The tool does *not* compare dimension *bounds* across sides — it only +checks that rank matches. Comparing standard-name dimension references +against Fortran local-name dimensions would require loading host metadata +too; that's a separate feature. Usage ----- @@ -35,7 +51,7 @@ import os import re import sys -from typing import Dict, List, NamedTuple, Optional, Set +from typing import Dict, List, NamedTuple, Optional, Set, Tuple # Ensure the capgen-ng package is importable when invoked directly. _SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -77,6 +93,43 @@ _IDENT_CHAR_RE = re.compile(r'[A-Za-z_0-9]') +class _ArgAttrs(NamedTuple): + """Per-dummy-argument attributes parsed from a Fortran type-decl line. + + All string fields are lowercased and stripped. Missing / unknown is + represented by an empty string (or ``False`` for ``optional``, ``0`` + for ``rank``). + + Attributes + ---------- + type_ : str + Intrinsic type (``'real'``, ``'integer'``, ``'logical'``, + ``'complex'``, ``'character'``) or a derived-type spec + (``'type(my_type)'``, ``'class(other)'``). Empty for args + not declared in the body (parser missed the decl). + kind_ : str + Kind selector. For numeric types this is the kind name + (``'kind_phys'``, ``'8'``, ``'int64'``). For character it is + the length selector (``'len=10'``, ``'len=*'``, ``'len=:'``). + Empty when no selector was present. + intent : str + ``'in'`` / ``'out'`` / ``'inout'``, or ``''`` when no intent was + declared (treated as INOUT by Fortran, but for validation we + prefer to flag the absence explicitly). + optional : bool + True iff the type-decl line carried the ``optional`` attribute. + rank : int + Number of dimensions. Computed from ``dimension(...)`` on the + line, or from ``var(:,:,...)``-style trailing parens on the + variable token. ``0`` for scalar. + """ + type_: str + kind_: str + intent: str + optional: bool + rank: int + + class _SubSig(NamedTuple): """Parsed signature of one Fortran subroutine. @@ -89,9 +142,15 @@ class _SubSig(NamedTuple): subroutine body. These args may be absent from the metadata without producing a validation error — they will simply never be passed at the cap call site. + attrs : dict + Mapping of lowercase arg name to :class:`_ArgAttrs`. Args + whose type-decl line couldn't be parsed are absent from the + dict; ``_validate_arg_attributes`` skips per-attribute checks + for those args (the name-set check still applies). """ - args: List[str] + args: List[str] optional: Set[str] + attrs: Dict[str, '_ArgAttrs'] def _paren_aware_split(s: str, sep: str) -> List[str]: @@ -124,43 +183,164 @@ def _paren_aware_split(s: str, sep: str) -> List[str]: return result -def _line_optional_names(line: str) -> List[str]: - """Return lowercase var names from a type-decl line carrying ``optional``. +_INTENT_RE = re.compile(r'(?i)^intent\s*\(\s*(in\s*out|inout|in|out)\s*\)\s*$') +_DIM_ATTR_RE = re.compile(r'(?i)^dimension\s*\(\s*(.*?)\s*\)\s*$') +_TYPE_SPEC_RE = re.compile( + r'(?i)^\s*(real|integer|logical|complex|character|double\s*precision' + r'|type\s*\([^)]*\)|class\s*\([^)]*\))\s*(\(.*\))?\s*$' +) +_KIND_SELECTOR_RE = re.compile(r'(?i)^\(\s*(.*?)\s*\)$') + + +def _split_type_spec(spec: str) -> "Tuple[str, str]": + """Split a Fortran type spec into ``(type, kind)``. - Matches Fortran lines of the form - `` [, ...] :: [, ...]`` where one of the - comma-separated attributes (paren-aware) is the bare token - ``optional``. Returns an empty list when ``::`` is absent or when no - ``optional`` attribute is present. + The type is lowercased; the kind selector is left in its raw form + (lowercased, whitespace stripped). Returns ``('', '')`` if *spec* + isn't a recognised type spec. Examples -------- - >>> _line_optional_names('integer, optional, intent(in) :: innie') - ['innie'] - >>> _line_optional_names('real, intent(out), optional :: outie') - ['outie'] - >>> _line_optional_names('real(kind=kind_phys), optional :: x, y(:,:)') - ['x', 'y'] - >>> _line_optional_names('integer :: not_optional') - [] - >>> _line_optional_names(' ! a comment, optional :: not_a_decl') - [] + >>> _split_type_spec('real') + ('real', '') + >>> _split_type_spec('real(kind=kind_phys)') + ('real', 'kind_phys') + >>> _split_type_spec('real(kind_phys)') + ('real', 'kind_phys') + >>> _split_type_spec('real(8)') + ('real', '8') + >>> _split_type_spec('integer(int64)') + ('integer', 'int64') + >>> _split_type_spec('character(len=10)') + ('character', 'len=10') + >>> _split_type_spec('character(len=*)') + ('character', 'len=*') + >>> _split_type_spec('character(*)') + ('character', 'len=*') + >>> _split_type_spec('character') + ('character', '') + >>> _split_type_spec('type(my_t)') + ('type(my_t)', '') + >>> _split_type_spec('double precision') + ('double precision', '') + >>> _split_type_spec('not_a_type') + ('', '') + """ + m = _TYPE_SPEC_RE.match(spec.strip()) + if m is None: + return ('', '') + type_raw = m.group(1).lower() + kind_paren = m.group(2) or '' + # Normalise whitespace inside "double precision". + type_ = re.sub(r'\s+', ' ', type_raw) + if type_.startswith('type(') or type_.startswith('class('): + # Strip whitespace inside the parens. + type_ = re.sub(r'\s+', '', type_) + return (type_, '') + if not kind_paren: + return (type_, '') + inner_match = _KIND_SELECTOR_RE.match(kind_paren.strip()) + if inner_match is None: + return (type_, '') + inner = inner_match.group(1).strip() + if type_ == 'character': + # character has its own selector grammar. Accept: + # * -> len=* + # len=... -> len=... + # -> len= + # len=...,kind=... -> use the len= portion + if inner == '*': + return ('character', 'len=*') + if inner.lower().startswith('len='): + # Strip trailing ",kind=..." if present. + len_part = inner.split(',')[0].strip() + return ('character', len_part.lower()) + if re.match(r'^\d+$', inner) or inner == ':': + return ('character', 'len={}'.format(inner)) + # Anything else: store raw, lowercased. + return ('character', inner.lower()) + # Numeric types: accept ``kind=`` or bare ````. + if inner.lower().startswith('kind='): + inner = inner[len('kind='):].strip() + return (type_, inner.lower()) + + +def _parse_decl_line(line: str) -> Dict[str, _ArgAttrs]: + """Parse a Fortran type-declaration line into per-name attributes. + + Returns a (possibly empty) mapping ``{lower_name: _ArgAttrs}``. Lines + that aren't type declarations (no ``::``) or whose type spec doesn't + parse return ``{}``. + + Examples + -------- + >>> attrs = _parse_decl_line('integer, intent(in) :: im') + >>> attrs['im'] + _ArgAttrs(type_='integer', kind_='', intent='in', optional=False, rank=0) + >>> attrs = _parse_decl_line('real(kind=kind_phys), intent(inout) :: temp(:,:)') + >>> attrs['temp'] + _ArgAttrs(type_='real', kind_='kind_phys', intent='inout', optional=False, rank=2) + >>> attrs = _parse_decl_line('character(len=*), intent(out) :: errmsg') + >>> attrs['errmsg'] + _ArgAttrs(type_='character', kind_='len=*', intent='out', optional=False, rank=0) + >>> attrs = _parse_decl_line('real, optional, intent(in), dimension(:) :: a, b(:,:), c') + >>> sorted(attrs.items()) + [('a', _ArgAttrs(type_='real', kind_='', intent='in', optional=True, rank=1)), ('b', _ArgAttrs(type_='real', kind_='', intent='in', optional=True, rank=2)), ('c', _ArgAttrs(type_='real', kind_='', intent='in', optional=True, rank=1))] + >>> _parse_decl_line(' ! comment :: not a decl') + {} + >>> _parse_decl_line('integer :: only_local') + {'only_local': _ArgAttrs(type_='integer', kind_='', intent='', optional=False, rank=0)} """ - # Strip any inline ``!`` comment so an ``optional`` token inside a - # comment can't be misread as an attribute declaration. line = _COMMENT_RE.sub('', line) if '::' not in line: - return [] + return {} before, _, after = line.partition('::') - attrs = [a.strip().lower() for a in _paren_aware_split(before, ',')] - if 'optional' not in attrs: - return [] - names: List[str] = [] - for tok in _paren_aware_split(after, ','): - m = re.match(r'\s*(\w+)', tok) + tokens = _paren_aware_split(before, ',') + if not tokens: + return {} + type_, kind_ = _split_type_spec(tokens[0]) + if not type_: + return {} + intent = '' + optional = False + line_rank = 0 + for tok in tokens[1:]: + t = tok.strip() + tl = t.lower() + if tl == 'optional': + optional = True + continue + m = _INTENT_RE.match(t) if m: - names.append(m.group(1).lower()) - return names + iv = m.group(1).lower().replace(' ', '') + intent = iv # 'in' / 'out' / 'inout' + continue + m = _DIM_ATTR_RE.match(t) + if m: + line_rank = len(_paren_aware_split(m.group(1), ',')) + continue + # Anything else (allocatable, pointer, target, parameter, save, + # public, private, contiguous, asynchronous, volatile, value) is + # ignored — we only validate the attrs metadata declares. + result: Dict[str, _ArgAttrs] = {} + for var_tok in _paren_aware_split(after, ','): + var_tok = var_tok.strip() + if not var_tok: + continue + name_match = re.match(r'(\w+)\s*(\((.*)\))?\s*(=.*)?$', var_tok) + if name_match is None: + continue + name = name_match.group(1).lower() + inner = name_match.group(3) + if inner is not None: + rank = len(_paren_aware_split(inner, ',')) + else: + rank = line_rank + result[name] = _ArgAttrs( + type_=type_, kind_=kind_, intent=intent, + optional=optional, rank=rank, + ) + return result def _join_continuation( @@ -349,16 +529,21 @@ def _parse_subroutines( ['x', 'y', 'z'] >>> sorted(sig.optional) ['y', 'z'] + >>> sig.attrs['x'].intent, sig.attrs['x'].type_ + ('in', 'integer') + >>> sig.attrs['y'].optional + True """ logical = _join_continuation( source.splitlines(keepends=True), filename=filename, ) - args_by_name: Dict[str, List[str]] = {} - optional_by_name: Dict[str, Set[str]] = {} + args_by_name: Dict[str, List[str]] = {} + optional_by_name: Dict[str, Set[str]] = {} + attrs_by_name: Dict[str, Dict[str, _ArgAttrs]] = {} # Stack of names whose body we are currently scanning. Each entry is - # the recorded name (for which we collect optionals) or ``None`` when - # this is a duplicate-name sub whose body should be skipped for the - # purpose of optional-attribution (its args were already discarded). + # the recorded name (for which we collect attrs) or ``None`` when + # this is a duplicate-name sub whose body should be skipped (its + # args were already discarded). stack: List[Optional[str]] = [] for line in logical: @@ -371,6 +556,7 @@ def _parse_subroutines( if name not in args_by_name: args_by_name[name] = args optional_by_name[name] = set() + attrs_by_name[name] = {} stack.append(name) else: stack.append(None) # duplicate: ignore @@ -382,13 +568,20 @@ def _parse_subroutines( if stack and stack[-1] is not None: tracked = stack[-1] arg_set = set(args_by_name[tracked]) - for n in _line_optional_names(line): - if n in arg_set: - optional_by_name[tracked].add(n) + for var_name, attrs in _parse_decl_line(line).items(): + if var_name not in arg_set: + continue + # First decl line wins (Fortran disallows redeclaration, + # so this only matters for malformed input). + if var_name not in attrs_by_name[tracked]: + attrs_by_name[tracked][var_name] = attrs + if attrs.optional: + optional_by_name[tracked].add(var_name) return { name: _SubSig(args=args_by_name[name], - optional=optional_by_name[name]) + optional=optional_by_name[name], + attrs=attrs_by_name[name]) for name in args_by_name } @@ -526,9 +719,194 @@ def _validate_scheme( sub_name, sorted(only_fort_required) ) ) + # Per-arg attribute checks for args present in BOTH sides. + meta_by_name = {v.local_name.lower(): v for v in meta_vars} + for name in sorted(meta_set & fort_set): + fattrs = sig.attrs.get(name) + if fattrs is None: + # Decl line failed to parse; skip attribute checks for + # this arg. Name-set check already covered presence. + continue + mvar = meta_by_name[name] + errors.extend( + _check_arg_attributes(sub_name, name, mvar, fattrs) + ) + # Optional flag — asymmetric: + # - metadata says optional, Fortran doesn't → hard error + # (cap may pass a missing arg, but Fortran requires it). + # - Fortran says optional, metadata doesn't → warning + # (cap always passes it; that's a valid subset of the + # Fortran contract, but the metadata writer may have + # intended to mark it optional). + if mvar.optional and not fattrs.optional: + errors.append( + "Arg '{}' on '{}': metadata declares optional=True " + "but Fortran does not carry the 'optional' attribute " + "(cap-side present() checks would be invalid)".format( + name, sub_name, + ) + ) + elif fattrs.optional and not mvar.optional: + logger.warning( + "Fortran argument '%s' on subroutine '%s' is " + "declared optional but metadata does not mark it " + "optional; cap will always pass it", + name, sub_name, + ) + # Fortran-only optional args (absent from metadata entirely): + # silently allowed but worth a heads-up — the host won't see + # them and the metadata writer may have meant to declare them. + for name in sorted(fort_only_optional): + logger.warning( + "Optional Fortran argument '%s' on subroutine '%s' is " + "absent from metadata; it will never be passed at the " + "call site", + name, sub_name, + ) return errors +_EXTERNAL_TYPE_PREFIX_RE = re.compile(r'(?i)^external\s*:\s*[^:]+\s*:\s*') +_DDT_WRAPPER_RE = re.compile(r'(?i)^(?:type|class)\s*\(\s*(.+?)\s*\)\s*$') + + +def _normalize_type_for_comparison(type_str: str) -> str: + """Return a comparison-friendly form of a CCPP type string. + + Rules: + + * Lowercase, whitespace collapsed. + * ``type(name)`` / ``class(name)`` wrapper → bare ``name``. + * ``external::`` → bare ``typename`` (the module + part is metadata-only; Fortran uses the bare type name once the + module is brought in via a ``use`` clause). + * ``doubleprecision`` → ``double precision``. + * Anything else is returned as-is (intrinsics, DDT names, etc.). + + With this normalisation, a metadata declaration ``type = ty_rad_lw`` + matches a Fortran ``type(ty_rad_lw)`` dummy, and a metadata + ``type = external:mpi_f08:mpi_comm`` matches a Fortran + ``type(mpi_comm)`` dummy. Intrinsic comparisons (``real`` vs + ``real``) are unaffected. + + Examples + -------- + >>> _normalize_type_for_comparison('real') + 'real' + >>> _normalize_type_for_comparison('REAL') + 'real' + >>> _normalize_type_for_comparison('double precision') + 'double precision' + >>> _normalize_type_for_comparison('doubleprecision') + 'double precision' + >>> _normalize_type_for_comparison('ty_rad_lw') + 'ty_rad_lw' + >>> _normalize_type_for_comparison('type(ty_rad_lw)') + 'ty_rad_lw' + >>> _normalize_type_for_comparison('Type( Ty_Rad_LW )') + 'ty_rad_lw' + >>> _normalize_type_for_comparison('class(ty_rad_lw)') + 'ty_rad_lw' + >>> _normalize_type_for_comparison('external:mpi_f08:mpi_comm') + 'mpi_comm' + >>> _normalize_type_for_comparison('external : esmf_mod : esmf_clock') + 'esmf_clock' + """ + s = type_str.strip().lower() + s = re.sub(r'\s+', ' ', s) + s = _EXTERNAL_TYPE_PREFIX_RE.sub('', s) + m = _DDT_WRAPPER_RE.match(s) + if m: + s = m.group(1).strip() + if s == 'doubleprecision': + s = 'double precision' + return s + + +def _check_arg_attributes( + sub_name: str, + arg_name: str, + meta_var, + fort: _ArgAttrs, +) -> List[str]: + """Compare per-attribute consistency for one dummy argument. + + Compared attributes: ``intent``, ``type``, ``kind``, dimension + *rank* (number of dims). ``character`` kinds treat ``len=*`` on + either side as a wildcard against any concrete ``len=N`` / + ``len=:``. The ``optional`` attribute is checked at the call site + in :func:`_validate_scheme` because one direction emits a warning + (logger-dependent) rather than an error. + + Returns a list of error message strings (empty on full match). + """ + errs: List[str] = [] + prefix = "Arg '{}' on '{}': ".format(arg_name, sub_name) + + # intent — only check when the metadata actually declares one (it's + # required for scheme vars but we don't reach this helper for + # non-scheme tables anyway). + meta_intent = (meta_var.intent or '').lower() + if meta_intent and fort.intent and meta_intent != fort.intent: + errs.append( + prefix + "intent mismatch (metadata={!r}, Fortran={!r})".format( + meta_intent, fort.intent, + ) + ) + elif meta_intent and not fort.intent: + errs.append( + prefix + "intent declared as {!r} in metadata but absent " + "from Fortran declaration".format(meta_intent) + ) + + # type — case-insensitive, with normalisation that puts intrinsic, + # DDT, and external types on equal footing. See + # :func:`_normalize_type_for_comparison` for the rules. + meta_type = (meta_var.type or '').strip() + if meta_type and fort.type_: + meta_norm = _normalize_type_for_comparison(meta_type) + fort_norm = _normalize_type_for_comparison(fort.type_) + if meta_norm != fort_norm: + errs.append( + prefix + "type mismatch (metadata={!r}, Fortran={!r})".format( + meta_var.type, fort.type_, + ) + ) + + # kind — case-insensitive. Empty matches empty. character has + # the ``len=*`` wildcard on either side. + meta_kind = (meta_var.kind or '').strip().lower() + fort_kind = fort.kind_ + if meta_type == 'character' or fort.type_ == 'character': + if meta_kind == 'len=*' or fort_kind == 'len=*': + pass # wildcard match + elif meta_kind != fort_kind: + errs.append( + prefix + "character length mismatch " + "(metadata={!r}, Fortran={!r})".format(meta_kind, fort_kind) + ) + else: + if meta_kind != fort_kind: + errs.append( + prefix + "kind mismatch (metadata={!r}, Fortran={!r})".format( + meta_kind or '', fort_kind or '', + ) + ) + + # rank — number of dimensions. Metadata stores them as a list of + # standard-name strings; Fortran rank is counted from the decl. + meta_rank = len(meta_var.dimensions or []) + if meta_rank != fort.rank: + errs.append( + prefix + "rank mismatch (metadata declares {} dimension(s) {}, " + "Fortran declares rank {})".format( + meta_rank, list(meta_var.dimensions or []), fort.rank, + ) + ) + + return errs + + _FORTRAN_EXTENSIONS = ('.F90', '.f90', '.F', '.f') diff --git a/capgen-ng/generator/datatable.py b/capgen-ng/generator/datatable.py index d838f82c..9053b52c 100644 --- a/capgen-ng/generator/datatable.py +++ b/capgen-ng/generator/datatable.py @@ -24,7 +24,7 @@ /abs/path/ccpp_kinds.F90 - /abs/path/ccpp_static_api.F90 + /abs/path/_ccpp_cap.F90 /abs/path/ccpp__cap.F90 @@ -135,7 +135,7 @@ def _build_capgen_files( ```` and are emitted by :func:`_build_inspection_files`. ``host_file_paths`` lists files generated for the host-facing API. In - capgen-ng this is the static API (``ccpp_static_api.F90``); the section is + capgen-ng this is the static API (``_ccpp_cap.F90``); the section is emitted unconditionally (possibly empty) to keep the schema stable. """ capgen_files = ET.SubElement(root, 'capgen_files') @@ -401,7 +401,7 @@ def write_datatable( Output directory. host_file_paths : list of str, optional Absolute paths to host-facing API files (capgen-ng emits - ``ccpp_static_api.F90`` here). The ```` section is + ``_ccpp_cap.F90`` here). The ```` section is always written (possibly empty). scheme_file_paths : list of str, optional Absolute paths to the Fortran source files (``.F90`` / ``.F`` / ...) diff --git a/capgen-ng/generator/static_api.py b/capgen-ng/generator/static_api.py index 65aa3e7d..66ff264b 100644 --- a/capgen-ng/generator/static_api.py +++ b/capgen-ng/generator/static_api.py @@ -1104,7 +1104,7 @@ def _generate_static_api( # With --no-host-introspection, each routine retains its signature # but the body is replaced with an errflg=1 stub (or, for # suite_list, an error_unit write + empty allocation), shrinking - # ccpp_static_api.F90 dramatically for multi-suite builds. + # _ccpp_cap.F90 dramatically for multi-suite builds. lines.extend(_suite_list_subroutine( suite_names, stub_body=no_host_introspection, )) diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 900a7175..51567380 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -15,7 +15,7 @@ ``_physics_final`` — dispatch by ``group_name`` to the appropriate group cap subroutine. -The static API (``ccpp_static_api.F90``) dispatches by ``suite_name`` to +The static API (``_ccpp_cap.F90``) dispatches by ``suite_name`` to these subroutines. """ diff --git a/doc/briefing.md b/doc/briefing.md index 69f60336..35c68808 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -65,8 +65,9 @@ to the right scheme. ### 3.2 Three layers of generated cap -- **Static API** (`ccpp_static_api.F90`) — public entry points; one per - build. Dispatches by `suite_name` → suite cap. +- **Static API** (`_ccpp_cap.F90`) — public entry points; one per + host build (filename and module name derived from `--host-name`). + Dispatches by `suite_name` → suite cap. - **Suite cap** (`ccpp__cap.F90`) — per-suite state machine, plus dispatch by `group_name` → group cap. Suite-owned interstitial data lives in a sibling `ccpp__data.F90`. @@ -108,7 +109,13 @@ For each scheme arg: Fortran parsing. - `ccpp_validator.py` — the standalone Fortran-vs-metadata checker. The ONE place capgen-ng parses Fortran. Run by developers / - CMake before generation. + CMake before generation. Checks per-arg `intent`, `type`, `kind`, + and dimension rank; `character len=*` is a wildcard; DDT and + `external::` types compare against the Fortran + `type(name)` wrapper. `optional` is asymmetric: metadata + `optional=True` against a Fortran-required dummy is an error; the + reverse is a warning. See `doc/migration.md` §7 for the full + rule table. Both share the same metadata-parsing library (`metadata/`). @@ -341,7 +348,7 @@ don't rebuild downstream objects unless something actually moved. being patched around in the host). Most of the `phys_ps` group now builds end-to-end via `--legacy-mode`. - **`--no-host-introspection`** (new, 2026-05-14): stubs the bodies of - the five suite-introspection routines in `ccpp_static_api.F90`, + the five suite-introspection routines in `_ccpp_cap.F90`, shrinking the file from ~33k lines to ~800 for the 10-suite SCM build (the introspection case-blocks were making even `-O1` compilation effectively hang). Signatures stay so existing host diff --git a/doc/constituents.md b/doc/constituents.md index c7581801..65f906eb 100644 --- a/doc/constituents.md +++ b/doc/constituents.md @@ -52,10 +52,11 @@ In capgen-ng, the constituent layer has three concerns: All constituent state lives in **one generated module**: `ccpp_host_constituents.F90` (one per generator run, emitted only when at least one suite touches constituent state). Public symbols from this module -are also re-exported by `ccpp_static_api`, so most host code only needs +are also re-exported by `_ccpp_cap` (the per-host static API; filename +and module name derived from `--host-name`), so most host code only needs ```fortran -use ccpp_static_api, only: ccpp_register_constituents, ccpp_initialize_constituents, & +use _ccpp_cap, only: ccpp_register_constituents, ccpp_initialize_constituents, & ccpp_constituents_array, ccpp_const_get_index, ... ``` @@ -355,7 +356,7 @@ of its own. ## 5. Public API reference All routines below live in `ccpp_host_constituents` and are also -re-exported from `ccpp_static_api` for convenience. The dummy-argument +re-exported from `_ccpp_cap` for convenience. The dummy-argument name `instance_number` is the **standard name**; the actual emitted dummy uses the host's local name for it (typically also `instance_number` or `inst_num`). @@ -537,7 +538,7 @@ module ccpp_host_constituents public :: index_of_ public :: ccpp_model_const_stdnames ! parameter array - ! ----- public routines (also re-exported from ccpp_static_api) -------- + ! ----- public routines (also re-exported from _ccpp_cap) -------- public :: ccpp_register_constituents public :: ccpp_initialize_constituents public :: ccpp_is_scheme_constituent @@ -982,10 +983,12 @@ end module ccpp_host_constituents ```fortran subroutine my_host_run() - use ccpp_static_api, only: ccpp_register, ccpp_register_constituents, & - ccpp_initialize_constituents, ccpp_init, & - ccpp_physics_run, ccpp_final, & - ccpp_deallocate_dynamic_constituents + ! my_host_ccpp_cap is the per-host static API module + ! (filename and module name derived from --host-name). + use my_host_ccpp_cap, only: ccpp_register, ccpp_register_constituents, & + ccpp_initialize_constituents, ccpp_init, & + ccpp_physics_run, ccpp_final, & + ccpp_deallocate_dynamic_constituents use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t type(ccpp_constituent_properties_t), allocatable :: host_consts(:) diff --git a/doc/migration.md b/doc/migration.md index d69f41c3..58b19ccd 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -675,24 +675,67 @@ classification, three reform proposals). ## 7. Validator `capgen-ng/ccpp_validator.py` — standalone Fortran-vs-metadata checker. -Today validates **scheme** metadata against scheme Fortran files -(subroutine signatures, optional args, paren-aware decl splitting). - -Continuation-line handling covers both free-form (`&` at trailing end -of prior line only) and fixed-form / dual-form (`&` at both ends, with -the leading marker at column 6). Comment-only and blank lines -interleaved between continuation lines are skipped as Fortran 90+ -permits. - -When the signature parser finds a subroutine but extracts zero args -while metadata declares many, the "Argument count mismatch" error -appends a HINT pointing at the parser rather than masquerading as a -real mismatch — common cause is an unsupported signature feature. - -**Known gap**: host-metadata validation is not yet implemented. When -invoked with non-scheme `.meta` files, the validator silently filters -to zero schemes and reports "Validation passed." Slated for revisit -after the e2e test suite settles (`unit_conv` + `variable_transform` +Validates **scheme** metadata against scheme Fortran files. + +### 7.1 What the validator checks + +For every `(scheme, phase)` declared in the supplied `.meta` files: + +1. The Fortran subroutine `_` **exists** in the source + tree (auto-discovered via `source_path` on the table, or supplied + explicitly with `--source-files`). +2. **Argument count** — the number of dummy arguments matches the + metadata, after subtracting any optional-only-in-Fortran args (see + §7.2). +3. **Argument names** — the set of metadata `local_name` values + matches the Fortran dummy-arg list (order-insensitive, + case-insensitive). +4. For every argument present on **both sides**, per-attribute + consistency: + + | Attribute | Behavior | + |------------|----------| + | `intent` | Strict match (`in` / `out` / `inout`). Metadata declares it but Fortran omits → error. | + | `type` | Case-insensitive match. `double precision` / `doubleprecision` / `double precision` are normalized to the same form. DDT names match the Fortran `type(name)` / `class(name)` wrapper — metadata `type = ty_rad_lw` matches Fortran `type(ty_rad_lw)`. External types match by typename — metadata `type = external:mpi_f08:mpi_comm` matches Fortran `type(mpi_comm)` (the module qualifier is metadata-only). | + | `kind` | Case-insensitive match. **Character `len=*` is a wildcard** on either side — matches any concrete `len=N` or `len=:`. | + | `rank` | Number of dimensions only. Reads both `dimension(...)` line attributes and var-attached `foo(:,:)` syntax. Per-dimension bound comparison is NOT done. | + +### 7.2 Asymmetric `optional` rule + +| Metadata | Fortran | Outcome | +|-----------------|--------------------------|----------| +| (absent) | `optional` | warning | +| (absent) | required (no `optional`) | error | +| `optional=False`| `optional` | warning | +| `optional=False`| required (no `optional`) | OK | +| `optional=True` | `optional` | OK | +| `optional=True` | required (no `optional`) | **error**| + +Reason for the asymmetry: a metadata-side `optional=True` is a promise +to the cap that the value may be absent at the call site. If Fortran +requires the dummy, the cap's `present()` check is invalid. The +reverse direction (Fortran allows optional, metadata always passes it) +is a valid subset of the Fortran contract — the arg is always present +and any optional Fortran dummy can accept that — so we warn but don't +fail the build. + +### 7.3 Continuation-line handling + +Covers both free-form (`&` at trailing end of prior line only) and +fixed-form / dual-form (`&` at both ends, with the leading marker at +column 6). Comment-only and blank lines interleaved between +continuation lines are skipped as Fortran 90+ permits. When the +signature parser finds a subroutine but extracts zero args while +metadata declares many, the "Argument count mismatch" error appends a +HINT pointing at the parser rather than masquerading as a real +mismatch — common cause is an unsupported signature feature. + +### 7.4 Known gap + +Host-metadata validation is not yet implemented. When invoked with +non-scheme `.meta` files, the validator silently filters to zero +schemes and reports "Validation passed." Slated for revisit after +the e2e test suite settles (`unit_conv` + `variable_transform` complete). See `project_validator_host_check_deferred.md` (memory). --- diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index 2831871c..3f47aa09 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -51,9 +51,20 @@ discrepancies. Run by developers before invoking the generator — e.g., during development or in CI. Does **not** generate any Fortran output. For each scheme phase declared in a `.meta` file, the validator checks that the -corresponding Fortran subroutine: (1) exists in the source tree, (2) has the same number -of dummy arguments, and (3) the argument names match the `local_name` values in the -metadata (order-insensitive). +corresponding Fortran subroutine: (1) exists in the source tree, (2) has the same +number of dummy arguments, (3) the argument names match the `local_name` values in +the metadata (order-insensitive), and (4) for every argument present in both sides, +the `intent`, `type`, `kind`, and dimension *rank* agree. `character` arguments +treat `len=*` on either side as a wildcard. DDT names compare against the Fortran +`type(name)` / `class(name)` wrapper; `external::` metadata +compares against the Fortran `type(typename)` (the module qualifier is +metadata-only). + +The `optional` attribute is asymmetric: metadata `optional=True` paired with a +Fortran dummy that is *not* declared `optional` is a hard error (the cap's +`present()` check would be invalid on a Fortran-required dummy); the reverse +direction (Fortran-only `optional`) is a warning, since always passing the arg is +a valid subset of the Fortran contract. Fortran source files can be supplied explicitly on the CLI (`--source-files`). When omitted, the validator auto-discovers the Fortran source for each scheme table using the @@ -422,7 +433,7 @@ variables from `_host_data` since the host owns those. All three levels are fully auto-generated. No hand-written components in the cap layer. -### 6.1 Static API (`ccpp_static_api.F90`) +### 6.1 Static API (`_ccpp_cap.F90`) - Imports all host DDTs and flat fields via `module use` (resolved from host metadata) - Does not USE `ccpp_kinds` directly: the static API has no kind-typed declarations of @@ -1006,13 +1017,13 @@ All files are written to `--output-root`. | File | Contents | |---|---| | `ccpp_kinds.F90` | Kind parameter definitions. **Always generated.** Re-exports specs from `iso_fortran_env` (default) or host-supplied modules as `integer, parameter, public :: = `. If no `--kind-type` is supplied, `kind_phys=iso_fortran_env:REAL64` is injected automatically (logged at INFO). | -| `ccpp_static_api.F90` | Static API — host imports, suite_name dispatch | +| `_ccpp_cap.F90` | Static API — host imports, suite_name dispatch (filename and module name derived from `--host-name`) | | `ccpp__cap.F90` | Suite cap — suite data import, state machine, group dispatch | | `ccpp___cap.F90` | Group cap — scheme call sites, state array, optionals, transformations. USEs `ccpp_kinds` for any kind referenced in transformation temporaries. | | `ccpp__data.F90` | Suite data module — framework-owned interstitial DDT. USEs `ccpp_kinds` for any kind referenced in suite-var declarations. | | `ccpp__types.F90` | Shared cap types — optional pointer wrapper types, transformation locals. USEs `ccpp_kinds` for any kind referenced in pointer wrappers. | | `ccpp__data.meta` | Generated `type = suite` metadata table — pairs with `ccpp__data.F90` (output-only, for inspection) | -| `datatable.xml` | Generator database for `ccpp_datafile.py` queries. `ccpp_kinds.F90` and `ccpp_static_api.F90` appear in `...` (matches original capgen). | +| `datatable.xml` | Generator database for `ccpp_datafile.py` queries. `ccpp_kinds.F90` appears under ``; `_ccpp_cap.F90` appears under ``. | `ccpp_kinds.F90` is a dependency of all generated Fortran files that reference any kind parameter (group cap, suite types, suite data). The static API and suite cap have no kind references and do not USE it. @@ -1077,8 +1088,10 @@ The XML structure is: /abs/path/ccpp_kinds.F90 - /abs/path/ccpp_static_api.F90 + + /abs/path/_ccpp_cap.F90 + /abs/path/ccpp__cap.F90 ... diff --git a/unit-tests/sample_files/scheme_multipart_correct.F90 b/unit-tests/sample_files/scheme_multipart_correct.F90 index e56fec95..881427ae 100644 --- a/unit-tests/sample_files/scheme_multipart_correct.F90 +++ b/unit-tests/sample_files/scheme_multipart_correct.F90 @@ -15,11 +15,12 @@ end subroutine temp_calc_adjust_init subroutine temp_calc_adjust_run(im, timestep, temp, & errmsg, errflg) - integer, intent(in) :: im - real, intent(in) :: timestep - real, intent(inout) :: temp(:,:) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg + use ccpp_kinds, only: kind_phys + integer, intent(in) :: im + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(inout) :: temp(:,:) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg errmsg = '' errflg = 0 end subroutine temp_calc_adjust_run diff --git a/unit-tests/test_validator.py b/unit-tests/test_validator.py index 465f5525..677fb38c 100644 --- a/unit-tests/test_validator.py +++ b/unit-tests/test_validator.py @@ -1,6 +1,7 @@ """Unit tests for ccpp_validator.""" import doctest +import logging import os import tempfile import textwrap @@ -741,6 +742,311 @@ def test_returns_none_when_not_found(self): self.assertIsNone(result) +class TestArgAttributeChecks(unittest.TestCase): + """Per-arg type/kind/intent/rank/optional mismatch detection. + + Each test builds a tiny in-memory metadata + Fortran source pair and + runs ``validate`` end-to-end. The Fortran source has the same arg + names as the metadata so the name-set check passes; we deliberately + perturb one attribute per test to exercise one check at a time. + """ + + _BASE_META = ( + '[ccpp-table-properties]\n' + ' name = s\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = s_run\n' + ' type = scheme\n' + '[ a ]\n' + ' standard_name = a_std\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = {intent_a}\n' + '[ b ]\n' + ' standard_name = b_std\n' + ' units = K\n' + ' dimensions = {dims_b}\n' + ' type = {type_b}\n' + ' kind = {kind_b}\n' + ' intent = {intent_b}\n' + ' optional = {optional_b}\n' + ) + + def _run(self, meta_text, f90_text): + with tempfile.TemporaryDirectory() as d: + meta_path = os.path.join(d, 's.meta') + f90_path = os.path.join(d, 's.F90') + with open(meta_path, 'w') as fh: + fh.write(meta_text) + with open(f90_path, 'w') as fh: + fh.write(f90_text) + return validate([meta_path], [f90_path]) + + def _meta(self, **overrides): + defaults = dict(intent_a='in', dims_b='()', type_b='real', + kind_b='kind_phys', intent_b='in', optional_b='False') + defaults.update(overrides) + return self._BASE_META.format(**defaults) + + _F90_TEMPLATE = ( + 'module m\n' + 'contains\n' + ' subroutine s_run({sig})\n' + ' use ccpp_kinds, only: kind_phys\n' + '{decls}' + ' end subroutine s_run\n' + 'end module m\n' + ) + + def _f90(self, sig, decls): + return self._F90_TEMPLATE.format(sig=sig, decls=decls) + + def test_clean_match_no_errors(self): + errs = self._run( + self._meta(), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), intent(in) :: b\n' + )), + ) + self.assertEqual(errs, []) + + def test_intent_mismatch(self): + errs = self._run( + self._meta(intent_b='in'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), intent(out) :: b\n' + )), + ) + self.assertTrue(any("intent mismatch" in e and "'b'" in e for e in errs), + msg=errs) + + def test_type_mismatch(self): + errs = self._run( + self._meta(type_b='real'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' integer, intent(in) :: b\n' + )), + ) + self.assertTrue(any("type mismatch" in e and "'b'" in e for e in errs), + msg=errs) + + def test_kind_mismatch(self): + errs = self._run( + self._meta(kind_b='kind_phys'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real, intent(in) :: b\n' # missing kind + )), + ) + self.assertTrue(any("kind mismatch" in e and "'b'" in e for e in errs), + msg=errs) + + def test_character_len_star_is_wildcard(self): + errs = self._run( + self._meta(type_b='character', kind_b='len=512'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' character(len=*), intent(in) :: b\n' + )), + ) + char_errs = [e for e in errs if "'b'" in e and 'character' in e] + self.assertEqual(char_errs, [], msg=errs) + + def test_rank_mismatch(self): + errs = self._run( + self._meta(dims_b='(d1, d2)'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), intent(in) :: b\n' # rank 0, metadata says rank 2 + )), + ) + self.assertTrue(any("rank mismatch" in e and "'b'" in e for e in errs), + msg=errs) + + def test_rank_via_var_attached_dims(self): + errs = self._run( + self._meta(dims_b='(d1, d2)'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), intent(in) :: b(:,:)\n' + )), + ) + self.assertEqual(errs, []) + + def test_metadata_optional_but_fortran_required_is_error(self): + # Metadata says optional=True, Fortran doesn't carry the + # 'optional' attribute -> hard error (cap would emit invalid + # present() checks on a required dummy). + errs = self._run( + self._meta(optional_b='True'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), intent(in) :: b\n' + )), + ) + self.assertTrue( + any("optional=True" in e and "'b'" in e for e in errs), + msg=errs, + ) + + def test_ddt_metadata_bare_name_matches_fortran_type_wrapper(self): + # Metadata: type = ty_rad_lw (bare DDT name). + # Fortran: type(ty_rad_lw), intent(in) :: b + # These should compare equal after type-name normalisation. + errs = self._run( + self._meta(type_b='ty_rad_lw', kind_b=''), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' type(ty_rad_lw), intent(in) :: b\n' + )), + ) + b_errs = [e for e in errs if "'b'" in e] + self.assertEqual(b_errs, [], msg=errs) + + def test_ddt_class_wrapper_matches_metadata_bare_name(self): + # Fortran polymorphic wrapper: class(...) on the Fortran side + # still matches a bare DDT name in metadata. + errs = self._run( + self._meta(type_b='ty_rad_lw', kind_b=''), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' class(ty_rad_lw), intent(in) :: b\n' + )), + ) + b_errs = [e for e in errs if "'b'" in e] + self.assertEqual(b_errs, [], msg=errs) + + def test_ddt_name_mismatch_is_error(self): + # Different DDT names on each side -> error. + errs = self._run( + self._meta(type_b='ty_rad_lw', kind_b=''), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' type(ty_rad_sw), intent(in) :: b\n' + )), + ) + self.assertTrue( + any("type mismatch" in e and "'b'" in e for e in errs), + msg=errs, + ) + + def test_external_type_matches_fortran_bare_typename(self): + # Metadata: type = external:mpi_f08:mpi_comm (module + name). + # Fortran: type(mpi_comm), intent(in) :: b + # The module is metadata-only; Fortran sees the bare typename. + errs = self._run( + self._meta(type_b='external:mpi_f08:mpi_comm', kind_b=''), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' type(mpi_comm), intent(in) :: b\n' + )), + ) + b_errs = [e for e in errs if "'b'" in e] + self.assertEqual(b_errs, [], msg=errs) + + def test_external_type_mismatched_typename_is_error(self): + errs = self._run( + self._meta(type_b='external:mpi_f08:mpi_comm', kind_b=''), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' type(mpi_request), intent(in) :: b\n' + )), + ) + self.assertTrue( + any("type mismatch" in e and "'b'" in e for e in errs), + msg=errs, + ) + + def test_fortran_optional_but_metadata_required_is_warning(self): + # Reverse direction: metadata=False (default), Fortran=optional + # -> NOT an error. The cap always passes the arg; that's a + # valid subset of the Fortran contract. A warning is emitted. + import io + log_buf = io.StringIO() + handler = logging.StreamHandler(log_buf) + handler.setLevel(logging.WARNING) + log = logging.getLogger('test_validator_fopt_metareq') + log.addHandler(handler) + log.setLevel(logging.WARNING) + try: + with tempfile.TemporaryDirectory() as d: + meta_path = os.path.join(d, 's.meta') + f90_path = os.path.join(d, 's.F90') + with open(meta_path, 'w') as fh: + fh.write(self._meta()) + with open(f90_path, 'w') as fh: + fh.write(self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' real(kind=kind_phys), optional, intent(in) :: b\n' + ))) + errs = validate([meta_path], [f90_path], logger=log) + finally: + log.removeHandler(handler) + # No errors. + b_errs = [e for e in errs if "'b'" in e] + self.assertEqual(b_errs, [], msg=errs) + # But a warning for 'b'. + self.assertIn("Fortran argument 'b'", log_buf.getvalue()) + self.assertIn("optional", log_buf.getvalue()) + + +class TestFortranOnlyOptionalWarning(unittest.TestCase): + """A Fortran-optional arg absent from metadata triggers a logger.warning + but no validation error.""" + + _META = ( + '[ccpp-table-properties]\n' + ' name = s\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = s_run\n' + ' type = scheme\n' + '[ a ]\n' + ' standard_name = a_std\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = integer\n' + ' intent = in\n' + ) + + _F90 = ( + 'module m\n' + 'contains\n' + ' subroutine s_run(a, b)\n' + ' integer, intent(in) :: a\n' + ' integer, optional, intent(in) :: b\n' + ' end subroutine s_run\n' + 'end module m\n' + ) + + def test_warning_and_no_error(self): + import io + with tempfile.TemporaryDirectory() as d: + meta_path = os.path.join(d, 's.meta') + f90_path = os.path.join(d, 's.F90') + with open(meta_path, 'w') as fh: + fh.write(self._META) + with open(f90_path, 'w') as fh: + fh.write(self._F90) + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setLevel(logging.WARNING) + log = logging.getLogger('test_validator_optional_warn') + log.addHandler(handler) + log.setLevel(logging.WARNING) + try: + errs = validate([meta_path], [f90_path], logger=log) + finally: + log.removeHandler(handler) + self.assertEqual(errs, []) + self.assertIn("Optional Fortran argument 'b'", stream.getvalue()) + + def load_tests(loader, tests, ignore): tests.addTests(doctest.DocTestSuite(val_mod)) return tests From 4ddb0e06601f6d09966a9f206aac4e7a2e34cae8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 May 2026 15:43:21 -0600 Subject: [PATCH 34/74] Bug fix validate host active attribute against scheme optional attribute --- capgen-ng/generator/suite_resolver.py | 24 ++++++++ doc/migration.md | 32 ++++++++++ unit-tests/test_static_api.py | 53 ++++++++++++++-- unit-tests/test_suite_resolver.py | 87 +++++++++++++++++++++++++++ 4 files changed, 192 insertions(+), 4 deletions(-) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 94bcb44f..f19ee381 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -1348,6 +1348,30 @@ def _resolve_one_arg( # active is a host-model-only attribute; read it from the host entry only. active = host_entry.active if host_entry is not None else '' + # Coherence check: a host-declared active expression means the host's + # variable is only valid when the condition holds. The generator's + # pointer-association pattern (transform_case 2 / 4) handles that — + # but only fires when the scheme arg is itself ``optional``. A + # non-optional scheme arg would be passed unconditionally, reading + # host memory regardless of the active condition. Reject the + # incoherent combination at resolution time. + if active and not optional: + raise CCPPError( + "Scheme '{scheme}', phase '{phase}', arg '{arg}' " + "(standard_name '{std}'): host metadata declares " + "active = ({active}) on the matching variable, but the " + "scheme metadata does not declare this argument as " + "optional. An ``active`` condition means the host's " + "variable is only valid when the condition holds; the " + "generated cap can only honor that contract via the " + "optional/pointer-association pattern. Add " + "``optional = True`` to the scheme metadata entry (and " + "``optional`` to the matching Fortran dummy declaration), " + "or remove the ``active`` attribute from the host entry.".format( + scheme=scheme_name, phase=phase, arg=local, + std=std_name, active=active, + ) + ) suite_var: Optional[SuiteVar] = suite_vars.get(std_name) if host_entry is not None and suite_var is None: diff --git a/doc/migration.md b/doc/migration.md index 58b19ccd..cf639fe7 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -82,6 +82,38 @@ Inside a `[ var_name ]` section. All optional. | `molar_mass` | float | `0.0` | Scheme metadata only. | | `diagnostic_name` | str | (defaults to `local_name`) | Host-tooling hint; mutually exclusive with `diagnostic_name_fixed`. | +#### 1.3.1 `active` requires the scheme arg to be `optional` + +When a host variable carries `active = ()`, the host's +contract with the cap is "this variable's storage is only valid when +the condition holds". capgen-ng honors that contract via the +pointer-association pattern: at every call site that consumes the var, +the cap emits + +```fortran +if () then + ptr%ptr => () +else + nullify(ptr%ptr) +end if +call scheme(..., my_arg=ptr%ptr, ...) +``` + +The pointer-association path is only safe when the scheme's Fortran +dummy declaration is itself `optional`. Therefore: **every scheme arg +whose host counterpart carries `active = (...)` MUST declare +`optional = True` in its scheme metadata, and the matching Fortran +dummy MUST carry the `optional` attribute.** + +The resolver enforces this at code-generation time with a clear error +naming the scheme, the argument, and the host's `active` expression. +If you hit it, two valid fixes: + +- Add `optional = True` to the scheme metadata entry and `optional` + to the Fortran dummy declaration; or +- Remove the `active` attribute from the host metadata entry (only if + the host's variable really is always valid). + ### 1.4 Sliced local names with long subscript indices Local names with array slices may carry CCPP standard names as subscript diff --git a/unit-tests/test_static_api.py b/unit-tests/test_static_api.py index 707fc90b..2fa9ef75 100644 --- a/unit-tests/test_static_api.py +++ b/unit-tests/test_static_api.py @@ -770,9 +770,8 @@ class TestCollectHostIoIncludesActiveExpr(unittest.TestCase): are present.""" def setUp(self): - from test_suite_resolver import _load_scheme_store from metadata.metadata_table import _parse_lines - from metadata.variable_resolver import build_flat_host_dict + from metadata.variable_resolver import build_flat_host_dict, SchemeStore from generator.suite_resolver import resolve_suite import tempfile, logging from generator.suite_xml import parse_suite_xml @@ -781,6 +780,12 @@ def setUp(self): # an active=() expression on another host var. No scheme takes # the flag as a direct argument — without the active-expr walk # in _collect_host_io it would silently disappear. + # + # The matching scheme arg is declared optional, which the + # resolver's active+optional coherence check requires (host + # ``active`` means the host's variable is only valid when the + # condition holds; the cap honors that via the optional/ + # pointer-association pattern). host_src = ''' [ccpp-table-properties] name = active_helper @@ -821,6 +826,43 @@ def setUp(self): type = real kind = kind_phys active = (flag_for_passive_check) +''' + # Inline scheme whose ``temp`` arg is optional — paired with the + # host-side active above. + scheme_src = ''' +[ccpp-table-properties] + name = active_scheme + type = scheme +[ccpp-arg-table] + name = active_scheme_run + type = scheme +[ im ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in +[ temp ] + standard_name = air_temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + optional = True +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out ''' from test_suite_resolver import _SAMPLES_DIR from metadata.metadata_table import parse_metadata_file @@ -831,11 +873,14 @@ def setUp(self): ctrl_only = [t for t in ctrl_tbls if t.table_type == 'control'] self.hd = build_flat_host_dict(host_tbls, ctrl_only, []) - store = _load_scheme_store() + scheme_tbls = _parse_lines( + scheme_src.splitlines(keepends=True), 's.meta', + ) + store = SchemeStore.build_from(scheme_tbls) suite_xml = ( '\n' '\n' - ' temp_calc_adjust\n' + ' active_scheme\n' '\n' ) with tempfile.TemporaryDirectory() as tmp: diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 220f8ae0..26baaac7 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -1235,6 +1235,93 @@ def test_optional_sets_ptr_name(self): self.assertEqual(arg.transform_case, 2) +######################################################################## +# Tests: active + optional coherence +######################################################################## + +class TestActiveRequiresOptional(unittest.TestCase): + """When the host declares ``active = ()`` on a variable, any + scheme arg matching that variable must declare ``optional = True``. + The cap honors the active condition via pointer-association, which + is only valid when the scheme's Fortran dummy is optional.""" + + _HOST_SRC = ( + '[ccpp-table-properties]\n' + ' name = active_host\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = active_host\n' + ' type = host\n' + '[ ncols ]\n' + ' standard_name = horizontal_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ nlev ]\n' + ' standard_name = vertical_layer_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ flag_passive ]\n' + ' standard_name = flag_for_passive_check\n' + ' units = flag\n' + ' dimensions = ()\n' + ' type = logical\n' + '[ gt0 ]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real\n' + ' kind = kind_phys\n' + ' active = (flag_for_passive_check)\n' + ) + + def _host_dict(self): + from metadata.metadata_table import _parse_lines + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + host_tbls = _parse_lines( + self._HOST_SRC.splitlines(keepends=True), 'h.meta', + ) + ctrl_only = [t for t in ctrl_tbls if t.table_type == 'control'] + return build_flat_host_dict(host_tbls, ctrl_only, []) + + def _scheme_var(self, optional): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar('temp', ctx) + v.set_attr('standard_name', 'air_temperature', ctx) + v.set_attr('units', 'K', ctx) + v.set_attr('dimensions', + '(horizontal_dimension, vertical_layer_dimension)', ctx) + v.set_attr('type', 'real', ctx) + v.set_attr('kind', 'kind_phys', ctx) + v.set_attr('intent', 'inout', ctx) + if optional: + v.set_attr('optional', 'True', ctx) + return v + + def test_optional_scheme_arg_passes(self): + hd = self._host_dict() + arg = _resolve_one_arg(self._scheme_var(optional=True), 'run', hd, + {}, 'my_scheme', set()) + self.assertEqual(arg.active, '(flag_for_passive_check)') + self.assertTrue(arg.is_optional) + # Cap uses pointer-association: transform_case 2 (or 4 with + # transform). Both are pointer-pattern paths. + self.assertIn(arg.transform_case, (2, 4)) + + def test_required_scheme_arg_raises(self): + hd = self._host_dict() + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(self._scheme_var(optional=False), 'run', hd, + {}, 'my_scheme', set()) + msg = str(cm.exception) + self.assertIn("my_scheme", msg) + self.assertIn("air_temperature", msg) + self.assertIn("(flag_for_passive_check)", msg) + self.assertIn("optional", msg.lower()) + + ######################################################################## # Tests: vertical-flip transform (top_at_one mismatch) ######################################################################## From 780562dedf02a14a15349afaab50a65c15dc5b5a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 May 2026 16:05:29 -0600 Subject: [PATCH 35/74] Rename all remnants of ccpp_static_api to _ccpp_cap --- capgen-ng/ccpp_capgen_ng.py | 6 +- .../generator/{static_api.py => host_cap.py} | 10 +-- capgen-ng/generator/suite_resolver.py | 6 +- doc/briefing.md | 15 +++- doc/briefing_pm.md | 2 +- doc/constituents_overhaul.md | 2 +- doc/migration.md | 2 +- doc/redesign_prompt.md | 4 +- unit-tests/test_ccpp_datafile.py | 6 +- unit-tests/test_datatable.py | 12 ++-- .../{test_static_api.py => test_host_cap.py} | 70 +++++++++---------- unit-tests/test_integration.py | 16 ++--- unit-tests/test_suite_resolver.py | 4 +- 13 files changed, 83 insertions(+), 72 deletions(-) rename capgen-ng/generator/{static_api.py => host_cap.py} (99%) rename unit-tests/{test_static_api.py => test_host_cap.py} (95%) diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index 15dd726f..cf1cad01 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -95,7 +95,7 @@ from generator.suite_data import write_suite_data, write_suite_meta from generator.suite_cap import write_suite_cap from generator.suite_types import write_suite_types -from generator.static_api import write_static_api +from generator.host_cap import write_host_cap from generator.host_constituents import write_host_constituents from generator.datatable import write_datatable @@ -943,8 +943,8 @@ def capgen( trace=trace, ) - # ---- static API (one file for all suites) ------------------------------ - write_static_api( + # ---- host cap (one file for all suites) -------------------------------- + write_host_cap( host_name, suite_names, suite_resolutions, output_root, host_dict, scheme_store, logger=log, diff --git a/capgen-ng/generator/static_api.py b/capgen-ng/generator/host_cap.py similarity index 99% rename from capgen-ng/generator/static_api.py rename to capgen-ng/generator/host_cap.py index 66ff264b..1c4dabd5 100644 --- a/capgen-ng/generator/static_api.py +++ b/capgen-ng/generator/host_cap.py @@ -701,7 +701,7 @@ def _suite_list_subroutine( ``error_unit`` and return an empty list. ``ccpp_physics_suite_list`` has no errflg/errmsg arguments, so ``error_unit`` is the only available error channel. The module-level ``use iso_fortran_env`` - that this references is added by ``_generate_static_api`` when + that this references is added by ``_generate_host_cap`` when ``no_host_introspection`` is on. """ i1 = _INDENT @@ -970,7 +970,7 @@ def _suite_io_subroutine( # Module generator ######################################################################## -def _generate_static_api( +def _generate_host_cap( host_name: str, suite_names: List[str], suite_resolutions: List[SuiteResolution], @@ -1132,7 +1132,7 @@ def _generate_static_api( # Public API ######################################################################## -def write_static_api( +def write_host_cap( host_name: str, suite_names: List[str], suite_resolutions: List[SuiteResolution], @@ -1173,7 +1173,7 @@ def write_static_api( Signatures remain so existing callers still link. Use this to shrink ``_ccpp_cap.F90`` from ~33k lines to ~800 for multi-suite builds where the introspection case-blocks make - ``-O3`` compilation impractical. + even ``-O1`` compilation impractical. Returns ------- @@ -1184,7 +1184,7 @@ def write_static_api( filename = '{}_ccpp_cap.F90'.format(host_name) out_path = os.path.join(output_root, filename) - lines = _generate_static_api( + lines = _generate_host_cap( host_name, suite_names, suite_resolutions, host_dict, scheme_store, no_host_introspection=no_host_introspection, trace=trace, diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index f19ee381..33ede96e 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -900,7 +900,7 @@ class ResolvedArg: # ``ccpp_constituents``). These are not USE'd from any module — the # value is reached via the per-instance constituent object — but # they're surfaced as inputs by the introspection routines in - # :mod:`generator.static_api`. Replaces the older trick of stuffing + # :mod:`generator.host_cap`. Replaces the older trick of stuffing # them into :attr:`used_dim_std_names`. used_const_dim_std_names: Set[str] = field(default_factory=set) @@ -1632,7 +1632,7 @@ def _const_dim_part( The trailing dim ``number_of_ccpp_constituents`` is emitted as ``':'`` (whole-axis slice). The std name is added to ``used_const_dim_std`` so the introspection routine - (:func:`generator.static_api._collect_host_io`) can include it in + (:func:`generator.host_cap._collect_host_io`) can include it in its inputs list — original capgen reports framework-constituent dim names there. No USE statement is emitted for the name: it isn't in host_dict (the framework provides it via the per-instance @@ -1812,7 +1812,7 @@ def _common_kwargs(base_expr, subscript, call_expr, # Framework-constituent dim refs (e.g. number_of_ccpp_constituents) # travel on the dedicated used_const_dim_std_names channel — no # USE statement, but surfaced as inputs by the introspection - # routines in generator.static_api. + # routines in generator.host_cap. return ResolvedArg(**_common_kwargs( base_expr=base_expr, subscript=subscript, call_expr=call_expr, used_host_std=used_host_std, diff --git a/doc/briefing.md b/doc/briefing.md index 35c68808..9774026f 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -339,14 +339,25 @@ don't rebuild downstream objects unless something actually moved. ## 10. Where things stand right now -- **Unit tests**: 1316 passing on `feature/capgen-ng` (as of 2026-05-19). +- **Unit tests**: 1335 passing on `feature/capgen-ng` (as of 2026-05-20). - **End-to-end tests passing**: `advection`, `unit_conv`, `nested_suite`, `variable_transform`, `instances`, `instances_advection`, `ddt`. - **CCPP-SCM**: actively driving development — every build / runtime failure surfaced this week landed as a fix in capgen-ng (rather than being patched around in the host). Most of the `phys_ps` group now - builds end-to-end via `--legacy-mode`. + builds end-to-end via `--legacy-mode`. On 2026-05-20 the + per-arg-attribute validator caught **67 real metadata/Fortran + disagreements** in the SCM physics tree (12 missing `kind = kind_phys` + + 42 intent mismatches + a mix of optional-flag and bare-`real` + cases); all fixed. +- **Validator** now checks per-argument `intent`, `type`, `kind`, and + dimension rank in addition to the original name/count check. + Asymmetric `optional` rule, DDT + `external::` + type normalisation, character `len=*` wildcard. Resolver also + enforces an `active`-vs-`optional` coherence rule: a host variable + with `active = (...)` paired with a non-optional scheme arg is now + a parse-time error. See `doc/migration.md` §7 + §1.3.1. - **`--no-host-introspection`** (new, 2026-05-14): stubs the bodies of the five suite-introspection routines in `_ccpp_cap.F90`, shrinking the file from ~33k lines to ~800 for the 10-suite SCM diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md index ccbbcd2f..4e4ba7e6 100644 --- a/doc/briefing_pm.md +++ b/doc/briefing_pm.md @@ -251,7 +251,7 @@ Features that exist only in capgen-ng (some exist in prebuild): ## 6. Where things stand right now (2026-05-18) -- **Unit tests**: 1319 passing. No known failures. +- **Unit tests**: 1335 passing. No known failures. - **End-to-end tests**: 10 passing — `chunked_data`, `opt_arg`, `nested_suite`, `ddthost`, `instances`, `capgen_ng`, `var_compat`, `advection`, and the new `instances_advection` diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index 2bf6fad8..4ec40b4b 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -458,7 +458,7 @@ adds a code path that most contributors don't read. If we drop it ### 4.8 Capgen-ng: `_FRAMEWORK_CONST_DIM_INPUTS` cleanup (LANDED 2026-05-13) -`generator/static_api.py` no longer carries the hand-curated frozenset of +`generator/host_cap.py` no longer carries the hand-curated frozenset of standard names; framework-constituent dimension references now ride on a dedicated `used_const_dim_std_names` field on `ResolvedArg`. Closes the "hand-curated → structured field" REVISIT note that was in the code. diff --git a/doc/migration.md b/doc/migration.md index cf639fe7..bdd68f4d 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -6,7 +6,7 @@ Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to **capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and `doc/redesign_analysis.md` (analysis of the old systems). -*Last revised: 2026-05-14 (end-of-day).* Current unit-test suite: 1229 passing. +*Last revised: 2026-05-20 (end-of-day).* Current unit-test suite: 1335 passing. **Repository layout** (post-2026-05-13 cleanup): tooling lives under `capgen-ng/` (top-level of this repo). Unit tests live at the top diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index 3f47aa09..316addf4 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -1212,7 +1212,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` `metadata/legacy_compat.py` and tagged `# legacy-compat:` for clean removal once scheme metadata has been migrated. - **`_FRAMEWORK_CONST_DIM_INPUTS` cleanup** — the hand-curated - frozenset in `generator/static_api.py` was removed; framework- + frozenset in `generator/host_cap.py` was removed; framework- constituent dimension references now ride on a dedicated `used_const_dim_std_names` field on `ResolvedArg`. - **`active` expression case-folding** — mixed-case standard names @@ -1285,7 +1285,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` ### Test status -- **Unit tests**: 1229 passing (`python -m pytest unit-tests/`). +- **Unit tests**: 1335 passing (`python -m pytest unit-tests/`). - **End-to-end tests**: `advection`, `unit_conv`, `nested_suite`, `variable_transform`, `instances`, `ddt` covered. SCM running against ccpp-physics is the active driver right now — most of the diff --git a/unit-tests/test_ccpp_datafile.py b/unit-tests/test_ccpp_datafile.py index 937f3ae6..70363bdf 100644 --- a/unit-tests/test_ccpp_datafile.py +++ b/unit-tests/test_ccpp_datafile.py @@ -56,7 +56,7 @@ def _build_datatable(tmpdir, suite_file_paths or ['/out/ccpp_test_simple_cap.F90', '/out/ccpp_test_simple_physics_cap.F90'], tmpdir, - host_file_paths=host_file_paths or ['/out/ccpp_static_api.F90'], + host_file_paths=host_file_paths or ['/out/test_host_ccpp_cap.F90'], scheme_file_paths=scheme_file_paths, dependency_paths=dependency_paths or [], suite_meta_paths=suite_meta_paths, @@ -83,7 +83,7 @@ class TestDatatableReportFileActions(_DTBase): def test_host_files(self): out = datatable_report(self._datatable, DatatableReport('host_files'), ',') - self.assertEqual(out, '/out/ccpp_static_api.F90') + self.assertEqual(out, '/out/test_host_ccpp_cap.F90') def test_suite_files(self): out = datatable_report(self._datatable, @@ -102,7 +102,7 @@ def test_capgen_files_returns_all(self): DatatableReport('capgen_files'), ',') items = out.split(',') self.assertIn('/out/ccpp_kinds.F90', items) - self.assertIn('/out/ccpp_static_api.F90', items) + self.assertIn('/out/test_host_ccpp_cap.F90', items) self.assertIn('/out/ccpp_test_simple_cap.F90', items) def test_separator_honored(self): diff --git a/unit-tests/test_datatable.py b/unit-tests/test_datatable.py index a6c203be..5d70d7c8 100644 --- a/unit-tests/test_datatable.py +++ b/unit-tests/test_datatable.py @@ -31,7 +31,7 @@ def _write(tmpdir, suite_xml='suite_test_simple.xml', utility_paths=None, write_datatable( [suite_resolution], store, - utility_paths or ['/out/ccpp_kinds.F90', '/out/ccpp_static_api.F90'], + utility_paths or ['/out/ccpp_kinds.F90', '/out/test_host_ccpp_cap.F90'], suite_file_paths or ['/out/ccpp_test_simple_cap.F90'], tmpdir, host_file_paths=host_file_paths, @@ -85,7 +85,7 @@ class TestCcppFilesSection(unittest.TestCase): def setUp(self): self._tmpdir = tempfile.mkdtemp() - utils = ['/out/ccpp_kinds.F90', '/out/ccpp_static_api.F90'] + utils = ['/out/ccpp_kinds.F90', '/out/test_host_ccpp_cap.F90'] sfiles = ['/out/ccpp_test_simple_cap.F90', '/out/ccpp_test_simple_data.F90'] path, _, _ = _write(self._tmpdir, utility_paths=utils, suite_file_paths=sfiles) self._root = ET.parse(path).getroot() @@ -105,7 +105,7 @@ def test_utilities_subsection(self): self.assertIsNotNone(utils) files = [f.text for f in utils.findall('file')] self.assertIn('/out/ccpp_kinds.F90', files) - self.assertIn('/out/ccpp_static_api.F90', files) + self.assertIn('/out/test_host_ccpp_cap.F90', files) def test_suite_files_subsection(self): sfiles_elem = self._capgen_files().find('suite_files') @@ -464,7 +464,7 @@ def setUp(self): self._tmpdir = tempfile.mkdtemp() self._path, _, _ = _write( self._tmpdir, - host_file_paths=['/out/ccpp_static_api.F90'], + host_file_paths=['/out/test_host_ccpp_cap.F90'], ) self._root = ET.parse(self._path).getroot() @@ -472,10 +472,10 @@ def tearDown(self): import shutil shutil.rmtree(self._tmpdir) - def test_host_files_contains_static_api(self): + def test_host_files_contains_host_cap(self): host_files = self._root.find('capgen_files').find('host_files') names = [f.text for f in host_files.findall('file')] - self.assertEqual(names, ['/out/ccpp_static_api.F90']) + self.assertEqual(names, ['/out/test_host_ccpp_cap.F90']) class TestVarDictionariesSection(unittest.TestCase): diff --git a/unit-tests/test_static_api.py b/unit-tests/test_host_cap.py similarity index 95% rename from unit-tests/test_static_api.py rename to unit-tests/test_host_cap.py index 2fa9ef75..0a16c802 100644 --- a/unit-tests/test_static_api.py +++ b/unit-tests/test_host_cap.py @@ -1,4 +1,4 @@ -"""Unit tests for generator.static_api.""" +"""Unit tests for generator.host_cap.""" import doctest import os @@ -8,18 +8,18 @@ from metadata.parse_tools import CCPPError from generator.suite_resolver import resolve_suite -from generator.static_api import ( +from generator.host_cap import ( _all_ctrl_args_for_phase, _arg_top_level_name, _build_local_to_std_top_level_map, _collect_host_io, _emit_var_set_loop, - _generate_static_api, + _generate_host_cap, _suite_io_subroutine, _suite_list_subroutine, _suite_part_list_subroutine, _suite_schemes_subroutine, - write_static_api, + write_host_cap, ) from test_suite_resolver import ( _load_full_host_dict, @@ -37,7 +37,7 @@ def _resolve(): def _generate(): suite_resolution = _resolve() - return _generate_static_api('test_host', ['test_simple'], [suite_resolution]) + return _generate_host_cap('test_host', ['test_simple'], [suite_resolution]) class TestAllCtrlArgsForPhase(unittest.TestCase): @@ -52,10 +52,10 @@ def test_only_error_ctrl_args_in_test_case(self): def test_mismatched_lengths_raises(self): from metadata.parse_tools import CCPPError with self.assertRaises(CCPPError): - _generate_static_api('test_host', ['a', 'b'], [_resolve()]) + _generate_host_cap('test_host', ['a', 'b'], [_resolve()]) -class TestGenerateStaticApiModule(unittest.TestCase): +class TestGenerateHostCapModule(unittest.TestCase): """Static API: ccpp_register/init/final are mandatory entry points and are always emitted with the minimal lifecycle signature.""" @@ -103,16 +103,16 @@ def test_contains_block(self): def test_no_constituent_reexport_when_absent(self): # The test_simple fixture has no constituents — host_constituents - # module isn't emitted, so static_api must not USE or re-export it. + # module isn't emitted, so host_cap must not USE or re-export it. self.assertNotIn('use ccpp_host_constituents', self.text) self.assertNotIn('ccpp_register_constituents', self.text) self.assertNotIn('ccpp_initialize_constituents', self.text) -class TestStaticApiConstituentReexport(unittest.TestCase): - """When any suite uses constituent state, static_api USEs +class TestHostCapConstituentReexport(unittest.TestCase): + """When any suite uses constituent state, host_cap USEs ccpp_host_constituents and re-publics every host-facing routine plus - the constituent object so hosts can ``use ccpp_static_api, only: ...`` + the constituent object so hosts can ``use _ccpp_cap, only: ...`` for everything they need from CCPP.""" def setUp(self): @@ -126,7 +126,7 @@ def setUp(self): suite = _parse_suite('suite_consume_constituent.xml') suite_resolution = resolve_suite(suite, store, hd) self.text = '\n'.join( - _generate_static_api('test_host', ['consume_consts'], [suite_resolution], host_dict=hd, + _generate_host_cap('test_host', ['consume_consts'], [suite_resolution], host_dict=hd, scheme_store=store), ) @@ -254,7 +254,7 @@ class TestCcppPhysicsUnknownSuiteErrors(unittest.TestCase): def setUp(self): hd = _load_full_host_dict() suite_resolution = _resolve() - self.text = '\n'.join(_generate_static_api('test_host', ['test_simple'], [suite_resolution], hd)) + self.text = '\n'.join(_generate_host_cap('test_host', ['test_simple'], [suite_resolution], hd)) def test_physics_run_has_default_case_with_errflg(self): run_block_start = self.text.index('subroutine ccpp_physics_run') @@ -282,7 +282,7 @@ def setUp(self): from copy import deepcopy sr2 = deepcopy(suite_resolution) sr2.suite_name = 'suite_b' - lines = _generate_static_api('test_host', ['test_simple', 'suite_b'], [suite_resolution, sr2]) + lines = _generate_host_cap('test_host', ['test_simple', 'suite_b'], [suite_resolution, sr2]) self.text = '\n'.join(lines) def test_both_suites_in_register(self): @@ -295,19 +295,19 @@ def test_both_suite_caps_used(self): self.assertIn('use ccpp_suite_b_cap', self.text) -class TestWriteStaticApi(unittest.TestCase): +class TestWriteHostCap(unittest.TestCase): def test_writes_file(self): suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: - path = write_static_api('test_host', ['test_simple'], [suite_resolution], tmpdir) + path = write_host_cap('test_host', ['test_simple'], [suite_resolution], tmpdir) self.assertTrue(os.path.isfile(path)) self.assertEqual(os.path.basename(path), 'test_host_ccpp_cap.F90') def test_file_content(self): suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: - path = write_static_api('test_host', ['test_simple'], [suite_resolution], tmpdir) + path = write_host_cap('test_host', ['test_simple'], [suite_resolution], tmpdir) with open(path) as fh: content = fh.read() self.assertIn('module test_host_ccpp_cap', content) @@ -319,13 +319,13 @@ def test_creates_output_dir(self): suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: subdir = os.path.join(tmpdir, 'api') - write_static_api('test_host', ['test_simple'], [suite_resolution], subdir) + write_host_cap('test_host', ['test_simple'], [suite_resolution], subdir) self.assertTrue(os.path.isdir(subdir)) def test_returns_absolute_path(self): suite_resolution = _resolve() with tempfile.TemporaryDirectory() as tmpdir: - path = write_static_api('test_host', ['test_simple'], [suite_resolution], tmpdir) + path = write_host_cap('test_host', ['test_simple'], [suite_resolution], tmpdir) self.assertTrue(os.path.isabs(path)) @@ -336,7 +336,7 @@ class TestCcppInitMultiInstance(unittest.TestCase): def setUp(self): hd = _load_full_host_dict() suite_resolution = _resolve() - lines = _generate_static_api('test_host', ['test_simple'], [suite_resolution], hd) + lines = _generate_host_cap('test_host', ['test_simple'], [suite_resolution], hd) self.text = '\n'.join(lines) def test_init_signature_has_instance_pair(self): @@ -382,7 +382,7 @@ def setUp(self): hd = {k: v for k, v in _load_full_host_dict().items() if k not in ('number_of_instances', 'instance_number')} suite_resolution = _resolve() - lines = _generate_static_api('test_host', ['test_simple'], [suite_resolution], hd) + lines = _generate_host_cap('test_host', ['test_simple'], [suite_resolution], hd) self.text = '\n'.join(lines) def test_init_signature_no_instance_args(self): @@ -1249,7 +1249,7 @@ def test_module_imports_error_unit_unconditionally(self): # emits a gated ``if (trace) write(error_unit, *) ...`` line. # Stub-on and stub-off both include the same USE. for stub in (True, False): - text = '\n'.join(_generate_static_api( + text = '\n'.join(_generate_host_cap( 'test_host', ['test_simple'], [self.suite_resolution], self.hd, no_host_introspection=stub, @@ -1263,7 +1263,7 @@ def test_module_imports_error_unit_unconditionally(self): def test_public_declarations_unchanged_when_stubbed(self): # All five introspection routines remain public — callers must # still link against them. - text = '\n'.join(_generate_static_api( + text = '\n'.join(_generate_host_cap( 'test_host', ['test_simple'], [self.suite_resolution], self.hd, no_host_introspection=True, @@ -1283,19 +1283,19 @@ def test_line_count_drops_dramatically(self): """The motivating case: 33k+ lines → ~800. We don't have 80 suites in unit-test fixtures, but even with one suite the stubbed module must be strictly shorter than the full one.""" - full = _generate_static_api('test_host', ['test_simple'], [self.suite_resolution], + full = _generate_host_cap('test_host', ['test_simple'], [self.suite_resolution], self.hd, no_host_introspection=False) - stub = _generate_static_api('test_host', ['test_simple'], [self.suite_resolution], + stub = _generate_host_cap('test_host', ['test_simple'], [self.suite_resolution], self.hd, no_host_introspection=True) self.assertLess(len(stub), len(full), 'stubbed module should be shorter than the full one ' '(full={}, stub={})'.format(len(full), len(stub))) - def test_write_static_api_passes_flag_through(self): - """``write_static_api(no_host_introspection=True, ...)`` must + def test_write_host_cap_passes_flag_through(self): + """``write_host_cap(no_host_introspection=True, ...)`` must produce a file containing stub bodies, not full ones.""" with tempfile.TemporaryDirectory() as tmpdir: - out_path = write_static_api( + out_path = write_host_cap( 'test_host', ['test_simple'], [self.suite_resolution], tmpdir, self.hd, no_host_introspection=True, @@ -1321,7 +1321,7 @@ def setUp(self): self.suite_resolution = _resolve() def test_module_gate_default_off(self): - text = '\n'.join(_generate_static_api( + text = '\n'.join(_generate_host_cap( 'test_host', ['test_simple'], [self.suite_resolution], self.hd, )) @@ -1329,7 +1329,7 @@ def test_module_gate_default_off(self): self.assertNotIn('logical, parameter :: trace = .true.', text) def test_module_gate_default_on(self): - text = '\n'.join(_generate_static_api( + text = '\n'.join(_generate_host_cap( 'test_host', ['test_simple'], [self.suite_resolution], self.hd, trace=True, @@ -1338,7 +1338,7 @@ def test_module_gate_default_on(self): self.assertNotIn('logical, parameter :: trace = .false.', text) def test_trace_block_present_in_physics_phases(self): - text = '\n'.join(_generate_static_api( + text = '\n'.join(_generate_host_cap( 'test_host', ['test_simple'], [self.suite_resolution], self.hd, )) @@ -1352,7 +1352,7 @@ def test_trace_block_present_in_physics_phases(self): ) def test_trace_block_present_in_lifecycle_routines(self): - text = '\n'.join(_generate_static_api( + text = '\n'.join(_generate_host_cap( 'test_host', ['test_simple'], [self.suite_resolution], self.hd, )) @@ -1362,9 +1362,9 @@ def test_trace_block_present_in_lifecycle_routines(self): msg='trace string missing for {}'.format(sub), ) - def test_write_static_api_threads_trace_flag(self): + def test_write_host_cap_threads_trace_flag(self): with tempfile.TemporaryDirectory() as tmpdir: - out_path = write_static_api( + out_path = write_host_cap( 'test_host', ['test_simple'], [self.suite_resolution], tmpdir, self.hd, trace=True, @@ -1375,7 +1375,7 @@ def test_write_static_api_threads_trace_flag(self): def load_tests(loader, tests, ignore): - import generator.static_api as sa + import generator.host_cap as sa tests.addTests(doctest.DocTestSuite(sa)) return tests diff --git a/unit-tests/test_integration.py b/unit-tests/test_integration.py index 60ca4acf..ea3624a8 100644 --- a/unit-tests/test_integration.py +++ b/unit-tests/test_integration.py @@ -65,7 +65,7 @@ def tearDown(self): def _path(self, name): return os.path.join(self._tmpdir, name) - def test_static_api_exists(self): + def test_host_cap_exists(self): self.assertTrue(os.path.isfile(self._path('test_host_ccpp_cap.F90'))) def test_suite_cap_exists(self): @@ -376,7 +376,7 @@ def test_no_framework_dependencies_in_utilities(self): # Test: static API content # --------------------------------------------------------------------------- -class TestStaticApiContent(unittest.TestCase): +class TestHostCapContent(unittest.TestCase): def setUp(self): self._tmpdir = tempfile.mkdtemp() @@ -521,7 +521,7 @@ def test_suite_file_in_capgen_files(self): self.assertIn('ccpp_test_simple_cap.F90', names) self.assertIn('ccpp_test_simple_physics_cap.F90', names) - def test_static_api_in_host_files(self): + def test_host_cap_in_host_files(self): host_files = self._root.find('capgen_files').find('host_files') self.assertIsNotNone(host_files) names = [os.path.basename(f.text) for f in host_files.findall('file')] @@ -658,7 +658,7 @@ def test_both_suite_caps_exist(self): os.path.isfile(os.path.join(self._tmpdir, 'ccpp_test_subcycle_cap.F90')) ) - def test_static_api_dispatches_both(self): + def test_host_cap_dispatches_both(self): with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: text = fh.read() self.assertIn("case('test_simple')", text) @@ -805,7 +805,7 @@ def tearDown(self): import shutil shutil.rmtree(self._tmpdir) - def test_static_api_ccpp_init_omits_inst_num(self): + def test_host_cap_ccpp_init_omits_inst_num(self): with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: text = fh.read() # ``suite_name_var`` is the local name declared by @@ -817,7 +817,7 @@ def test_static_api_ccpp_init_omits_inst_num(self): # And NOT the multi-instance shape. self.assertNotIn('inst_num', text) - def test_static_api_ccpp_register_omits_inst_num(self): + def test_host_cap_ccpp_register_omits_inst_num(self): with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: text = fh.read() self.assertIn( @@ -825,7 +825,7 @@ def test_static_api_ccpp_register_omits_inst_num(self): text, ) - def test_static_api_ccpp_final_omits_inst_num(self): + def test_host_cap_ccpp_final_omits_inst_num(self): with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: text = fh.read() self.assertIn( @@ -2108,7 +2108,7 @@ def test_suite_cap_passes_inst_num_to_group_run(self): self.assertIn('inst_num', physics_run) self.assertIn('call diag_group_run', physics_run) - def test_static_api_physics_run_has_inst_num(self): + def test_host_cap_physics_run_has_inst_num(self): """Static API ccpp_physics_run must include inst_num when groups need it.""" with open(os.path.join(self._tmpdir, 'test_host_ccpp_cap.F90')) as fh: text = fh.read() diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 26baaac7..6ca8d3a8 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -3261,7 +3261,7 @@ def test_index_symbol_in_extras(self): def test_constituent_args_excluded_from_introspection(self): # source != 'host' — constituent args do not appear in suite # input/output lists (validated indirectly via _collect_host_io - # in static_api tests; here we just confirm the source). + # in host_cap tests; here we just confirm the source). for arg in self.run_args.values(): if arg.scheme_local_name in ('cldliq', 'tend_cldliq'): self.assertNotEqual(arg.source, 'host') @@ -3296,7 +3296,7 @@ def test_no_number_of_ccpp_constituents_in_extras(self): class TestUsedConstDimStdNames(unittest.TestCase): """``ResolvedArg.used_const_dim_std_names`` carries framework- constituent dim refs (notably ``number_of_ccpp_constituents``) so - the introspection routines in :mod:`generator.static_api` can list + the introspection routines in :mod:`generator.host_cap` can list them as inputs without polluting the host-side :attr:`used_dim_std_names` channel.""" From 2653ab80a0d189070bf7fc090f5f15596c113ee2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 May 2026 20:17:59 -0600 Subject: [PATCH 36/74] Catch conditionally-allocated variables when not allocated but scheme requires them --- capgen-ng/generator/group_cap.py | 70 +++++++++++++++++++- capgen-ng/generator/suite_resolver.py | 32 +++------- doc/migration.md | 51 ++++++++++----- unit-tests/test_suite_resolver.py | 92 ++++++++++++++++++++++----- 4 files changed, 188 insertions(+), 57 deletions(-) diff --git a/capgen-ng/generator/group_cap.py b/capgen-ng/generator/group_cap.py index 9c92c578..c458d90a 100644 --- a/capgen-ng/generator/group_cap.py +++ b/capgen-ng/generator/group_cap.py @@ -532,6 +532,46 @@ def _transform_comment(arg: ResolvedArg, reverse: bool = False) -> str: return '! ' + '; '.join(bits) +def _active_required_guard_lines( + arg: ResolvedArg, + scheme_name: str, + phase: str, + errflg_local: Optional[str], + errmsg_local: Optional[str], + indent: str, +) -> List[str]: + """Runtime guard for a non-optional scheme arg whose host declares ``active``. + + The host says the variable is only valid when ``active`` is true. + The scheme demands the variable unconditionally. We emit a runtime + check at the call site so a violation surfaces as a clean errflg/errmsg + rather than as a silent read of unallocated/stale memory. + + Emitted before any transform pre-emission and before the call itself. + If the host did not declare both ``ccpp_error_code`` and + ``ccpp_error_message`` (extremely unusual), the guard is omitted — + there is no way to report the violation. + """ + if not arg.active_local or arg.is_optional: + return [] + if not errflg_local or not errmsg_local: + return [] + msg = ( + "scheme '{scheme}' phase '{phase}' requires variable " + "'{std}' but host active condition ({active}) is false".format( + scheme=scheme_name, phase=phase, + std=arg.standard_name, active=arg.active, + ) + ) + return [ + '{}if (.not. ({})) then'.format(indent, arg.active_local), + "{} {} = \"{}\"".format(indent, errmsg_local, msg), + '{} {} = 1'.format(indent, errflg_local), + '{} return'.format(indent), + '{}end if'.format(indent), + ] + + def _pre_call_lines(arg: ResolvedArg) -> List[str]: """Generate pre-call Fortran lines for one argument.""" lines = [] @@ -645,6 +685,9 @@ def _loop_counter_name(depth: int) -> str: def _emit_phase_items( items, indent: str, lines: List[str], depth: int, + phase: str = '', + errflg_local: Optional[str] = None, + errmsg_local: Optional[str] = None, ) -> None: """Recursively emit Fortran for a list of :data:`PhaseItem` objects. @@ -655,7 +698,9 @@ def _emit_phase_items( """ for item in items: if isinstance(item, ResolvedCall): - _emit_one_call(item, indent, lines) + _emit_one_call( + item, indent, lines, phase, errflg_local, errmsg_local, + ) elif isinstance(item, ResolvedSubcycle): counter = _loop_counter_name(depth) lines.append( @@ -663,6 +708,9 @@ def _emit_phase_items( ) _emit_phase_items( item.calls, indent + _INDENT, lines, depth=depth + 1, + phase=phase, + errflg_local=errflg_local, + errmsg_local=errmsg_local, ) lines.append('{}end do'.format(indent)) lines.append('') @@ -672,8 +720,21 @@ def _emit_one_call( resolved_call: ResolvedCall, indent: str, lines: List[str], + phase: str = '', + errflg_local: Optional[str] = None, + errmsg_local: Optional[str] = None, ) -> None: """Append Fortran lines for a single scheme call (with transforms + errcheck).""" + # Pre-call: runtime guard for any non-optional arg whose host declares + # ``active = (...)``. Emitted before transforms so an inactive-but-required + # var bails out with a clear error rather than reading host memory through + # the transform pipeline. + for arg in resolved_call.args: + lines.extend(_active_required_guard_lines( + arg, resolved_call.scheme_name, resolved_call.phase, + errflg_local, errmsg_local, indent, + )) + # Pre-call transformations. for arg in resolved_call.args: lines.extend(_pre_call_lines(arg)) @@ -917,7 +978,12 @@ def _generate_phase_subroutine( ) # ---- scheme calls --------------------------------------------------- - _emit_phase_items(phase_items, call_indent, lines, depth=1) + _emit_phase_items( + phase_items, call_indent, lines, depth=1, + phase=phase, + errflg_local=errflg_local, + errmsg_local=errmsg_local, + ) # ---- post-call state transitions ------------------------------------ if phase == 'init': diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 33ede96e..99fece69 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -1347,31 +1347,15 @@ def _resolve_one_arg( host_entry: Optional[HostVarEntry] = host_dict.get(std_name) # active is a host-model-only attribute; read it from the host entry only. + # When the scheme arg is optional, the group cap emits the + # pointer-association pattern (transform_case 2 / 4) so the scheme + # sees PRESENT()=.false. when the active condition is false. When + # the scheme arg is non-optional, the group cap emits a runtime + # guard before the call: if (.not. (active)) raise errflg and return. + # The suite designer is responsible for ensuring the active condition + # holds at call time; the guard converts a silent invalid-memory read + # into a clear runtime error. active = host_entry.active if host_entry is not None else '' - # Coherence check: a host-declared active expression means the host's - # variable is only valid when the condition holds. The generator's - # pointer-association pattern (transform_case 2 / 4) handles that — - # but only fires when the scheme arg is itself ``optional``. A - # non-optional scheme arg would be passed unconditionally, reading - # host memory regardless of the active condition. Reject the - # incoherent combination at resolution time. - if active and not optional: - raise CCPPError( - "Scheme '{scheme}', phase '{phase}', arg '{arg}' " - "(standard_name '{std}'): host metadata declares " - "active = ({active}) on the matching variable, but the " - "scheme metadata does not declare this argument as " - "optional. An ``active`` condition means the host's " - "variable is only valid when the condition holds; the " - "generated cap can only honor that contract via the " - "optional/pointer-association pattern. Add " - "``optional = True`` to the scheme metadata entry (and " - "``optional`` to the matching Fortran dummy declaration), " - "or remove the ``active`` attribute from the host entry.".format( - scheme=scheme_name, phase=phase, arg=local, - std=std_name, active=active, - ) - ) suite_var: Optional[SuiteVar] = suite_vars.get(std_name) if host_entry is not None and suite_var is None: diff --git a/doc/migration.md b/doc/migration.md index bdd68f4d..5dfea5eb 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -82,13 +82,15 @@ Inside a `[ var_name ]` section. All optional. | `molar_mass` | float | `0.0` | Scheme metadata only. | | `diagnostic_name` | str | (defaults to `local_name`) | Host-tooling hint; mutually exclusive with `diagnostic_name_fixed`. | -#### 1.3.1 `active` requires the scheme arg to be `optional` +#### 1.3.1 Host `active` + scheme arg shape When a host variable carries `active = ()`, the host's contract with the cap is "this variable's storage is only valid when -the condition holds". capgen-ng honors that contract via the -pointer-association pattern: at every call site that consumes the var, -the cap emits +the condition holds". capgen-ng honors that contract differently +depending on the matching scheme arg's optionality: + +**Scheme arg is `optional = True`** — the cap uses pointer association +so the scheme observes `PRESENT()` according to the active condition: ```fortran if () then @@ -99,20 +101,37 @@ end if call scheme(..., my_arg=ptr%ptr, ...) ``` -The pointer-association path is only safe when the scheme's Fortran -dummy declaration is itself `optional`. Therefore: **every scheme arg -whose host counterpart carries `active = (...)` MUST declare -`optional = True` in its scheme metadata, and the matching Fortran -dummy MUST carry the `optional` attribute.** +**Scheme arg is non-optional** — the scheme is asserting the variable +is mandatory. The cap emits a runtime guard before the call so an +inactive-but-required variable surfaces as a clean error rather than a +silent read of unallocated memory: -The resolver enforces this at code-generation time with a clear error -naming the scheme, the argument, and the host's `active` expression. -If you hit it, two valid fixes: +```fortran +if (.not. ()) then + errmsg = "scheme 'X' phase 'Y' requires variable '' but " & + // "host active condition () is false" + errflg = 1 + return +end if +call scheme(..., my_arg=(), ...) +``` -- Add `optional = True` to the scheme metadata entry and `optional` - to the Fortran dummy declaration; or -- Remove the `active` attribute from the host metadata entry (only if - the host's variable really is always valid). +The guard runs before any unit/kind/vertical-flip transform pre-call +code, so transforms never see invalid host memory. Multiple required +arguments with `active` conditions each get their own guard block — one +per arg keeps the error messages targeted. + +It is the suite designer's responsibility to schedule the call so the +host's active condition holds when a required-arg scheme runs. The +guard converts violations from latent runtime bugs into immediate +errflg/errmsg returns. + +> **Earlier (relaxed 2026-05-20)**: a previous iteration of this rule +> rejected `active` + non-optional pairings at resolution time and +> required scheme metadata to declare `optional = True`. That forced +> scheme metadata to misrepresent the Fortran for schemes that +> legitimately require a host-conditional variable. The current rule +> defers the check to runtime and leaves the metadata honest. ### 1.4 Sliced local names with long subscript indices diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 6ca8d3a8..3bdad8a2 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -41,6 +41,7 @@ SuiteResolution, ) from generator.group_cap import ( + _active_required_guard_lines, _fortran_type_str, _dim_decl, _dim_decl_local, @@ -1236,14 +1237,21 @@ def test_optional_sets_ptr_name(self): ######################################################################## -# Tests: active + optional coherence +# Tests: host active + scheme arg coherence ######################################################################## -class TestActiveRequiresOptional(unittest.TestCase): - """When the host declares ``active = ()`` on a variable, any - scheme arg matching that variable must declare ``optional = True``. - The cap honors the active condition via pointer-association, which - is only valid when the scheme's Fortran dummy is optional.""" +class TestActiveHostHandling(unittest.TestCase): + """When the host declares ``active = ()`` on a variable: + + * Scheme arg ``optional = True`` -> the cap emits the pointer- + association pattern (transform_case 2 or 4); the scheme observes + PRESENT()=.false. when the condition is false. + * Scheme arg non-optional -> the resolver still succeeds (the + scheme is asserting the variable is mandatory); the group cap + emits a runtime guard that raises errflg/errmsg if the condition + is false at call time. The asymmetric optional rule in the + validator covers the metadata/Fortran consistency check; this + class only exercises resolver-level behaviour.""" _HOST_SRC = ( '[ccpp-table-properties]\n' @@ -1310,16 +1318,70 @@ def test_optional_scheme_arg_passes(self): # transform). Both are pointer-pattern paths. self.assertIn(arg.transform_case, (2, 4)) - def test_required_scheme_arg_raises(self): + def test_required_scheme_arg_resolves_with_active(self): + """A non-optional scheme arg paired with a host ``active = (...)`` + variable resolves cleanly: the resolver passes the active + condition through (so the group cap can emit a runtime guard) + and selects a non-pointer transform_case (1 or 3).""" hd = self._host_dict() - with self.assertRaises(CCPPError) as cm: - _resolve_one_arg(self._scheme_var(optional=False), 'run', hd, - {}, 'my_scheme', set()) - msg = str(cm.exception) - self.assertIn("my_scheme", msg) - self.assertIn("air_temperature", msg) - self.assertIn("(flag_for_passive_check)", msg) - self.assertIn("optional", msg.lower()) + arg = _resolve_one_arg(self._scheme_var(optional=False), 'run', hd, + {}, 'my_scheme', set()) + self.assertEqual(arg.active, '(flag_for_passive_check)') + # active_local is the same string here since flag_for_passive_check + # is referenced via its standard name with no rename. + self.assertIn('flag_passive', arg.active_local) + self.assertFalse(arg.is_optional) + # No pointer wrapper for a required arg; case 1 (direct) or 3 (transform). + self.assertIn(arg.transform_case, (1, 3)) + self.assertEqual(arg.ptr_name, '') + + def test_required_scheme_arg_emits_runtime_guard(self): + """Group-cap emitter renders the guard block for a non-optional + arg whose host declares ``active = (...)`` — guard is emitted at + the call indent, raises errflg/errmsg with a clear message, and + does *not* wrap the arg in a pointer.""" + hd = self._host_dict() + arg = _resolve_one_arg(self._scheme_var(optional=False), 'run', hd, + {}, 'my_scheme', set()) + guard = _active_required_guard_lines( + arg, scheme_name='my_scheme', phase='run', + errflg_local='errflg', errmsg_local='errmsg', + indent=' ', + ) + self.assertTrue(guard, "expected a non-empty guard block") + body = '\n'.join(guard) + self.assertIn('if (.not. (', body) + self.assertIn('flag_passive', body) + self.assertIn("my_scheme", body) + self.assertIn("air_temperature", body) + self.assertIn('errflg = 1', body) + self.assertIn('return', body) + + def test_optional_scheme_arg_skips_runtime_guard(self): + """The runtime guard is only for non-optional args — optional + args use the pointer-association pattern and need no guard.""" + hd = self._host_dict() + arg = _resolve_one_arg(self._scheme_var(optional=True), 'run', hd, + {}, 'my_scheme', set()) + guard = _active_required_guard_lines( + arg, scheme_name='my_scheme', phase='run', + errflg_local='errflg', errmsg_local='errmsg', + indent=' ', + ) + self.assertEqual(guard, []) + + def test_guard_skipped_when_no_error_locals(self): + """If the host did not declare ccpp_error_code/ccpp_error_message, + the guard is suppressed — there is no way to report the violation.""" + hd = self._host_dict() + arg = _resolve_one_arg(self._scheme_var(optional=False), 'run', hd, + {}, 'my_scheme', set()) + guard = _active_required_guard_lines( + arg, scheme_name='my_scheme', phase='run', + errflg_local=None, errmsg_local=None, + indent=' ', + ) + self.assertEqual(guard, []) ######################################################################## From 37ea00ece3d1cbdbe4cd6617799ac2c5d597b54b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 May 2026 20:45:05 -0600 Subject: [PATCH 37/74] Fix comments in metadata --- capgen-ng/metadata/metadata_table.py | 35 ++++++++++++++++ unit-tests/test_metadata_table.py | 61 ++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+) diff --git a/capgen-ng/metadata/metadata_table.py b/capgen-ng/metadata/metadata_table.py index aa762b1e..51bef9dd 100644 --- a/capgen-ng/metadata/metadata_table.py +++ b/capgen-ng/metadata/metadata_table.py @@ -168,6 +168,37 @@ def _is_blank(line: str) -> bool: return _BLANK_RE.match(line) is not None +def _strip_inline_comment(line: str) -> str: + """Drop a trailing ``# ...`` comment from a metadata line. + + Metadata files use ``#`` (and ``;`` at column 0) as comment markers. + A ``#`` anywhere in a line — not just at column 0 — starts a comment + that runs to the end of the line; the parser discards it before any + further processing. No escape mechanism: ``#`` is not a legitimate + character in any metadata value (units, kinds, identifiers, dim + lists, Fortran conditional expressions). + + Trailing whitespace left behind by the strip is also removed so that + section/variable headers like ``[ name ]`` and key=value lines parse + cleanly with their existing regexes. + + >>> _strip_inline_comment('dimensions = () # (nap_indices)') + 'dimensions = ()' + >>> _strip_inline_comment('[ ap_indices ] # legacy slot') + '[ ap_indices ]' + >>> _strip_inline_comment('# whole-line comment') + '' + >>> _strip_inline_comment('plain line with no comment') + 'plain line with no comment' + >>> _strip_inline_comment('') + '' + """ + idx = line.find('#') + if idx < 0: + return line + return line[:idx].rstrip() + + def _parse_bool(value: str, context: ParseContext) -> bool: """Parse a Fortran/Python boolean string to a Python bool. @@ -1214,6 +1245,10 @@ def flush_table_props() -> None: for lineno, raw_line in enumerate(lines): line = raw_line.rstrip('\n').rstrip('\r') + # Discard any inline ``# ...`` comment so headers, key=value lines, + # and the blank-line check all see the same content the user + # intended as data. + line = _strip_inline_comment(line) # ---- [ccpp-table-properties] ---------------------------------------- if line.strip().lower() == _TABLE_PROPS_HDR: diff --git a/unit-tests/test_metadata_table.py b/unit-tests/test_metadata_table.py index 2cd3aff9..89609ea4 100644 --- a/unit-tests/test_metadata_table.py +++ b/unit-tests/test_metadata_table.py @@ -45,6 +45,7 @@ _parse_dimensions, _check_var_type, _parse_config_line, + _strip_inline_comment, ) from metadata.parse_tools.parse_source import ParseContext @@ -96,6 +97,66 @@ def test_bracket_header(self): self.assertFalse(_is_blank('[ccpp-table-properties]')) +class TestStripInlineComment(unittest.TestCase): + """Tests for :func:`_strip_inline_comment` — trailing ``# ...`` is + a comment that the parser must discard before any other handling. + """ + + def test_plain_line_unchanged(self): + self.assertEqual( + _strip_inline_comment('dimensions = (horizontal_dimension)'), + 'dimensions = (horizontal_dimension)', + ) + + def test_strips_trailing_hash_comment(self): + self.assertEqual( + _strip_inline_comment('dimensions = () # (nap_indices)'), + 'dimensions = ()', + ) + + def test_strips_section_header_comment(self): + self.assertEqual( + _strip_inline_comment('[ ap_indices ] # legacy'), + '[ ap_indices ]', + ) + + def test_full_line_comment_collapses_to_empty(self): + self.assertEqual(_strip_inline_comment('# whole line'), '') + + def test_hash_at_column_zero(self): + self.assertEqual(_strip_inline_comment('#x'), '') + + +class TestInlineCommentInParser(unittest.TestCase): + """End-to-end check that the parser ignores trailing ``#`` comments + on any metadata line — the user-reported bug surfaced on a + ``dimensions =`` attribute value but the fix is universal.""" + + _SRC = ( + '[ccpp-table-properties]\n' + ' name = mod # the host module\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = mod\n' + ' type = host\n' + '[ ap_indices ] # legacy index slot\n' + ' standard_name = ap_indices\n' + ' units = index\n' + ' dimensions = () # (nap_indices)\n' + ' type = integer\n' + ) + + def test_parses_cleanly(self): + tables = _parse_text(self._SRC) + self.assertEqual(len(tables), 1) + sec = tables[0].sections()[0] + var = sec.variables[0] + self.assertEqual(var.local_name, 'ap_indices') + self.assertEqual(var.dimensions, []) + self.assertEqual(var.type, 'integer') + self.assertEqual(tables[0].table_name, 'mod') + + class TestParseBool(unittest.TestCase): """Tests for :func:`_parse_bool`.""" From 6948a9255d0b40d2f3651019beecc331fd15ef0b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 May 2026 21:06:23 -0600 Subject: [PATCH 38/74] True variable compatibility checks --- capgen-ng/generator/suite_resolver.py | 90 +++++++++++ unit-tests/test_suite_resolver.py | 215 +++++++++++++++++++++++++- 2 files changed, 302 insertions(+), 3 deletions(-) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 99fece69..2291a79e 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -626,6 +626,44 @@ def _dim_has_vertical(dim: str) -> bool: return upper in _VDIM_STDS +def _canonical_dim(dim: str) -> str: + """Return a dimension entry in canonical ``lower:upper`` form for + identity comparison. + + Three spellings of the implicit/default lower bound all collapse + to a single representative: + + * bare ``foo`` (no explicit lower bound) + * ``1:foo`` (the integer literal one) + * ``ccpp_constant_one:foo`` (the standard name) + + Any *other* lower bound is distinct: ``2:foo`` is not the same + axis as ``1:foo``, ``bar:foo`` is not the same as ``1:foo``, etc. + Different lower bound describes a different sub-range and must + not compare equal. + + No name aliasing on the upper bound happens here. + ``horizontal_loop_extent`` and ``horizontal_dimension`` are + different names — the :func:`metadata.legacy_compat` shim is the + canonical place that rewrites the legacy name to the new one at + parse time when ``--legacy-mode`` is enabled. Without that shim + the two should never appear on opposite sides of a host/scheme + pairing. + """ + if ':' in dim: + lower, upper = dim.split(':', 1) + else: + lower, upper = _CCPP_CONSTANT_ONE, dim + lower = lower.strip().lower() + upper = upper.strip().lower() + # Collapse the integer literal '1' and the standard name + # 'ccpp_constant_one' to a single representative so all three + # spellings of the default lower bound compare equal. + if lower == '1': + lower = _CCPP_CONSTANT_ONE + return '{}:{}'.format(lower, upper) + + def _substitute_scalar_idx( expr: str, host_dict: Dict[str, HostVarEntry], ) -> str: @@ -1405,14 +1443,66 @@ def _resolve_one_arg( host_dims = host_entry.dimensions host_units = host_entry.units host_kind = host_entry.kind + host_type = host_entry.type host_allocatable = host_entry.allocatable else: base_expr = suite_var.access_path host_dims = suite_var.dimensions host_units = suite_var.units host_kind = suite_var.kind + host_type = suite_var.type_ host_allocatable = suite_var.allocatable + # ---- type identity check --------------------------------------------- + # The defining source (host metadata or the first scheme to write a + # suite-owned var) sets the variable's type; every subsequent consumer + # must agree. Numeric/kind coercion happens via the transform pipeline, + # but the *type kind* itself (real vs integer vs logical vs DDT) must + # match identically — there is no transform that crosses those. + if (host_type or '').strip().lower() != (scheme_var.type or '').strip().lower(): + raise CCPPError( + "Variable '{}' (standard_name='{}'): {} declares type='{}' but " + "scheme '{}' declares type='{}'; cross-type assignment is not " + "supported, the scheme metadata must match the defining type".format( + local, std_name, source, host_type, + scheme_name, scheme_var.type, + ) + ) + + # ---- rank check ------------------------------------------------------ + if len(host_dims) != len(scheme_var.dimensions): + raise CCPPError( + "Variable '{}' (standard_name='{}'): {} declares rank {} " + "(dimensions={}) but scheme '{}' declares rank {} " + "(dimensions={}); the scheme metadata's dimension list must " + "match the defining rank".format( + local, std_name, source, len(host_dims), + list(host_dims), scheme_name, len(scheme_var.dimensions), + list(scheme_var.dimensions), + ) + ) + + # ---- per-position dimension identity check --------------------------- + # Each dimension entry is canonicalized to ``lower:upper`` form (bare + # ``X`` -> ``ccpp_constant_one:X``) and compared for strict identity. + # No name aliasing: ``horizontal_loop_extent`` and ``horizontal_dimension`` + # are distinct; the legacy-compat shim is the one place that rewrites + # legacy names at parse time. + for pos, (hdim, sdim) in enumerate(zip(host_dims, scheme_var.dimensions)): + if _canonical_dim(hdim) != _canonical_dim(sdim): + raise CCPPError( + "Variable '{}' (standard_name='{}'): {} declares " + "dimension {} as '{}' but scheme '{}' declares it as " + "'{}'; per-position dimension entries must match " + "(the scheme metadata's dimension list must name the " + "defining axes; bare names are equivalent to " + "ccpp_constant_one:, all other lower bounds are " + "distinct)".format( + local, std_name, source, pos, hdim, + scheme_name, sdim, + ) + ) + # ---- allocatable compatibility check --------------------------------- # An actual argument that is not allocatable cannot be passed to an # allocatable dummy. The reverse direction (allocatable host -> plain diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 3bdad8a2..9d844e53 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -1124,7 +1124,7 @@ def test_case1_2d_array_run(self): """2D array in run phase → subscript applied.""" hd = self._host_dict() suite_var = self._scheme_var('temp', 'air_temperature', 'inout', 'K', - '(horizontal_loop_extent, vertical_layer_dimension)', + '(horizontal_dimension, vertical_layer_dimension)', 'real', 'kind_phys') arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) # access_path = 'gt0', subscript = '(lb:ub, 1:nlev)' @@ -1236,6 +1236,215 @@ def test_optional_sets_ptr_name(self): self.assertEqual(arg.transform_case, 2) +######################################################################## +# Tests: host vs scheme metadata compatibility +######################################################################## + +class TestHostSchemeCompatibility(unittest.TestCase): + """Resolver-level cross-metadata checks: the scheme's metadata must + agree with the defining source (host or suite) on type, rank, and + dimension identity. Units and character kind have their own existing + tests; numeric kind is intentionally lenient (triggers a transform + copy, see [[design_numeric_kind_silent_transform]]). + """ + + _HOST_SRC = ( + '[ccpp-table-properties]\n' + ' name = host_mod\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = host_mod\n' + ' type = host\n' + '[ ncols ]\n' + ' standard_name = horizontal_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ nlev ]\n' + ' standard_name = vertical_layer_dimension\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ nap ]\n' + ' standard_name = nap_indices\n' + ' units = count\n' + ' dimensions = ()\n' + ' type = integer\n' + '[ tair ]\n' + ' standard_name = air_temperature\n' + ' units = K\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real\n' + ' kind = kind_phys\n' + '[ scalar_flag ]\n' + ' standard_name = a_scalar_flag\n' + ' units = 1\n' + ' dimensions = ()\n' + ' type = real\n' + ' kind = kind_phys\n' + '[ ap_arr ]\n' + ' standard_name = ap_indexed_array\n' + ' units = count\n' + ' dimensions = (nap_indices)\n' + ' type = integer\n' + ) + + def _host_dict(self): + from metadata.metadata_table import _parse_lines + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + host_tbls = _parse_lines( + self._HOST_SRC.splitlines(keepends=True), 'h.meta', + ) + ctrl_only = [t for t in ctrl_tbls if t.table_type == 'control'] + return build_flat_host_dict(host_tbls, ctrl_only, []) + + def _scheme_var(self, std_name, type_, dims, units='K', kind='kind_phys', + intent='inout'): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar('x', ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', units, ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', type_, ctx) + if kind: + v.set_attr('kind', kind, ctx) + v.set_attr('intent', intent, ctx) + return v + + def test_host_scalar_scheme_rank1_raises(self): + """Host scalar `()` with scheme `(horizontal_dimension)` is a rank + mismatch — user-reported gap that the resolver now catches.""" + hd = self._host_dict() + sv = self._scheme_var( + 'a_scalar_flag', 'real', + '(horizontal_dimension)', units='1', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('rank', msg) + self.assertIn('a_scalar_flag', msg) + + def test_host_dim_mismatch_raises(self): + """Host `(nap_indices)` with scheme `(horizontal_dimension)` — + same rank, different axis — is a metadata error.""" + hd = self._host_dict() + sv = self._scheme_var( + 'ap_indexed_array', 'integer', + '(horizontal_dimension)', units='count', kind='', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('dimension', msg.lower()) + self.assertIn('nap_indices', msg) + self.assertIn('horizontal_dimension', msg) + + def test_type_mismatch_raises(self): + """Host `integer` with scheme `real` is a type-identity error + even when units and rank align.""" + hd = self._host_dict() + sv = self._scheme_var( + 'horizontal_dimension', 'real', '()', units='count', kind='', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('type', msg.lower()) + + def test_default_lower_bound_spellings_equivalent(self): + """Three spellings of the default lower bound are equivalent: + bare ``X``, ``1:X``, and ``ccpp_constant_one:X``. The host's + bare ``vertical_layer_dimension`` matches any of these forms + on the scheme side.""" + hd = self._host_dict() + for sdim in ( + 'vertical_layer_dimension', + '1:vertical_layer_dimension', + 'ccpp_constant_one:vertical_layer_dimension', + ): + with self.subTest(scheme_dim=sdim): + sv = self._scheme_var( + 'air_temperature', 'real', + '(horizontal_dimension, {})'.format(sdim), + ) + arg = _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + self.assertEqual(arg.standard_name, 'air_temperature') + + def test_nondefault_integer_lower_bound_mismatch_raises(self): + """``2:nlev`` and ``1:nlev`` are NOT the same axis — different + lower bound means different sub-range, even when both are + integer literals.""" + hd = self._host_dict() + sv = self._scheme_var( + 'air_temperature', 'real', + '(horizontal_dimension, 2:vertical_layer_dimension)', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('dimension', msg.lower()) + self.assertIn('vertical_layer_dimension', msg) + self.assertIn('2:', msg) + + def test_nondefault_named_lower_bound_mismatch_raises(self): + """A non-default standard-name lower bound (``foo:nlev``) is + distinct from the default ``ccpp_constant_one:nlev``.""" + hd = self._host_dict() + sv = self._scheme_var( + 'air_temperature', 'real', + '(horizontal_dimension, ' + 'some_made_up_lower_bound:vertical_layer_dimension)', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('dimension', msg.lower()) + self.assertIn('vertical_layer_dimension', msg) + self.assertIn('some_made_up_lower_bound', msg) + + def test_horizontal_loop_extent_in_scheme_dims_raises(self): + """No name aliasing in the compat check: a scheme that uses the + legacy ``horizontal_loop_extent`` in a dimension list while the + host declares ``horizontal_dimension`` is a mismatch. The + legacy-compat shim is the only place such rewriting belongs.""" + hd = self._host_dict() + sv = self._scheme_var( + 'air_temperature', 'real', + '(horizontal_loop_extent, vertical_layer_dimension)', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(sv, 'run', hd, {}, 'sch', set()) + msg = str(cm.exception) + self.assertIn('horizontal_dimension', msg) + self.assertIn('horizontal_loop_extent', msg) + + def test_suite_var_second_reader_mismatch_raises(self): + """First scheme with intent=out fixes the SuiteVar's type/rank; + a later scheme that reads it with mismatched dims is rejected.""" + hd = self._host_dict() + # First scheme creates the suite var. + writer = self._scheme_var( + 'an_arbitrary_suite_quantity', 'real', + '(horizontal_dimension, vertical_layer_dimension)', + units='K', intent='out', + ) + suite_vars = {} + _resolve_one_arg(writer, 'run', hd, suite_vars, 'sch_a', set()) + self.assertIn('an_arbitrary_suite_quantity', suite_vars) + # Second scheme reads it — but with a wrong rank. + reader = self._scheme_var( + 'an_arbitrary_suite_quantity', 'real', + '(horizontal_dimension)', units='K', intent='in', + ) + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(reader, 'run', hd, suite_vars, 'sch_b', set()) + msg = str(cm.exception) + self.assertIn('rank', msg) + self.assertIn('suite', msg) + + ######################################################################## # Tests: host active + scheme arg coherence ######################################################################## @@ -1460,7 +1669,7 @@ def _build_host_and_scheme( suite_var.set_attr('standard_name', 'air_temperature', ctx) suite_var.set_attr('units', scheme_units, ctx) suite_var.set_attr('dimensions', - '(horizontal_loop_extent, vertical_layer_dimension)', ctx) + '(horizontal_dimension, vertical_layer_dimension)', ctx) suite_var.set_attr('type', 'real', ctx) suite_var.set_attr('kind', 'kind_phys', ctx) suite_var.set_attr('intent', intent, ctx) @@ -3489,7 +3698,7 @@ class TestConstituentResolverErrors(unittest.TestCase): '[ x ]\n' ' standard_name = {std}\n' ' units = kg kg-1\n' - ' dimensions = (horizontal_loop_extent, vertical_layer_dimension)\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' ' type = real | kind = kind_phys\n' ' intent = {intent}\n' ' {flag} = .true.\n' From 9baf74a21e178c04cb0bbd33d7ef0071b9cbd360 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 May 2026 21:19:33 -0600 Subject: [PATCH 39/74] Update docs --- doc/briefing.md | 20 ++++++++--- doc/briefing_pm.md | 4 +-- doc/migration.md | 82 +++++++++++++++++++++++++++++++++++++++++- doc/redesign_prompt.md | 2 +- 4 files changed, 99 insertions(+), 9 deletions(-) diff --git a/doc/briefing.md b/doc/briefing.md index 9774026f..39f9984c 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -339,7 +339,8 @@ don't rebuild downstream objects unless something actually moved. ## 10. Where things stand right now -- **Unit tests**: 1335 passing on `feature/capgen-ng` (as of 2026-05-20). +- **Unit tests**: 1353 passing on `feature/capgen-ng` (1365 with + doctests; as of 2026-05-20). - **End-to-end tests passing**: `advection`, `unit_conv`, `nested_suite`, `variable_transform`, `instances`, `instances_advection`, `ddt`. @@ -354,10 +355,19 @@ don't rebuild downstream objects unless something actually moved. - **Validator** now checks per-argument `intent`, `type`, `kind`, and dimension rank in addition to the original name/count check. Asymmetric `optional` rule, DDT + `external::` - type normalisation, character `len=*` wildcard. Resolver also - enforces an `active`-vs-`optional` coherence rule: a host variable - with `active = (...)` paired with a non-optional scheme arg is now - a parse-time error. See `doc/migration.md` §7 + §1.3.1. + type normalisation, character `len=*` wildcard. +- **Resolver cross-metadata checks** (late 2026-05-20): host/scheme + (and suite-owned-var first-writer/follow-on) consistency on type, + rank, and per-position dimension entries. Default lower bound has + three equivalent spellings (bare `X`, `1:X`, `ccpp_constant_one:X`); + other lower bounds stay distinct. Numeric kind remains lenient + (transform path). See `doc/migration.md` §1.3.2. +- **Host `active` + scheme arg shape**: when the scheme arg is + optional, the cap uses pointer association (PRESENT()-aware); when + the scheme arg is non-optional, the cap emits a runtime guard + (`if (.not. (active)) errflg = 1; return`) before the call. + Replaces an earlier static rule that forced scheme metadata to lie + about optionality. See `doc/migration.md` §1.3.1. - **`--no-host-introspection`** (new, 2026-05-14): stubs the bodies of the five suite-introspection routines in `_ccpp_cap.F90`, shrinking the file from ~33k lines to ~800 for the 10-suite SCM diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md index 4e4ba7e6..bd59f75b 100644 --- a/doc/briefing_pm.md +++ b/doc/briefing_pm.md @@ -249,9 +249,9 @@ Features that exist only in capgen-ng (some exist in prebuild): --- -## 6. Where things stand right now (2026-05-18) +## 6. Where things stand right now (2026-05-20) -- **Unit tests**: 1335 passing. No known failures. +- **Unit tests**: 1353 passing (1365 with doctests). No known failures. - **End-to-end tests**: 10 passing — `chunked_data`, `opt_arg`, `nested_suite`, `ddthost`, `instances`, `capgen_ng`, `var_compat`, `advection`, and the new `instances_advection` diff --git a/doc/migration.md b/doc/migration.md index 5dfea5eb..6aa35548 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -6,7 +6,7 @@ Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to **capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and `doc/redesign_analysis.md` (analysis of the old systems). -*Last revised: 2026-05-20 (end-of-day).* Current unit-test suite: 1335 passing. +*Last revised: 2026-05-20 (end-of-day).* Current unit-test suite: 1353 passing (1365 with doctests). **Repository layout** (post-2026-05-13 cleanup): tooling lives under `capgen-ng/` (top-level of this repo). Unit tests live at the top @@ -133,6 +133,54 @@ errflg/errmsg returns. > legitimately require a host-conditional variable. The current rule > defers the check to runtime and leaves the metadata honest. +#### 1.3.2 Host/scheme metadata cross-checks + +The resolver enforces three cross-metadata checks per scheme arg +against its defining source (host metadata or, for suite-owned +variables, the first scheme to write the var with `intent=out`): + +| Aspect | Rule | Notes | +|---|---|---| +| **Type identity** | Strict string match after `strip().lower()`. | No coercion across `real` / `integer` / `logical` / DDT. DDT names match identically; `external:m:t` matches `external:m:t`. | +| **Rank** | `len(host.dimensions) == len(scheme.dimensions)`. | A scheme that asks for `(horizontal_dimension)` while the host declares `()` is rejected. | +| **Per-position dimension identity** | Each entry is canonicalized to `lower:upper`; strict match per position. | See "default lower bound" below. | + +**Default lower bound — three equivalent spellings:** + +- bare `foo` (no explicit lower bound) +- `1:foo` (integer literal one) +- `ccpp_constant_one:foo` (the standard name) + +All three collapse to a single canonical representative, so the host +declaring `(vertical_layer_dimension)` matches the scheme declaring +`(1:vertical_layer_dimension)` and vice versa. + +**Every other lower bound is distinct.** `2:nlev` is not the same +axis as `1:nlev`; `start_idx:nlev` is not the same as +`ccpp_constant_one:nlev`. Different lower bound describes a +different sub-range and must be spelled identically on both sides. + +**No upper-bound name aliasing.** `horizontal_dimension` and +`horizontal_loop_extent` are different names at the resolver layer. +The `--legacy-mode` shim (see §3) rewrites legacy names at parse +time when enabled; without that shim the legacy spellings should not +appear in metadata at all. + +**Numeric kind is *not* checked here.** Host `kind_phys` vs scheme +`real32` silently triggers the transform-copy pipeline (§5.3). This +is deliberate: real CCPP-physics schemes legitimately mix precisions +and rely on the cap to handle the copy. Watch for unintended +narrowing — there is no static guard. **Character `len=`** has its +own block: matching `len=N` values pass, mismatched specific lengths +are an error unless the scheme uses `len=*` (wildcard). + +**Suite-owned variables.** The first scheme to write a standard +name with `intent=out` freezes the var's type/kind/dimensions/units +on the SuiteVar; every later scheme that consumes it goes through +the same checks against the frozen fields. Error messages name the +source as `host`, `control`, or `suite` so you know whose contract +you're violating. + ### 1.4 Sliced local names with long subscript indices Local names with array slices may carry CCPP standard names as subscript @@ -222,6 +270,30 @@ Migration paths: invisible. This shim *will be removed*; treat it as a runway, not a destination. +### 1.9 Inline comments + +`#` starts a comment **anywhere on a line**, not just at column 0. +Everything from the `#` to end-of-line is discarded before the +parser sees the rest of the line. Trailing whitespace left behind +by the strip is also removed, so section headers and key=value +lines parse cleanly. + +``` +[ ap_indices ] # legacy index slot + standard_name = ap_indices + units = count + dimensions = () # was (nap_indices) before the refactor + type = integer +``` + +No escape mechanism is provided — `#` is not a legitimate character +in any metadata value (units, kinds, identifiers, dim lists, +Fortran conditional expressions). `;` is still accepted as a +full-line comment marker (at column 0 after whitespace), matching +the historic blank-line convention, but is *not* treated as an +inline comment marker (`;` can plausibly appear inside a +`long_name`). + --- ## 2. Suite definition file (SDF) changes @@ -612,6 +684,14 @@ The generator emits three kinds of transform on a per-arg basis: | Kind conversion | `host.kind != scheme.kind` (different strings). | | Vertical flip | `host.top_at_one != scheme.top_at_one` on a var with a vertical dim. | +Transforms only smooth over *representation* differences. Anything +the cap cannot bridge with a per-call copy — type identity, rank, +or per-position dimension identity — is rejected by the resolver as +a hard error (see §1.3.2). In particular, the kind-conversion entry +above is *not* gated on convertibility: any kind-string difference +triggers an implicit conversion copy. Watch for unintended +narrowing. + These compose. A scheme arg that needs unit + flip emits a single combined assignment through a transformation temp: diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index 316addf4..529168e7 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -1285,7 +1285,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` ### Test status -- **Unit tests**: 1335 passing (`python -m pytest unit-tests/`). +- **Unit tests**: 1353 passing (1365 with doctests). Run via `python unit-tests/run_tests.py [--doctest]`. - **End-to-end tests**: `advection`, `unit_conv`, `nested_suite`, `variable_transform`, `instances`, `ddt` covered. SCM running against ccpp-physics is the active driver right now — most of the From 24f6484ef50872fb0fff9f51277047353d780f17 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 21 May 2026 05:28:28 -0600 Subject: [PATCH 40/74] New legacy switch --gfs-dim-aliases; clean up previous use of horizontal_loop_extent --- capgen-ng/ccpp_capgen_ng.py | 24 ++ capgen-ng/ccpp_validator.py | 24 ++ capgen-ng/generator/group_cap.py | 16 +- capgen-ng/generator/suite_resolver.py | 37 ++- capgen-ng/metadata/dim_aliases.py | 196 +++++++++++++++ capgen-ng/metadata/metadata_table.py | 8 +- capgen-ng/metadata/variable_resolver.py | 2 +- doc/HelloWorld/hello_scheme.meta | 6 +- doc/HelloWorld/temp_adjust.meta | 4 +- doc/redesign_analysis.md | 6 +- .../sample_files/bad_duplicate_stdname.meta | 4 +- unit-tests/sample_files/host_full.meta | 6 - unit-tests/sample_files/host_no_instance.meta | 6 - unit-tests/sample_files/host_unit_conv.meta | 6 - .../sample_files/host_with_dependencies.meta | 6 - .../scheme_module_name_override.meta | 2 +- unit-tests/test_dim_aliases.py | 223 ++++++++++++++++++ unit-tests/test_host_cap.py | 5 - unit-tests/test_metadata_table.py | 20 +- unit-tests/test_suite_data.py | 4 +- unit-tests/test_suite_resolver.py | 27 +-- unit-tests/test_variable_resolver.py | 6 +- 22 files changed, 545 insertions(+), 93 deletions(-) create mode 100644 capgen-ng/metadata/dim_aliases.py create mode 100644 unit-tests/test_dim_aliases.py diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index cf1cad01..8d84bea8 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -255,6 +255,24 @@ def _build_arg_parser() -> argparse.ArgumentParser: "loud warning at startup. Will be removed." ), ) + # dim-aliases: transient GFS-physics shim (delete the argument, + # the enable() call below, and the rest of the dim_aliases + # touchpoints when the workaround is removed). + parser.add_argument( + '--gfs-dim-aliases', + action='store_true', + help=( + "TRANSIENT GFS-PHYSICS SHIM. Treat a small audited list of " + "physically-equivalent vertical-axis standard names as the " + "same dimension during the host/scheme dim-position " + "identity check only (e.g. " + "'adjusted_vertical_layer_dimension_for_radiation' and " + "'vertical_composition_dimension' both compare equal to " + "'vertical_layer_dimension'). Variables keep their " + "original names everywhere else. Emits a loud warning at " + "startup. Will be removed." + ), + ) parser.add_argument( '--no-host-introspection', action='store_true', @@ -1115,6 +1133,12 @@ def main(argv: Optional[List[str]] = None) -> int: from metadata import legacy_compat legacy_compat.enable(_LOGGER) + # dim-aliases: transient GFS-physics shim. Emit the loud banner + # before any parsing happens so user has fair warning. + if args.gfs_dim_aliases: + from metadata import dim_aliases + dim_aliases.enable(_LOGGER) + # ---- parse kind types -------------------------------------------------- try: kind_types = _parse_kind_types(args.kind_type) diff --git a/capgen-ng/ccpp_validator.py b/capgen-ng/ccpp_validator.py index 88d64d8b..b5cde2e0 100755 --- a/capgen-ng/ccpp_validator.py +++ b/capgen-ng/ccpp_validator.py @@ -1067,6 +1067,24 @@ def _build_parser() -> argparse.ArgumentParser: "loud warning at startup. Will be removed." ), ) + # dim-aliases: transient GFS-physics shim (delete the argument, + # the enable() call below, and the rest of the dim_aliases + # touchpoints when the workaround is removed). + parser.add_argument( + '--gfs-dim-aliases', + action='store_true', + help=( + "TRANSIENT GFS-PHYSICS SHIM. Treat a small audited list of " + "physically-equivalent vertical-axis standard names as the " + "same dimension during the host/scheme dim-position " + "identity check only (e.g. " + "'adjusted_vertical_layer_dimension_for_radiation' and " + "'vertical_composition_dimension' both compare equal to " + "'vertical_layer_dimension'). Variables keep their " + "original names everywhere else. Emits a loud warning at " + "startup. Will be removed." + ), + ) return parser @@ -1087,6 +1105,12 @@ def main(argv: Optional[List[str]] = None) -> int: from metadata import legacy_compat legacy_compat.enable(_LOGGER) + # dim-aliases: transient GFS-physics shim. Emit the loud banner + # before any parsing happens so user has fair warning. + if args.gfs_dim_aliases: + from metadata import dim_aliases + dim_aliases.enable(_LOGGER) + scheme_files = [f.strip() for f in args.scheme_files.split(',') if f.strip()] source_files = [f.strip() for f in args.source_files.split(',') if f.strip()] diff --git a/capgen-ng/generator/group_cap.py b/capgen-ng/generator/group_cap.py index c458d90a..f0a2f3e1 100644 --- a/capgen-ng/generator/group_cap.py +++ b/capgen-ng/generator/group_cap.py @@ -137,10 +137,10 @@ def _dim_decl(dimensions: List[str]) -> str: >>> _dim_decl([]) '' - >>> _dim_decl(['horizontal_loop_extent']) - ', dimension(horizontal_loop_extent)' - >>> _dim_decl(['horizontal_loop_extent', 'vertical_layer_dimension']) - ', dimension(horizontal_loop_extent, vertical_layer_dimension)' + >>> _dim_decl(['horizontal_dimension']) + ', dimension(horizontal_dimension)' + >>> _dim_decl(['horizontal_dimension', 'vertical_layer_dimension']) + ', dimension(horizontal_dimension, vertical_layer_dimension)' """ if not dimensions: return '' @@ -154,9 +154,9 @@ def _dim_decl_local(dimensions: List[str], host_dict) -> str: Fortran variable name. Falls back to the standard name when not found (should not happen with valid metadata). - Special case for ``horizontal_dimension`` / ``horizontal_loop_extent``: - the temp must match the chunk slice the scheme actually receives at - the call site (``host_var(lb:ub, …)`` via :func:`_one_dim_part`). + Special case for ``horizontal_dimension``: the temp must match the + chunk slice the scheme actually receives at the call site + (``host_var(lb:ub, …)`` via :func:`_one_dim_part`). Using the host's local name for ``horizontal_dimension`` (e.g. ``ncols``) would over-size the temp and break the unit-conversion assignment ``ps_l = factor * phys_state%ps(col_start:col_end)`` with a Fortran @@ -171,7 +171,7 @@ def _dim_decl_local(dimensions: List[str], host_dict) -> str: return '' locals_ = [] for std_name in dimensions: - if std_name in ('horizontal_dimension', 'horizontal_loop_extent'): + if std_name == 'horizontal_dimension': lb_entry = host_dict.get('horizontal_loop_begin') if host_dict else None ub_entry = host_dict.get('horizontal_loop_end') if host_dict else None lb = lb_entry.local_name if lb_entry else 'horizontal_loop_begin' diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 2291a79e..34d345e6 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -38,7 +38,7 @@ already in the access path for DDT-component fields, but needed here for a DDT instance variable itself when passed directly, and for any flat-array dim that hits the same registered name. -- ``horizontal_dimension`` / ``horizontal_loop_extent`` → +- ``horizontal_dimension`` → ``:`` (all phases). The lower bound must resolve to ``1`` (i.e. be ``ccpp_constant_one`` or the integer literal ``1``). - Everything else → ``:`` where both bounds are @@ -69,11 +69,16 @@ is_scalar_index_dim, ) from metadata.variable_resolver import HostVarEntry, _resolve_subscript - -# Dimension standard names that map to horizontal loop bounds. +# dim-aliases: transient GFS-physics shim (delete this import and the +# canonical() call in _canonical_dim when the shim is removed). +from metadata import dim_aliases + +# Dimension standard names that map to horizontal loop bounds. The +# legacy spelling ``horizontal_loop_extent`` is rejected at parse time +# (see ``_FORBIDDEN_DIMENSION_NAMES`` in ccpp_capgen_ng.py) or rewritten +# by the ``--legacy-mode`` shim, so it can never appear here. _HORIZ_LOOP_DIMS: frozenset = frozenset({ 'horizontal_dimension', - 'horizontal_loop_extent', }) # Standard names for horizontal loop bounds and full horizontal dimension. @@ -642,13 +647,22 @@ def _canonical_dim(dim: str) -> str: Different lower bound describes a different sub-range and must not compare equal. - No name aliasing on the upper bound happens here. + No name aliasing on the upper bound happens here by default. ``horizontal_loop_extent`` and ``horizontal_dimension`` are different names — the :func:`metadata.legacy_compat` shim is the canonical place that rewrites the legacy name to the new one at parse time when ``--legacy-mode`` is enabled. Without that shim the two should never appear on opposite sides of a host/scheme pairing. + + The one *opt-in* exception is the GFS dim-aliases shim + (:mod:`metadata.dim_aliases`, ``--gfs-dim-aliases``): when enabled + it collapses a small audited list of physically-equivalent + standard names (e.g. ``adjusted_vertical_layer_dimension_for_radiation`` + -> ``vertical_layer_dimension``) on the *upper bound only*, so + host and scheme metadata that use different historical spellings + of the same axis compare equal here. Variables keep their + original standard names elsewhere. """ if ':' in dim: lower, upper = dim.split(':', 1) @@ -661,6 +675,11 @@ def _canonical_dim(dim: str) -> str: # spellings of the default lower bound compare equal. if lower == '1': lower = _CCPP_CONSTANT_ONE + # dim-aliases: transient GFS-physics shim. No-op unless the + # ``--gfs-dim-aliases`` CLI flag has been passed. Only the upper + # bound is rewritten; lower bounds (loop-begin control vars, etc.) + # never alias. + upper = dim_aliases.canonical(upper) return '{}:{}'.format(lower, upper) @@ -1485,9 +1504,11 @@ def _resolve_one_arg( # ---- per-position dimension identity check --------------------------- # Each dimension entry is canonicalized to ``lower:upper`` form (bare # ``X`` -> ``ccpp_constant_one:X``) and compared for strict identity. - # No name aliasing: ``horizontal_loop_extent`` and ``horizontal_dimension`` - # are distinct; the legacy-compat shim is the one place that rewrites - # legacy names at parse time. + # No name aliasing happens here by default; the legacy-compat shim + # rewrites deprecated names at parse time, and the opt-in GFS + # dim-aliases shim (--gfs-dim-aliases) collapses a small audited + # list of physically-equivalent upper-bound names inside + # ``_canonical_dim`` itself. for pos, (hdim, sdim) in enumerate(zip(host_dims, scheme_var.dimensions)): if _canonical_dim(hdim) != _canonical_dim(sdim): raise CCPPError( diff --git a/capgen-ng/metadata/dim_aliases.py b/capgen-ng/metadata/dim_aliases.py new file mode 100644 index 00000000..fe9f8d1a --- /dev/null +++ b/capgen-ng/metadata/dim_aliases.py @@ -0,0 +1,196 @@ +"""TRANSIENT GFS-physics compatibility shim for equivalent dim names. + +A handful of CCPP-physics scheme groups (notably GFS radiation and +GFS chemistry / aerosol composition) declare array dimensions using +standard names that are *physically* the vertical layer dimension but +spelled differently because the legacy code path carries the historical +name around for clarity (e.g. ``adjusted_vertical_layer_dimension_for_radiation`` +on the radiation side, ``vertical_composition_dimension`` on the +composition side). + +These names cannot simply be substituted for ``vertical_layer_dimension`` +at parse time — each one is also exposed by some hosts (and consumed by +some schemes) as a standalone scalar control variable, so the +:mod:`metadata.legacy_compat` style of name rewriting would erase a +distinct variable. Instead we want them treated as *equivalent only at +the point where two metadata dimension entries (host side and scheme +side) are compared for identity*. + +This module provides an opt-in shim (``--gfs-dim-aliases`` on the +capgen-ng / ccpp_validator CLI) that collapses each member of an alias +group to a single canonical representative when ``_canonical_dim`` +prepares a dimension entry for the strict identity comparison in +:func:`generator.suite_resolver._check_compat`. Every other consumer +keeps the original name verbatim. + +This module is **deliberately self-contained** so the workaround can +be undone with a clean delete. Removing the feature is: + +1. Delete ``metadata/dim_aliases.py`` +2. Delete ``unit-tests/test_dim_aliases.py`` +3. ``grep -rn 'gfs-dim-aliases\\|dim_aliases\\|--gfs-dim-aliases' .`` and + remove every remaining touchpoint (each is a 1-3 line snippet + marked with a ``# dim-aliases:`` comment). + +Every hook in the rest of the codebase is a no-op when the mode is +not enabled, so the shim has zero impact on default workflows. + +Examples +-------- +>>> from metadata import dim_aliases +>>> dim_aliases.is_enabled() +False +>>> dim_aliases.canonical('adjusted_vertical_layer_dimension_for_radiation') +'adjusted_vertical_layer_dimension_for_radiation' + +When enabled, each alias collapses to its group representative: + +>>> import io, logging +>>> logger = logging.getLogger('dim_aliases_doctest') +>>> dim_aliases.enable(logger, _stream=io.StringIO()) +>>> dim_aliases.is_enabled() +True +>>> dim_aliases.canonical('adjusted_vertical_layer_dimension_for_radiation') +'vertical_layer_dimension' +>>> dim_aliases.canonical('vertical_composition_dimension') +'vertical_layer_dimension' +>>> dim_aliases.canonical('air_temperature') +'air_temperature' +>>> dim_aliases.disable() +>>> dim_aliases.is_enabled() +False +""" + +from __future__ import annotations + +import sys +from typing import Dict, Optional, TextIO + + +# ---------------------------------------------------------------------- +# Alias-member -> canonical representative. +# +# Keep this list short and audited. Each entry is a deliberate +# decision that a name is physically the same axis as the +# representative *for the purpose of host/scheme dim comparison*. +# The names remain distinct as standalone variables everywhere else. +# ---------------------------------------------------------------------- +_DIM_ALIAS_MAP: Dict[str, str] = { + # GFS radiation carries a separately-named "adjusted" vertical + # layer dimension that, in practice, is the same axis as + # vertical_layer_dimension. Collapse for the per-position dim + # identity check only. + 'adjusted_vertical_layer_dimension_for_radiation': + 'vertical_layer_dimension', + # GFS chemistry / aerosol composition uses + # vertical_composition_dimension where the layer count is meant. + 'vertical_composition_dimension': + 'vertical_layer_dimension', +} + + +# Process-level on/off flag. Module state is intentional: a single +# CLI invocation is the natural unit, and threading the flag through +# every parse call would bloat the API. Tests must use the +# ``disable()`` helper to restore the default between cases. +_ENABLED: bool = False + + +def enable(logger=None, _stream: Optional[TextIO] = None) -> None: + """Turn the GFS dim-aliases shim on and emit a bold warning banner. + + The warning goes to *_stream* (defaults to ``sys.stderr``) and is + also logged at WARNING level on *logger* (if supplied) so that + downstream consumers of the logger see it. + + Idempotent: a second call is a no-op (no double warning). + + Parameters + ---------- + logger : logging.Logger, optional + Logger to emit a ``WARNING``-level companion message on. + _stream : file-like, optional + Override for the banner destination (used by tests to capture + output). ``None`` (default) means ``sys.stderr``. + """ + global _ENABLED + if _ENABLED: + return + _ENABLED = True + + stream = _stream if _stream is not None else sys.stderr + border_width = 70 + border = '*' * border_width + _content_width = border_width - len('*** ') - len(' ***') + + def _pad(s: str) -> str: + """Format *s* as a banner row, left-padded to the border width.""" + return '*** {:<{w}} ***'.format(s[:_content_width], w=_content_width) + + banner_lines = ['', border, _pad('WARNING: GFS DIM-ALIASES ENABLED'), + _pad('')] + banner_lines += [ + _pad('The following dimension standard names will be'), + _pad('treated as equivalent to their canonical axis ONLY'), + _pad('during host/scheme dim-position comparison:'), + _pad(''), + ] + for alias, canon in sorted(_DIM_ALIAS_MAP.items()): + banner_lines.append(_pad(" '{}' => '{}'".format(alias, canon))) + banner_lines += [ + _pad(''), + _pad('Variables keep their original names everywhere'), + _pad('else. This is a TRANSIENT GFS-physics shim and'), + _pad('WILL BE REMOVED in a future capgen-ng release.'), + border, + '', + ] + stream.write('\n'.join(banner_lines) + '\n') + try: + stream.flush() + except Exception: # pylint: disable=broad-except + pass + + if logger is not None: + pair_str = ', '.join( + "'{}' => '{}'".format(alias, canon) + for alias, canon in sorted(_DIM_ALIAS_MAP.items()) + ) + logger.warning( + "GFS dim-aliases enabled: the following dimension names " + "will compare equal to their canonical axis: %s. This " + "shim is transient and will be removed.", + pair_str, + ) + + +def disable() -> None: + """Turn the dim-aliases shim off. Intended for tests and library + users that wrap a generator invocation.""" + global _ENABLED + _ENABLED = False + + +def is_enabled() -> bool: + """Return ``True`` iff the dim-aliases shim is enabled in this + process.""" + return _ENABLED + + +def canonical(name: str) -> str: + """Return the canonical representative for *name*, or *name* + unchanged. + + When the shim is **disabled** this is a strict identity — the + aliased names compare distinct, just as they would in a default + capgen-ng workflow. When the shim is **enabled** every entry in + :data:`_DIM_ALIAS_MAP` collapses to its representative; everything + else passes through unchanged. + + Intended for use by :func:`generator.suite_resolver._canonical_dim` + on the *upper bound* of a dimension entry; no other call site + should consult this function. + """ + if not _ENABLED: + return name + return _DIM_ALIAS_MAP.get(name, name) diff --git a/capgen-ng/metadata/metadata_table.py b/capgen-ng/metadata/metadata_table.py index 51bef9dd..7cfe7a4c 100644 --- a/capgen-ng/metadata/metadata_table.py +++ b/capgen-ng/metadata/metadata_table.py @@ -459,13 +459,13 @@ class MetaVar: >>> from metadata.parse_tools import ParseContext >>> ctx = ParseContext(10, 'example.meta') >>> v = MetaVar('im', ctx) - >>> v.set_attr('standard_name', 'horizontal_loop_extent', ctx) + >>> v.set_attr('standard_name', 'horizontal_dimension', ctx) >>> v.set_attr('units', 'count', ctx) >>> v.set_attr('dimensions', '()', ctx) >>> v.set_attr('type', 'integer', ctx) >>> v.set_attr('intent', 'in', ctx) >>> v.standard_name - 'horizontal_loop_extent' + 'horizontal_dimension' >>> v.intent 'in' >>> v.dimensions @@ -1150,7 +1150,7 @@ def parse_metadata_file(file_path: str) -> List[MetadataTable]: ... name = test_host ... type = host ... [ im ] - ... standard_name = horizontal_loop_extent + ... standard_name = horizontal_dimension ... units = count ... dimensions = () ... type = integer @@ -1168,7 +1168,7 @@ def parse_metadata_file(file_path: str) -> List[MetadataTable]: >>> tables[0].table_type 'host' >>> tables[0].sections()[0].variables[0].standard_name - 'horizontal_loop_extent' + 'horizontal_dimension' """ if not os.path.isfile(file_path): raise CCPPError("Metadata file '{}' does not exist".format(file_path)) diff --git a/capgen-ng/metadata/variable_resolver.py b/capgen-ng/metadata/variable_resolver.py index cae8aec0..dcde25e4 100644 --- a/capgen-ng/metadata/variable_resolver.py +++ b/capgen-ng/metadata/variable_resolver.py @@ -33,7 +33,7 @@ ``number_of_instances`` for the same role. Both names are treated as instance dimensions here; see ``_INSTANCE_DIMS``. -``horizontal_dimension``, ``horizontal_loop_extent`` +``horizontal_dimension`` Horizontal slice — emitted as ``lb:ub`` (run phase) or ``1:`` (non-run). The code generator handles the slicing; the resolver only records the dimension standard name. diff --git a/doc/HelloWorld/hello_scheme.meta b/doc/HelloWorld/hello_scheme.meta index a646c4cd..5c444adb 100644 --- a/doc/HelloWorld/hello_scheme.meta +++ b/doc/HelloWorld/hello_scheme.meta @@ -5,7 +5,7 @@ name = hello_scheme_run type = scheme [ ncol ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -33,14 +33,14 @@ [ temp_level ] standard_name = potential_temperature_at_interface units = K - dimensions = (ccpp_constant_one:horizontal_loop_extent, vertical_interface_dimension) + dimensions = (ccpp_constant_one:horizontal_dimension, vertical_interface_dimension) type = real kind = kind_phys intent = inout [ temp_layer ] standard_name = potential_temperature units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = out diff --git a/doc/HelloWorld/temp_adjust.meta b/doc/HelloWorld/temp_adjust.meta index 0f94ed01..15587e4d 100644 --- a/doc/HelloWorld/temp_adjust.meta +++ b/doc/HelloWorld/temp_adjust.meta @@ -5,7 +5,7 @@ name = temp_adjust_run type = scheme [ nbox ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension type = integer units = count dimensions = () @@ -19,7 +19,7 @@ [ temp_layer ] standard_name = potential_temperature units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout diff --git a/doc/redesign_analysis.md b/doc/redesign_analysis.md index 287e6ac2..c4e7469d 100644 --- a/doc/redesign_analysis.md +++ b/doc/redesign_analysis.md @@ -681,8 +681,8 @@ The `.meta` file itself uses an INI-style format: name = scheme_name_run type = scheme [ im ] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent + standard_name = horizontal_dimension + long_name = horizontal dimension units = count type = integer dimensions = () @@ -693,7 +693,7 @@ The `.meta` file itself uses an INI-style format: units = m type = real kind = kind_phys - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) intent = in ``` diff --git a/unit-tests/sample_files/bad_duplicate_stdname.meta b/unit-tests/sample_files/bad_duplicate_stdname.meta index 07dc00a8..f7e49f41 100644 --- a/unit-tests/sample_files/bad_duplicate_stdname.meta +++ b/unit-tests/sample_files/bad_duplicate_stdname.meta @@ -8,13 +8,13 @@ name = dup_scheme_run type = scheme [ a_var ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension units = count dimensions = () type = integer intent = in [ b_var ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension units = count dimensions = () type = integer diff --git a/unit-tests/sample_files/host_full.meta b/unit-tests/sample_files/host_full.meta index 854db157..38d60efa 100644 --- a/unit-tests/sample_files/host_full.meta +++ b/unit-tests/sample_files/host_full.meta @@ -16,12 +16,6 @@ units = count dimensions = () type = integer -[ im ] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent (ub - lb + 1) - units = count - dimensions = () - type = integer [ nlev ] standard_name = vertical_layer_dimension long_name = number of vertical layers diff --git a/unit-tests/sample_files/host_no_instance.meta b/unit-tests/sample_files/host_no_instance.meta index 92d5a383..66186c6f 100644 --- a/unit-tests/sample_files/host_no_instance.meta +++ b/unit-tests/sample_files/host_no_instance.meta @@ -15,12 +15,6 @@ units = count dimensions = () type = integer -[ im ] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent (ub - lb + 1) - units = count - dimensions = () - type = integer [ nlev ] standard_name = vertical_layer_dimension long_name = number of vertical layers diff --git a/unit-tests/sample_files/host_unit_conv.meta b/unit-tests/sample_files/host_unit_conv.meta index de0c3b98..07c421d7 100644 --- a/unit-tests/sample_files/host_unit_conv.meta +++ b/unit-tests/sample_files/host_unit_conv.meta @@ -16,12 +16,6 @@ units = count dimensions = () type = integer -[ im ] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - dimensions = () - type = integer [ data_array ] standard_name = data_array long_name = mandatory data array in m diff --git a/unit-tests/sample_files/host_with_dependencies.meta b/unit-tests/sample_files/host_with_dependencies.meta index 54d7a99d..ecc26dde 100644 --- a/unit-tests/sample_files/host_with_dependencies.meta +++ b/unit-tests/sample_files/host_with_dependencies.meta @@ -19,12 +19,6 @@ units = count dimensions = () type = integer -[ im ] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - dimensions = () - type = integer [ nlev ] standard_name = vertical_layer_dimension long_name = number of vertical layers diff --git a/unit-tests/sample_files/scheme_module_name_override.meta b/unit-tests/sample_files/scheme_module_name_override.meta index f5902861..a2ba5935 100644 --- a/unit-tests/sample_files/scheme_module_name_override.meta +++ b/unit-tests/sample_files/scheme_module_name_override.meta @@ -12,7 +12,7 @@ name = scheme_alt_name_run type = scheme [ im ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension units = count dimensions = () type = integer diff --git a/unit-tests/test_dim_aliases.py b/unit-tests/test_dim_aliases.py new file mode 100644 index 00000000..e2fb1159 --- /dev/null +++ b/unit-tests/test_dim_aliases.py @@ -0,0 +1,223 @@ +"""Tests for the transient GFS dim-aliases shim. + +This whole file is part of the dim-aliases shim and should be deleted +alongside ``metadata/dim_aliases.py`` when the GFS-physics rename is +complete. Search ``dim-aliases`` / ``gfs-dim-aliases`` to find every +touchpoint. +""" + +from __future__ import annotations + +import doctest +import io +import logging +import os +import sys +import unittest + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen-ng') +if _CAPGEN_DIR not in sys.path: + sys.path.insert(0, _CAPGEN_DIR) + +from metadata import dim_aliases # noqa: E402 +from generator.suite_resolver import _canonical_dim # noqa: E402 + + +class _DimAliasesFixture(unittest.TestCase): + """Mixin that flips the shim on for the duration of a test and + guarantees it goes back off afterwards (the flag is process state). + """ + + def setUp(self): + # Sanity: never start a test with the flag set by an earlier + # test that crashed before cleanup. + dim_aliases.disable() + + def tearDown(self): + dim_aliases.disable() + + +class TestCanonicalOff(_DimAliasesFixture): + """When the shim is disabled, ``canonical`` is a strict identity.""" + + def test_identity_on_aliased_name_when_disabled(self): + self.assertFalse(dim_aliases.is_enabled()) + self.assertEqual( + dim_aliases.canonical( + 'adjusted_vertical_layer_dimension_for_radiation'), + 'adjusted_vertical_layer_dimension_for_radiation', + ) + self.assertEqual( + dim_aliases.canonical('vertical_composition_dimension'), + 'vertical_composition_dimension', + ) + + def test_identity_on_unknown_name(self): + self.assertEqual( + dim_aliases.canonical('air_temperature'), 'air_temperature', + ) + + +class TestEnableDisable(_DimAliasesFixture): + + def test_enable_flips_flag_and_writes_banner(self): + sink = io.StringIO() + dim_aliases.enable(_stream=sink) + self.assertTrue(dim_aliases.is_enabled()) + out = sink.getvalue() + # Bold banner: starred border, both alias pairs, transient marker. + self.assertIn('GFS DIM-ALIASES ENABLED', out) + self.assertIn('adjusted_vertical_layer_dimension_for_radiation', out) + self.assertIn('vertical_composition_dimension', out) + self.assertIn('vertical_layer_dimension', out) + self.assertIn('TRANSIENT', out) + self.assertIn('REMOVED', out) + self.assertGreaterEqual(out.count('*' * 10), 2) + + def test_enable_is_idempotent(self): + sink1 = io.StringIO() + dim_aliases.enable(_stream=sink1) + first = sink1.getvalue() + sink2 = io.StringIO() + dim_aliases.enable(_stream=sink2) # second call — no banner + self.assertEqual(sink2.getvalue(), '') + self.assertIn('GFS DIM-ALIASES', first) + + def test_disable_resets(self): + dim_aliases.enable(_stream=io.StringIO()) + self.assertTrue(dim_aliases.is_enabled()) + dim_aliases.disable() + self.assertFalse(dim_aliases.is_enabled()) + + def test_logger_receives_warning(self): + logger = logging.getLogger('dim_aliases_test_logger') + records = [] + + class _Capture(logging.Handler): + def emit(self, record): + records.append(record) + + handler = _Capture(level=logging.WARNING) + logger.addHandler(handler) + try: + dim_aliases.enable(logger=logger, _stream=io.StringIO()) + self.assertEqual(len(records), 1) + self.assertEqual(records[0].levelno, logging.WARNING) + msg = records[0].getMessage() + self.assertIn('adjusted_vertical_layer_dimension_for_radiation', + msg) + self.assertIn('vertical_composition_dimension', msg) + self.assertIn('vertical_layer_dimension', msg) + finally: + logger.removeHandler(handler) + + +class TestCanonicalOn(_DimAliasesFixture): + """When the shim is enabled, the documented map applies.""" + + def setUp(self): + super().setUp() + dim_aliases.enable(_stream=io.StringIO()) + + def test_adjusted_radiation_collapses(self): + self.assertEqual( + dim_aliases.canonical( + 'adjusted_vertical_layer_dimension_for_radiation'), + 'vertical_layer_dimension', + ) + + def test_composition_collapses(self): + self.assertEqual( + dim_aliases.canonical('vertical_composition_dimension'), + 'vertical_layer_dimension', + ) + + def test_unknown_name_passes_through(self): + self.assertEqual( + dim_aliases.canonical('air_temperature'), 'air_temperature', + ) + + def test_canonical_target_is_idempotent(self): + # The representative itself is not in the alias map; calling + # canonical on it must be a no-op so repeated normalisation + # never drifts. + self.assertEqual( + dim_aliases.canonical('vertical_layer_dimension'), + 'vertical_layer_dimension', + ) + + +######################################################################## +# Integration through suite_resolver._canonical_dim +######################################################################## + +class TestCanonicalDimHook(_DimAliasesFixture): + """``_canonical_dim`` calls ``dim_aliases.canonical`` on the upper + bound only. Disabled mode keeps the original spelling; enabled + mode collapses every member of an alias group to one + representative so the per-position dim-identity check accepts the + pairing. + """ + + def test_disabled_keeps_aliased_dim_distinct(self): + a = _canonical_dim('adjusted_vertical_layer_dimension_for_radiation') + b = _canonical_dim('vertical_layer_dimension') + self.assertNotEqual(a, b) + + def test_enabled_collapses_radiation_alias(self): + dim_aliases.enable(_stream=io.StringIO()) + a = _canonical_dim('adjusted_vertical_layer_dimension_for_radiation') + b = _canonical_dim('vertical_layer_dimension') + self.assertEqual(a, b) + # And the collapsed form is the canonical representative. + self.assertEqual(a, 'ccpp_constant_one:vertical_layer_dimension') + + def test_enabled_collapses_composition_alias(self): + dim_aliases.enable(_stream=io.StringIO()) + a = _canonical_dim('vertical_composition_dimension') + b = _canonical_dim('vertical_layer_dimension') + self.assertEqual(a, b) + self.assertEqual(a, 'ccpp_constant_one:vertical_layer_dimension') + + def test_enabled_collapses_in_explicit_range_form(self): + # Aliasing applies on the *upper* bound of an explicit lower:upper + # form too. Lower bound never aliases. + dim_aliases.enable(_stream=io.StringIO()) + a = _canonical_dim( + 'ccpp_constant_one:vertical_composition_dimension') + b = _canonical_dim('vertical_layer_dimension') + self.assertEqual(a, b) + + def test_enabled_does_not_alias_unrelated_dim(self): + dim_aliases.enable(_stream=io.StringIO()) + a = _canonical_dim('horizontal_dimension') + b = _canonical_dim('vertical_layer_dimension') + self.assertNotEqual(a, b) + + def test_enabled_does_not_alias_lower_bound(self): + # Only the upper bound is rewritten. If somebody wrote + # 'adjusted_vertical_layer_dimension_for_radiation:foo' as a + # range, the lower bound stays verbatim — there is no host + # context in which an alias name appears as a loop-begin + # control var, so we keep this strict. + dim_aliases.enable(_stream=io.StringIO()) + a = _canonical_dim( + 'adjusted_vertical_layer_dimension_for_radiation:foo') + self.assertEqual( + a, + 'adjusted_vertical_layer_dimension_for_radiation:foo', + ) + + +######################################################################## +# Doctest loader for dim_aliases module +######################################################################## + +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(dim_aliases)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/unit-tests/test_host_cap.py b/unit-tests/test_host_cap.py index 0a16c802..e0ca78e9 100644 --- a/unit-tests/test_host_cap.py +++ b/unit-tests/test_host_cap.py @@ -803,11 +803,6 @@ def setUp(self): units = count dimensions = () type = integer -[ im ] - standard_name = horizontal_loop_extent - units = count - dimensions = () - type = integer [ flag_passive ] standard_name = flag_for_passive_check units = flag diff --git a/unit-tests/test_metadata_table.py b/unit-tests/test_metadata_table.py index 89609ea4..83bdb416 100644 --- a/unit-tests/test_metadata_table.py +++ b/unit-tests/test_metadata_table.py @@ -313,8 +313,8 @@ def test_standard_name_lowercased(self): """Standard names must be CF names (lowercased by the checker).""" ctx = _ctx() var = MetaVar('v', ctx) - var.set_attr('standard_name', 'Horizontal_Loop_Extent', ctx) - self.assertEqual(var.standard_name, 'horizontal_loop_extent') + var.set_attr('standard_name', 'Horizontal_Dimension', ctx) + self.assertEqual(var.standard_name, 'horizontal_dimension') def test_dimensions_scalar(self): var = self._make_var(dimensions='()') @@ -729,7 +729,7 @@ def test_scheme_three_phases(self): name = my_scheme_run type = scheme [ im ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension units = count dimensions = () type = integer @@ -778,7 +778,7 @@ def test_two_tables_in_one_file(self): name = my_scheme_run type = scheme [ im ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension units = count dimensions = () type = integer @@ -931,13 +931,13 @@ def test_duplicate_standard_name_in_section(self): name = s_run type = scheme [ a ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension units = count dimensions = () type = integer intent = in [ b ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension units = count dimensions = () type = integer @@ -957,7 +957,7 @@ def test_missing_intent_for_scheme_variable(self): name = s_run type = scheme [ im ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension units = count dimensions = () type = integer @@ -975,7 +975,7 @@ def test_invalid_intent_value(self): name = s_run type = scheme [ im ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension units = count dimensions = () type = integer @@ -1483,7 +1483,7 @@ def test_dedup_across_phases(self): type = character intent = out [ im ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension units = count dimensions = () type = integer @@ -1494,7 +1494,7 @@ def test_dedup_across_phases(self): snames = [v.standard_name for v in all_vars] # ccpp_error_message appears in both phases but should be returned once self.assertEqual(snames.count('ccpp_error_message'), 1) - self.assertIn('horizontal_loop_extent', snames) + self.assertIn('horizontal_dimension', snames) ######################################################################## diff --git a/unit-tests/test_suite_data.py b/unit-tests/test_suite_data.py index fcc438f1..4fabcf7e 100644 --- a/unit-tests/test_suite_data.py +++ b/unit-tests/test_suite_data.py @@ -68,11 +68,11 @@ def setUp(self): suite_vars = { 'air_temp_adjusted': _make_sv( 'air_temp_adjusted', 'temp_adj', 'real', 'kind_phys', 'K', - dims=['horizontal_loop_extent', 'vertical_layer_dimension'], + dims=['horizontal_dimension', 'vertical_layer_dimension'], ), 'humidity': _make_sv( 'humidity', 'q', 'real', 'kind_phys', 'kg kg-1', - dims=['horizontal_loop_extent'], + dims=['horizontal_dimension'], ), } self.lines = _generate_suite_data('suite_x', suite_vars) diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 9d844e53..5bab24a3 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -1105,11 +1105,15 @@ def _scheme_var(self, local, std_name, intent='in', units='1', def test_case1_direct_host(self): """Case 1: scalar host variable, no transform.""" hd = self._host_dict() - suite_var = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count') + suite_var = self._scheme_var( + 'im', 'horizontal_dimension', 'in', 'count') arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) self.assertEqual(arg.source, 'host') self.assertEqual(arg.transform_case, 1) - self.assertEqual(arg.call_expr, 'im') + # Scalar horizontal_dimension in run phase is synthesised from + # the chunk loop bounds (ub - lb + 1) rather than the host's + # full-domain ncols — see ``_HORIZ_DIM_STD`` in suite_resolver. + self.assertEqual(arg.call_expr, '(ub - lb + 1)') self.assertFalse(arg.needs_transform) def test_case1_control_var(self): @@ -1199,7 +1203,8 @@ def test_unit_transform_detected(self): def test_no_transform_same_units(self): """Identical units → no transformation.""" hd = self._host_dict() - suite_var = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count') + suite_var = self._scheme_var( + 'im', 'horizontal_dimension', 'in', 'count') arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) self.assertFalse(arg.needs_transform) @@ -1228,8 +1233,8 @@ def test_unknown_unit_mismatch_raises(self): def test_optional_sets_ptr_name(self): """Optional argument → ptr_name set.""" hd = self._host_dict() - suite_var = self._scheme_var('im', 'horizontal_loop_extent', 'in', 'count', - optional=True) + suite_var = self._scheme_var( + 'im', 'horizontal_dimension', 'in', 'count', optional=True) arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'my_scheme', set()) self.assertTrue(arg.is_optional) self.assertTrue(arg.ptr_name) @@ -1624,11 +1629,6 @@ def _build_host_and_scheme( units = count dimensions = () type = integer -[ im ] - standard_name = horizontal_loop_extent - units = count - dimensions = () - type = integer [ nlev ] standard_name = vertical_layer_dimension units = count @@ -2400,13 +2400,6 @@ def test_horizontal_dimension_uses_chunk_bounds(self): ', dimension(lb:ub)', ) - def test_horizontal_loop_extent_uses_chunk_bounds(self): - # Same special case for the alternative dim std name. - self.assertEqual( - _dim_decl_local(['horizontal_loop_extent'], self.hd), - ', dimension(lb:ub)', - ) - def test_vertical_dim_uses_local_name(self): # No special case for vertical dims — host's local name only. self.assertEqual( diff --git a/unit-tests/test_variable_resolver.py b/unit-tests/test_variable_resolver.py index 414fe0d2..2d2bf1e5 100644 --- a/unit-tests/test_variable_resolver.py +++ b/unit-tests/test_variable_resolver.py @@ -814,7 +814,7 @@ def test_empty_inputs(self): name = my_scheme_run type = scheme [ im ] - standard_name = horizontal_loop_extent + standard_name = horizontal_dimension units = count dimensions = () type = integer @@ -822,7 +822,7 @@ def test_empty_inputs(self): [ temp ] standard_name = air_temperature units = K - dimensions = (horizontal_loop_extent, vertical_layer_dimension) + dimensions = (horizontal_dimension, vertical_layer_dimension) type = real kind = kind_phys intent = inout @@ -1068,7 +1068,7 @@ def test_variables_for_run(self): vars_ = store.variables_for('my_scheme', 'run') self.assertIsNotNone(vars_) std_names = [v.standard_name for v in vars_] - self.assertEqual(std_names, ['horizontal_loop_extent', 'air_temperature']) + self.assertEqual(std_names, ['horizontal_dimension', 'air_temperature']) def test_variables_for_init(self): store = self._build() From b21b6acb03daeb5cd6fbcd2433ea1b2853f9864f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 21 May 2026 05:42:00 -0600 Subject: [PATCH 41/74] Remove unused argument --gfs-dim-names from ccpp_validator.py --- capgen-ng/ccpp_validator.py | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/capgen-ng/ccpp_validator.py b/capgen-ng/ccpp_validator.py index b5cde2e0..08a2ef65 100755 --- a/capgen-ng/ccpp_validator.py +++ b/capgen-ng/ccpp_validator.py @@ -1067,24 +1067,11 @@ def _build_parser() -> argparse.ArgumentParser: "loud warning at startup. Will be removed." ), ) - # dim-aliases: transient GFS-physics shim (delete the argument, - # the enable() call below, and the rest of the dim_aliases - # touchpoints when the workaround is removed). - parser.add_argument( - '--gfs-dim-aliases', - action='store_true', - help=( - "TRANSIENT GFS-PHYSICS SHIM. Treat a small audited list of " - "physically-equivalent vertical-axis standard names as the " - "same dimension during the host/scheme dim-position " - "identity check only (e.g. " - "'adjusted_vertical_layer_dimension_for_radiation' and " - "'vertical_composition_dimension' both compare equal to " - "'vertical_layer_dimension'). Variables keep their " - "original names everywhere else. Emits a loud warning at " - "startup. Will be removed." - ), - ) + # NB: --gfs-dim-aliases is NOT exposed here. That shim only takes + # effect inside generator.suite_resolver._canonical_dim, which the + # validator never invokes (the validator compares metadata against + # Fortran source, not host metadata against scheme metadata), so + # the flag would be a no-op. See capgen-ng/ccpp_capgen_ng.py. return parser @@ -1105,12 +1092,6 @@ def main(argv: Optional[List[str]] = None) -> int: from metadata import legacy_compat legacy_compat.enable(_LOGGER) - # dim-aliases: transient GFS-physics shim. Emit the loud banner - # before any parsing happens so user has fair warning. - if args.gfs_dim_aliases: - from metadata import dim_aliases - dim_aliases.enable(_LOGGER) - scheme_files = [f.strip() for f in args.scheme_files.split(',') if f.strip()] source_files = [f.strip() for f in args.source_files.split(',') if f.strip()] From 020415383c357b37071a9c8b41f2f01298b0f550 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 21 May 2026 15:34:48 -0600 Subject: [PATCH 42/74] Add legacy feature to support auto-cloned constituents --- capgen-ng/ccpp_capgen_ng.py | 35 + capgen-ng/ccpp_validator.py | 29 + capgen-ng/generator/suite_cap.py | 157 ++- capgen-ng/generator/suite_resolver.py | 203 +++ capgen-ng/metadata/auto_clone_constituents.py | 243 ++++ capgen-ng/metadata/metadata_table.py | 68 +- capgen-ng/metadata/parse_tools/__init__.py | 7 + .../metadata/parse_tools/parse_checkers.py | 205 +++ doc/auto_clone_constituents.md | 150 +++ end-to-end-tests/CMakeLists.txt | 3 + .../advection_auto_clone/CMakeLists.txt | 93 ++ .../advection_auto_clone/README.md | 10 + .../advection_test_reports.py | 127 ++ .../apply_constituent_tendencies.F90 | 39 + .../apply_constituent_tendencies.meta | 36 + .../advection_auto_clone/cld_ice.F90 | 125 ++ .../advection_auto_clone/cld_ice.meta | 137 ++ .../advection_auto_clone/cld_liq.F90 | 102 ++ .../advection_auto_clone/cld_liq.meta | 137 ++ .../advection_auto_clone/cld_suite.xml | 11 + .../advection_auto_clone/cld_suite_error.xml | 9 + .../advection_auto_clone/const_indices.F90 | 95 ++ .../advection_auto_clone/const_indices.meta | 108 ++ .../advection_auto_clone/dlc_liq.F90 | 41 + .../advection_auto_clone/dlc_liq.meta | 29 + .../test_advection_host_integration.F90 | 79 ++ .../advection_auto_clone/test_host.F90 | 1160 +++++++++++++++++ .../advection_auto_clone/test_host.meta | 70 + .../advection_auto_clone/test_host_data.F90 | 96 ++ .../advection_auto_clone/test_host_data.meta | 67 + .../advection_auto_clone/test_host_mod.F90 | 176 +++ .../advection_auto_clone/test_host_mod.meta | 64 + end-to-end-tests/cmake/ccpp_capgen.cmake | 12 +- .../scheme_auto_clone_consumer.meta | 48 + .../sample_suite_files/suite_auto_clone.xml | 9 + unit-tests/test_auto_clone_constituents.py | 693 ++++++++++ 36 files changed, 4662 insertions(+), 11 deletions(-) create mode 100644 capgen-ng/metadata/auto_clone_constituents.py create mode 100644 doc/auto_clone_constituents.md create mode 100644 end-to-end-tests/advection_auto_clone/CMakeLists.txt create mode 100644 end-to-end-tests/advection_auto_clone/README.md create mode 100644 end-to-end-tests/advection_auto_clone/advection_test_reports.py create mode 100644 end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.F90 create mode 100644 end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.meta create mode 100644 end-to-end-tests/advection_auto_clone/cld_ice.F90 create mode 100644 end-to-end-tests/advection_auto_clone/cld_ice.meta create mode 100644 end-to-end-tests/advection_auto_clone/cld_liq.F90 create mode 100644 end-to-end-tests/advection_auto_clone/cld_liq.meta create mode 100644 end-to-end-tests/advection_auto_clone/cld_suite.xml create mode 100644 end-to-end-tests/advection_auto_clone/cld_suite_error.xml create mode 100644 end-to-end-tests/advection_auto_clone/const_indices.F90 create mode 100644 end-to-end-tests/advection_auto_clone/const_indices.meta create mode 100644 end-to-end-tests/advection_auto_clone/dlc_liq.F90 create mode 100644 end-to-end-tests/advection_auto_clone/dlc_liq.meta create mode 100644 end-to-end-tests/advection_auto_clone/test_advection_host_integration.F90 create mode 100644 end-to-end-tests/advection_auto_clone/test_host.F90 create mode 100644 end-to-end-tests/advection_auto_clone/test_host.meta create mode 100644 end-to-end-tests/advection_auto_clone/test_host_data.F90 create mode 100644 end-to-end-tests/advection_auto_clone/test_host_data.meta create mode 100644 end-to-end-tests/advection_auto_clone/test_host_mod.F90 create mode 100644 end-to-end-tests/advection_auto_clone/test_host_mod.meta create mode 100644 unit-tests/sample_files/scheme_auto_clone_consumer.meta create mode 100644 unit-tests/sample_suite_files/suite_auto_clone.xml create mode 100644 unit-tests/test_auto_clone_constituents.py diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index 8d84bea8..dcc50c8c 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -273,6 +273,27 @@ def _build_arg_parser() -> argparse.ArgumentParser: "startup. Will be removed." ), ) + # auto-clone-constituents: transient legacy shim (delete the + # argument, the enable() call below, and the rest of the + # auto_clone_constituents touchpoints when legacy hosts have + # migrated to explicit registration). + parser.add_argument( + '--legacy-auto-clone-constituents', + action='store_true', + help=( + "TRANSIENT LEGACY SHIM. Replicate original capgen's " + "auto-clone-static-constituent path: every is_constituent " + "scheme arg (advected / constituent / molar_mass) without " + "an explicit register-phase source is auto-registered into " + "the per-suite dynamic-constituents buffer using values " + "lifted directly from its scheme metadata. Accepts four " + "legacy attributes on scheme args (default_value, " + "min_value, water_species, mixing_ratio_type). " + "SINGLE-INSTANCE ONLY — the host must not declare the " + "(instance_number, number_of_instances) multi-instance " + "pair. Emits a loud warning at startup. Will be removed." + ), + ) parser.add_argument( '--no-host-introspection', action='store_true', @@ -894,6 +915,11 @@ def capgen( # ---- Phase 1 validation: required control variables --------------------- _validate_required_control_vars(host_name, host_dict) + # auto-clone-constituents: enforce the single-instance constraint + # of the transient legacy shim. No-op when the shim is disabled. + from metadata import auto_clone_constituents + auto_clone_constituents.require_single_instance_host(host_dict) + # Signal which instance API the host opted into so users can tell which # branch the generator took. Paired-presence has already been enforced. if host_dict.get('instance_number') is not None: @@ -1139,6 +1165,15 @@ def main(argv: Optional[List[str]] = None) -> int: from metadata import dim_aliases dim_aliases.enable(_LOGGER) + # auto-clone-constituents: transient legacy shim. Emit the loud + # banner before any parsing happens so the user has fair warning. + # The single-instance assertion (host MUST NOT declare the + # instance_number / number_of_instances pair) runs later, after + # host metadata has been parsed, in ``capgen()``. + if args.legacy_auto_clone_constituents: + from metadata import auto_clone_constituents + auto_clone_constituents.enable(_LOGGER) + # ---- parse kind types -------------------------------------------------- try: kind_types = _parse_kind_types(args.kind_type) diff --git a/capgen-ng/ccpp_validator.py b/capgen-ng/ccpp_validator.py index 08a2ef65..cec30881 100755 --- a/capgen-ng/ccpp_validator.py +++ b/capgen-ng/ccpp_validator.py @@ -1072,6 +1072,27 @@ def _build_parser() -> argparse.ArgumentParser: # validator never invokes (the validator compares metadata against # Fortran source, not host metadata against scheme metadata), so # the flag would be a no-op. See capgen-ng/ccpp_capgen_ng.py. + # auto-clone-constituents: transient legacy shim. This one DOES + # belong on the validator because the shim extends the parser's + # ``_KNOWN_ATTRS`` set — without the flag the validator rejects + # the four legacy attrs (default_value/min_value/water_species/ + # mixing_ratio_type) with "Unknown variable attribute", which + # blocks pre-flight validation runs. The validator never builds + # a host_dict, so the shim's single-instance host guard is not + # called here (it's a no-op without a host metadata pass). + parser.add_argument( + '--legacy-auto-clone-constituents', + action='store_true', + help=( + "TRANSIENT LEGACY SHIM. Accept four legacy constituent " + "attributes (default_value, min_value, water_species, " + "mixing_ratio_type) on scheme args. Mirrors the same " + "flag on ccpp_capgen_ng so legacy scheme metadata that " + "needs auto-clone-static-constituent codegen can be " + "validated against its Fortran source. Emits a loud " + "warning at startup. Will be removed." + ), + ) return parser @@ -1092,6 +1113,14 @@ def main(argv: Optional[List[str]] = None) -> int: from metadata import legacy_compat legacy_compat.enable(_LOGGER) + # auto-clone-constituents: transient legacy shim. Emit the loud + # banner before any parsing happens so user has fair warning. + # The single-instance host guard is intentionally NOT invoked + # here (no host metadata pass in the validator). + if args.legacy_auto_clone_constituents: + from metadata import auto_clone_constituents + auto_clone_constituents.enable(_LOGGER) + scheme_files = [f.strip() for f in args.scheme_files.split(',') if f.strip()] source_files = [f.strip() for f in args.source_files.split(',') if f.strip()] diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 51567380..260bafa9 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -30,6 +30,8 @@ ResolvedGroup, SuiteResolution, iter_phase_calls, + # auto-clone-constituents: legacy-shim payload type. + AutoCloneEntry, ) from generator.trace import ( emit_module_gate, @@ -223,10 +225,24 @@ def _register_uses( # Per-suite dynamic-constituent buffer is owned by ccpp_host_constituents # and written into here. Pull in the constituent property type plus the # buffer symbol. - if suite_res.constituent_register_calls: + # auto-clone-constituents: the synthesised %instantiate calls + # also need the constituent property type and the buffer symbol, + # so include them when the auto-clone list is non-empty. + if suite_res.constituent_register_calls or suite_res.auto_cloned_constituents: uses.setdefault(_CONST_MOD, set()).add(_CONST_PROP_TYPE) buf = '{}_dynamic_constituents'.format(suite_name) uses.setdefault('ccpp_host_constituents', set()).add(buf) + # auto-clone-constituents: the synthesised %instantiate calls + # embed ``_kind_phys`` literals when an entry sets any of + # molar_mass / default_value / min_value. Pull in ``kind_phys`` + # from ccpp_kinds in that case so the literal resolves. + if any( + e.default_value is not None + or e.min_value is not None + or (e.molar_mass and e.molar_mass != 0.0) + for e in suite_res.auto_cloned_constituents + ): + uses.setdefault('ccpp_kinds', set()).add('kind_phys') return uses @@ -274,6 +290,90 @@ def _emit_register_call(resolved_call, indent: str, errflg_local: str, lines: Li lines.append('{}if ({} /= 0) return'.format(indent, errflg_local)) +# auto-clone-constituents: BEGIN legacy-shim emission helpers. +# Delete this block together with the rest of the +# auto-clone-constituents touchpoints. + +def _esc_fortran_char(value: str) -> str: + """Escape a Python string for embedding in a Fortran character + literal — double every embedded single quote.""" + return value.replace("'", "''") + + +def _fmt_kind_phys_real(value) -> str: + """Format a Python float as a Fortran ``kind_phys`` real literal. + + Always emits exponent notation so the result is unambiguously a + real (no risk of being parsed as an integer literal) and reuses + the same ``kind_phys`` suffix the framework's ``%instantiate`` + declares for ``default_value`` / ``min_value`` / ``molar_mass``. + """ + return '{:.17e}_kind_phys'.format(float(value)) + + +def _emit_auto_clone_instantiate( + entry: AutoCloneEntry, + buf: str, + inst_idx: str, + indent: str, + errflg_local: str, + errmsg_local: str, + lines: List[str], +) -> None: + """auto-clone-constituents: emit one synthesised ``%instantiate`` + call into the per-suite dynamic-constituents buffer for one + :class:`AutoCloneEntry`. + + Required kwargs (std_name, long_name, diag_name, units, + vertical_dim) are always emitted. Optional kwargs are emitted + only when the metadata explicitly set the value (``None`` on the + backing field means "unset" — let the framework default kick in). + ``errcode`` and ``errmsg`` are passed by keyword for clarity. + """ + i = indent + lines.append('{}num_consts = num_consts + 1'.format(i)) + lines.append( + '{}call {}({})%items(num_consts)%instantiate( &'.format( + i, buf, inst_idx, + ) + ) + lines.append("{} std_name = '{}', &".format( + i, _esc_fortran_char(entry.std_name))) + lines.append("{} long_name = '{}', &".format( + i, _esc_fortran_char(entry.long_name))) + lines.append("{} diag_name = '{}', &".format( + i, _esc_fortran_char(entry.diag_name))) + lines.append("{} units = '{}', &".format( + i, _esc_fortran_char(entry.units))) + lines.append("{} vertical_dim = '{}', &".format( + i, _esc_fortran_char(entry.vertical_dim))) + # Optional kwargs — only emit when explicitly set. ``advected`` + # defaults to .false. in metadata; only emit when True so we + # don't pollute the call with a redundant kwarg. + if entry.advected: + lines.append("{} advected = .true., &".format(i)) + if entry.molar_mass and entry.molar_mass != 0.0: + lines.append("{} molar_mass = {}, &".format( + i, _fmt_kind_phys_real(entry.molar_mass))) + if entry.default_value is not None: + lines.append("{} default_value= {}, &".format( + i, _fmt_kind_phys_real(entry.default_value))) + if entry.min_value is not None: + lines.append("{} min_value = {}, &".format( + i, _fmt_kind_phys_real(entry.min_value))) + if entry.water_species is not None: + lines.append("{} water_species= .{}., &".format( + i, 'true' if entry.water_species else 'false')) + if entry.mixing_ratio_type is not None: + lines.append("{} mixing_ratio_type = '{}', &".format( + i, _esc_fortran_char(entry.mixing_ratio_type))) + lines.append("{} errcode = {}, &".format(i, errflg_local)) + lines.append("{} errmsg = {})".format(i, errmsg_local)) + lines.append('{}if ({} /= 0) return'.format(i, errflg_local)) + +# auto-clone-constituents: END legacy-shim emission helpers. + + def _register_lines( suite_name: str, suite_res: SuiteResolution, @@ -340,14 +440,26 @@ def _register_lines( # Constituent merge: declare a per-scheme array temporary and a counter. has_consts = bool(suite_res.constituent_register_calls) - if has_consts: + # auto-clone-constituents: the legacy shim contributes additional + # constituent registrations synthesised in capgen-ng from + # is_constituent consumer metadata; ``has_dyn_consts`` covers + # both sources so the buffer-allocation + counter machinery is + # set up whenever any synthesised constituent will be emitted. + has_auto_cloned = bool(suite_res.auto_cloned_constituents) + has_dyn_consts = has_consts or has_auto_cloned + if has_dyn_consts: lines.append('') - lines.append( - '{}type({}), allocatable :: scheme_consts(:)'.format( - i2, _CONST_PROP_TYPE + if has_consts: + lines.append( + '{}type({}), allocatable :: scheme_consts(:)'.format( + i2, _CONST_PROP_TYPE + ) ) - ) - lines.append('{}integer :: num_consts, i'.format(i2)) + lines.append('{}integer :: num_consts, i'.format(i2)) + else: + # auto-clone-only path: no scheme-returned temp, no copy + # loop, so we don't need ``scheme_consts`` or ``i``. + lines.append('{}integer :: num_consts'.format(i2)) # Trace block: dummies referenced inside the gated write so strict # compilers don't flag intent(in) args as unused when the gate is off. @@ -383,7 +495,7 @@ def _register_lines( ) lines.append('') - if has_consts: + if has_dyn_consts: # Pack constituent-producing schemes' arrays into the per-suite # buffer in ccpp_host_constituents. The actual merge into each # instance's ``ccpp_model_constituents_obj(inst)`` happens later @@ -397,8 +509,15 @@ def _register_lines( # instance then runs its own two-pass count+pack into its slot. # The state-machine guard above this block ensures each instance # runs the fill at most once. + # + # auto-clone-constituents: the legacy shim contributes one + # additional ``%instantiate`` per consumer-side + # ``is_constituent`` arg with no register-phase source. Those + # synthesised entries participate in the same two-pass + # count+pack against the same per-instance buffer slot. const_scheme_names = {scheme_name for scheme_name, _ in suite_res.constituent_register_calls} buf = '{}_dynamic_constituents'.format(suite_name) + n_auto_clone = len(suite_res.auto_cloned_constituents) # Allocate the outer wrapper array on first call (any instance). lines.append( @@ -422,6 +541,15 @@ def _register_lines( ) ) lines.append('{}deallocate(scheme_consts)'.format(i2)) + # auto-clone-constituents: synthesised entries contribute a + # static count; add a literal at the end of the count pass. + if n_auto_clone > 0: + lines.append( + '{}! auto-clone-constituents: legacy-shim synthesised entries'.format( + i2, + ) + ) + lines.append('{}num_consts = num_consts + {}'.format(i2, n_auto_clone)) lines.append('') lines.append('{}allocate({}({})%items(num_consts))'.format( i2, buf, inst_idx, @@ -445,6 +573,19 @@ def _register_lines( ) ) lines.append('{}deallocate(scheme_consts)'.format(i2)) + # auto-clone-constituents: emit one synthesised %instantiate + # call per entry into the per-instance buffer slot. + if n_auto_clone > 0: + lines.append('') + lines.append( + '{}! auto-clone-constituents: legacy-shim synthesised %instantiate calls'.format( + i2, + ) + ) + for entry in suite_res.auto_cloned_constituents: + _emit_auto_clone_instantiate( + entry, buf, inst_idx, i2, errflg_local, errmsg_local, lines, + ) lines.append('') # Emit any non-constituent register calls in addition (always, per instance). for _gname, resolved_call in _register_calls(suite_res): diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 34d345e6..5d7b393d 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -72,6 +72,10 @@ # dim-aliases: transient GFS-physics shim (delete this import and the # canonical() call in _canonical_dim when the shim is removed). from metadata import dim_aliases +# auto-clone-constituents: transient legacy shim (delete this import +# and the AutoCloneEntry / _collect_auto_clone_entries touchpoints +# when the shim is removed). +from metadata import auto_clone_constituents # Dimension standard names that map to horizontal loop bounds. The # legacy spelling ``horizontal_loop_extent`` is rejected at parse time @@ -1142,6 +1146,27 @@ class ResolvedGroup: dim_uses: Dict[str, Set[str]] = field(default_factory=dict) +# auto-clone-constituents: snapshot of one consumer-side +# ``is_constituent`` scheme arg whose ``std_name`` has no +# register-phase source. Carries everything the emitter needs to +# synthesise a ``%instantiate(...)`` call in ``_register``. +# Captured at resolve time so the emitter doesn't reach back into +# raw scheme metadata. +@dataclass +class AutoCloneEntry: + std_name: str + long_name: str + diag_name: str # diagnostic_name or local_name fallback + units: str + vertical_dim: str # standard name of the vertical axis + advected: bool + molar_mass: float + default_value: Optional[float] + min_value: Optional[float] + water_species: Optional[bool] + mixing_ratio_type: Optional[str] + + @dataclass class SuiteResolution: """Complete resolution result for one suite. @@ -1189,6 +1214,12 @@ class SuiteResolution: uses_constituents: bool = False suite_init_call: Optional[ResolvedCall] = None suite_final_call: Optional[ResolvedCall] = None + # auto-clone-constituents: populated only when the legacy shim is + # enabled. Each entry is one is_constituent consumer whose + # std_name has no register-phase source; the suite cap synthesises + # a ``%instantiate(...)`` call per entry into the per-suite + # dynamic-constituents buffer. Empty when the shim is off. + auto_cloned_constituents: List[AutoCloneEntry] = field(default_factory=list) ######################################################################## @@ -2154,6 +2185,11 @@ def resolve_suite( ) ) + # auto-clone-constituents: collect synthesised %instantiate + # snapshots when the legacy shim is enabled. Returns [] when + # disabled (no-op for default builds). + auto_cloned = _collect_auto_clone_entries(resolved_groups, scheme_store) + return SuiteResolution( suite_name=suite.name, groups=resolved_groups, @@ -2164,7 +2200,174 @@ def resolve_suite( uses_constituents=uses_constituents, suite_init_call=suite_init_call, suite_final_call=suite_final_call, + # auto-clone-constituents: empty in default builds. + auto_cloned_constituents=auto_cloned, + ) + + +# auto-clone-constituents: BEGIN legacy-shim helpers. Delete this +# block together with the rest of the auto-clone-constituents +# touchpoints; nothing else in the resolver references these. + +def _vertical_dim_of(scheme_var) -> str: + """Extract the vertical-axis standard name from a constituent + consumer's ``dimensions`` list. + + Returns the upper-bound std name of the first vertical-axis + dimension entry (matches :data:`_VDIM_STDS`). Falls back to + ``'vertical_layer_dimension'`` when the scheme arg carries no + vertical dim (e.g. a 1-D horizontal-only consumer) — that's the + framework default and matches original capgen's behaviour. + """ + for dim in getattr(scheme_var, 'dimensions', ()) or (): + upper = dim.split(':', 1)[-1].strip().lower() if ':' in dim else dim.strip().lower() + if upper in _VDIM_STDS: + return upper + return 'vertical_layer_dimension' + + +def _synthesised_long_name_from_std(std_name: str) -> str: + """auto-clone-constituents: fall back to a human-readable long_name + derived from the std_name when the metadata doesn't supply one. + + Mirrors original capgen's auto-clone behaviour: replace each + underscore with a space, then capitalise the first character. + ``cloud_liquid_dry_mixing_ratio`` → ``Cloud liquid dry mixing ratio``. + + Keeps the auto-clone shim's emitted ``%instantiate(long_name=...)`` + consistent with what original capgen produces so existing legacy + fixtures (e.g. CAM-SIMA's advection_test) don't need a metadata + edit just for the long_name property. + """ + return std_name.replace('_', ' ').capitalize() + + +def _make_auto_clone_entry(scheme_var) -> 'AutoCloneEntry': + """Snapshot one scheme MetaVar into an :class:`AutoCloneEntry`. + + Captures every field the emitter needs to synthesise a + ``%instantiate(...)`` call: the required kwargs (std_name, + long_name, diag_name, units, vertical_dim) and the optional + kwargs (advected, molar_mass, default_value, min_value, + water_species, mixing_ratio_type) with ``None`` for any optional + the metadata didn't set so the emitter can omit it. + """ + # ``diagnostic_name`` is a property that falls back to local_name + # when neither diagnostic_name nor diagnostic_name_fixed was set. + diag = scheme_var.diagnostic_name or scheme_var.local_name + # auto-clone-constituents: when the scheme metadata omits the + # long_name attribute, synthesise one from the std_name (matches + # original capgen's behaviour — see ``_synthesised_long_name_from_std``). + long_name = ( + scheme_var.long_name + or _synthesised_long_name_from_std(scheme_var.standard_name) ) + return AutoCloneEntry( + std_name=scheme_var.standard_name, + long_name=long_name, + diag_name=diag, + units=scheme_var.units, + vertical_dim=_vertical_dim_of(scheme_var), + advected=bool(scheme_var.advected), + molar_mass=float(scheme_var.molar_mass or 0.0), + default_value=scheme_var.default_value, + min_value=scheme_var.min_value, + water_species=scheme_var.water_species, + mixing_ratio_type=scheme_var.mixing_ratio_type, + ) + + +def _lookup_scheme_var(scheme_store, scheme_name, phase, scheme_local_name): + """Return the scheme MetaVar matching *scheme_local_name* in + *scheme_name*'s *phase*, or ``None`` if not found. + + Used by the auto-clone collector to recover the full scheme + MetaVar (which carries the constituent-property fields) from a + ResolvedArg, which only carries a slim subset. + """ + vars_list = scheme_store.variables_for(scheme_name, phase) + if not vars_list: + return None + for mv in vars_list: + if mv.local_name == scheme_local_name: + return mv + return None + + +def _collect_auto_clone_entries(resolved_groups, scheme_store): + """Walk every ``is_constituent`` consumer arg and produce one + :class:`AutoCloneEntry` per unique standard name. + + Skips: + + * register-phase ``ccpp_constituent_properties_t`` args + (``is_constituent_arg`` is True for those — the scheme handles + its own registration explicitly); + * ``tendency_of_*`` std names (tendencies consume a constituent + but are not themselves a constituent registration); + * framework-named std names (``ccpp_constituents``, + ``ccpp_constituent_tendencies``, ``ccpp_constituent_properties``, + ``number_of_ccpp_constituents``, and ``index_of_*``) — these + reference the framework-provided constituent ARRAYS or scalar + counts, not individual species, and are not registrations; + * duplicates within the suite (first-occurrence wins; the + framework's runtime ``is_match`` would dedupe across the + eventual ``%new_field`` calls anyway). + + No-op when the auto-clone shim is disabled — returns an empty + list so :class:`SuiteResolution.auto_cloned_constituents` stays + empty in default builds. + """ + if not auto_clone_constituents.is_enabled(): + return [] + out: List['AutoCloneEntry'] = [] + seen: Set[str] = set() + for resolved_group in resolved_groups: + for items in resolved_group.phase_calls.values(): + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + if arg.source != 'constituent' or arg.is_constituent_arg: + continue + std = (arg.standard_name or '').strip().lower() + if not std or std.startswith('tendency_of_'): + continue + # auto-clone-constituents: framework-named std + # names (the whole constituent buffer / tendency + # buffer / properties array / count, and + # ``index_of_`` integers) resolve through the + # same ``source='constituent'`` channel but they + # are NOT individual constituent registrations — + # skip them so they don't get a spurious + # synthesised %instantiate call. + if (std in _FRAMEWORK_CONST_STDS + or std.startswith(_INDEX_PREFIX)): + continue + if std in seen: + continue + seen.add(std) + scheme_var = _lookup_scheme_var( + scheme_store, resolved_call.scheme_name, + resolved_call.phase, arg.scheme_local_name, + ) + if scheme_var is None: + # Should not happen — the resolver only + # produces ResolvedArg from a real MetaVar — + # but fail loudly rather than silently emit + # a malformed %instantiate. + raise CCPPError( + "auto-clone-constituents: cannot locate " + "scheme metadata for '{}' (scheme '{}', " + "phase '{}') while collecting auto-clone " + "entries".format( + arg.scheme_local_name, + resolved_call.scheme_name, + resolved_call.phase, + ) + ) + out.append(_make_auto_clone_entry(scheme_var)) + return out + +# auto-clone-constituents: END legacy-shim helpers. def _collect_scheme_names(group) -> List[str]: diff --git a/capgen-ng/metadata/auto_clone_constituents.py b/capgen-ng/metadata/auto_clone_constituents.py new file mode 100644 index 00000000..919b71ad --- /dev/null +++ b/capgen-ng/metadata/auto_clone_constituents.py @@ -0,0 +1,243 @@ +"""TRANSIENT shim — original-capgen auto-clone-static-constituent path. + +The legacy ccpp-prebuild / original-capgen toolchain auto-registered +every ``is_constituent`` scheme arg by lifting its metadata properties +(``long_name``, ``diagnostic_name``, ``units``, ``default_value``, …) +into a synthetic ``%instantiate(...)`` call emitted into the generated +host cap. Capgen-ng deliberately dropped that path in favour of +explicit registration (``host_constituents`` host arg + register-phase +``ccpp_constituent_properties_t(:)`` scheme args). + +This module re-enables the legacy auto-clone path as an opt-in shim +(``--legacy-auto-clone-constituents`` on the capgen-ng CLI). It exists +for legacy host models — notably CAM-SIMA — that drive original capgen +heavily today and have not yet migrated to explicit registration. + +When enabled, the shim: + +* extends :data:`MetaVar._KNOWN_ATTRS` (in ``metadata/metadata_table.py``) + with four legacy attributes that the strict-mode parser otherwise + rejects: ``default_value``, ``min_value``, ``water_species``, + ``mixing_ratio_type``; +* enables :class:`generator.suite_resolver.SuiteResolution.auto_cloned_constituents` + collection — every ``is_constituent`` consumer whose ``std_name`` has + no register-phase source is recorded with its metadata snapshot; +* drives :func:`generator.suite_cap._register_lines` to emit synthesised + ``%instantiate(...)`` calls into the per-suite dynamic-constituents + buffer, mirroring what a hand-written register-phase scheme would do. + +## Single-instance constraint + +The legacy code paths these models came from never supported multiple +in-memory host instances. The shim follows the same restriction: when +enabled, the host metadata MUST NOT declare both ``instance_number`` +and ``number_of_instances`` (the capgen-ng multi-instance opt-in pair +— see :data:`metadata.registered_dimensions.SCALAR_INDEX_DIMS`). The +gate is enforced in :func:`require_single_instance_host`, called from +:mod:`ccpp_capgen_ng` after host metadata parse. Code paths under the +shim assume ``instance_number`` is the literal ``1``. + +## Self-contained for clean removal + +Every touchpoint in the rest of the codebase is tagged +``# auto-clone-constituents:``. Removing the feature is: + +1. Delete ``metadata/auto_clone_constituents.py`` +2. Delete ``unit-tests/test_auto_clone_constituents.py`` +3. Delete the sample fixture files (search ``auto_clone`` under + ``unit-tests/sample_files``) +4. ``grep -rn 'auto-clone-constituents\\|--legacy-auto-clone-constituents' .`` + and remove every remaining touchpoint (each is a 1-5 line snippet + marked with a ``# auto-clone-constituents:`` comment). + +Every hook is a no-op when the mode is not enabled — the shim has zero +impact on default capgen-ng workflows. + +Examples +-------- +>>> from metadata import auto_clone_constituents +>>> auto_clone_constituents.is_enabled() +False +>>> auto_clone_constituents.extra_known_attrs() +frozenset() + +When enabled, four legacy attrs become recognised: + +>>> import io, logging +>>> logger = logging.getLogger('auto_clone_doctest') +>>> auto_clone_constituents.enable(logger, _stream=io.StringIO()) +>>> auto_clone_constituents.is_enabled() +True +>>> sorted(auto_clone_constituents.extra_known_attrs()) +['default_value', 'min_value', 'mixing_ratio_type', 'water_species'] +>>> auto_clone_constituents.disable() +>>> auto_clone_constituents.is_enabled() +False +""" + +from __future__ import annotations + +import sys +from typing import FrozenSet, Optional, TextIO + + +# ---------------------------------------------------------------------- +# Legacy metadata attributes the shim accepts on scheme args. +# +# Mapped to the matching kwargs of ``ccp_instantiate`` in +# ``src/ccpp_constituent_prop_mod.F90``: +# +# * default_value -> default_value (real, kind_phys) +# * min_value -> min_value (real, kind_phys) +# * water_species -> water_species (logical) +# * mixing_ratio_type -> mixing_ratio_type (character) +# +# The remaining %instantiate kwargs (std_name, long_name, diag_name, +# units, vertical_dim, advected, molar_mass) are already accepted by +# the strict-mode parser, just under canonical capgen-ng names. +# ---------------------------------------------------------------------- +_EXTRA_KNOWN_ATTRS: FrozenSet[str] = frozenset({ + 'default_value', + 'min_value', + 'water_species', + 'mixing_ratio_type', +}) + + +# Process-level on/off flag. Mirrors the model used by +# :mod:`metadata.legacy_compat` and :mod:`metadata.dim_aliases`. +_ENABLED: bool = False + + +def enable(logger=None, _stream: Optional[TextIO] = None) -> None: + """Turn the auto-clone shim on and emit a bold warning banner. + + Idempotent: a second call is a no-op (no double warning). + + Parameters + ---------- + logger : logging.Logger, optional + Logger to emit a ``WARNING``-level companion message on. + _stream : file-like, optional + Override for the banner destination (used by tests to capture + output). ``None`` (default) means ``sys.stderr``. + """ + global _ENABLED + if _ENABLED: + return + _ENABLED = True + + stream = _stream if _stream is not None else sys.stderr + border_width = 70 + border = '*' * border_width + _content_width = border_width - len('*** ') - len(' ***') + + def _pad(s: str) -> str: + """Format *s* as a banner row, left-padded to the border width.""" + return '*** {:<{w}} ***'.format(s[:_content_width], w=_content_width) + + banner_lines = ['', border, + _pad('WARNING: LEGACY AUTO-CLONE-CONSTITUENTS ENABLED'), + _pad('')] + banner_lines += [ + _pad('Every is_constituent scheme arg (advected /'), + _pad('constituent / molar_mass) without a register-phase'), + _pad('source will be auto-registered into the per-suite'), + _pad('dynamic-constituents buffer from its scheme'), + _pad('metadata. Four legacy attributes become accepted'), + _pad('on scheme args:'), + _pad(''), + ] + for name in sorted(_EXTRA_KNOWN_ATTRS): + banner_lines.append(_pad(" '{}'".format(name))) + banner_lines += [ + _pad(''), + _pad('The host metadata MUST NOT declare instance_number'), + _pad('/ number_of_instances (single-instance only).'), + _pad(''), + _pad('This is a TRANSIENT shim for legacy hosts that have'), + _pad('not migrated to explicit registration. It WILL BE'), + _pad('REMOVED in a future capgen-ng release.'), + border, + '', + ] + stream.write('\n'.join(banner_lines) + '\n') + try: + stream.flush() + except Exception: # pylint: disable=broad-except + pass + + if logger is not None: + logger.warning( + "Legacy auto-clone-constituents enabled: is_constituent " + "scheme args without an explicit register-phase source " + "will be auto-registered from their metadata; legacy " + "attributes %s become accepted on scheme args; host must " + "be single-instance. This shim is transient and will be " + "removed.", + ', '.join("'{}'".format(n) for n in sorted(_EXTRA_KNOWN_ATTRS)), + ) + + +def disable() -> None: + """Turn the auto-clone shim off. Intended for tests and library + users that wrap a generator invocation.""" + global _ENABLED + _ENABLED = False + + +def is_enabled() -> bool: + """Return ``True`` iff the auto-clone shim is enabled in this + process.""" + return _ENABLED + + +def extra_known_attrs() -> FrozenSet[str]: + """Return the legacy scheme-arg attribute names that the shim adds + to :data:`MetaVar._KNOWN_ATTRS` when enabled. + + When the shim is disabled, returns an empty frozenset so that the + parser's strict-mode "unknown attribute" rejection fires on these + names just as it would for any other unrecognised key. + """ + if not _ENABLED: + return frozenset() + return _EXTRA_KNOWN_ATTRS + + +def require_single_instance_host(host_dict) -> None: + """Raise :class:`CCPPError` if the host declares the multi-instance + pair (``instance_number`` + ``number_of_instances``) while the shim + is enabled. + + The auto-clone path's emitted code paths assume the literal ``1`` + for every per-instance subscript. Multi-instance support is not in + scope for this transient shim — legacy hosts that need this path + were always single-instance. Called from + :mod:`ccpp_capgen_ng` after the host metadata has been flattened. + + No-op when the shim is disabled. Accepts the resolved + ``host_dict`` flat mapping (or any container with ``__contains__``) + so it can be wired in wherever the host has been parsed. + """ + if not _ENABLED: + return + # Lazy import to keep the shim free of generator dependencies. + from metadata.parse_tools import CCPPError # noqa: E402 + has_inst = 'instance_number' in host_dict + has_ninst = 'number_of_instances' in host_dict + if has_inst or has_ninst: + raise CCPPError( + "--legacy-auto-clone-constituents is single-instance only " + "but the host metadata declares " + "{found}. Either remove the multi-instance pair from the " + "host metadata or drop the --legacy-auto-clone-constituents " + "flag.".format( + found=' and '.join( + name for name, present in ( + ('instance_number', has_inst), + ('number_of_instances', has_ninst), + ) if present + ) + ) + ) diff --git a/capgen-ng/metadata/metadata_table.py b/capgen-ng/metadata/metadata_table.py index 7cfe7a4c..ab6e3285 100644 --- a/capgen-ng/metadata/metadata_table.py +++ b/capgen-ng/metadata/metadata_table.py @@ -78,6 +78,10 @@ # legacy-compat: transient migration shim (delete with the rest of # the legacy_compat touchpoints — grep for ``legacy-compat``). from . import legacy_compat +# auto-clone-constituents: transient shim (delete with the rest of +# the auto-clone-constituents touchpoints — grep for +# ``auto-clone-constituents``). +from . import auto_clone_constituents from .parse_tools import ( CCPPError, ParseContext, @@ -91,6 +95,11 @@ check_fortran_ref, check_fortran_intrinsic, check_molar_mass, + # auto-clone-constituents: legacy-shim checkers. + check_default_value, + check_min_value, + check_water_species, + check_mixing_ratio_type, ) ######################################################################## @@ -488,6 +497,19 @@ class MetaVar: 'top_at_one', }) + # auto-clone-constituents: the legacy auto-clone shim extends the + # accepted set with four ``%instantiate``-kwarg-mapped attrs + # (default_value, min_value, water_species, mixing_ratio_type). + # ``_known_attrs()`` returns the union when the shim is enabled, + # the base set otherwise — strict mode keeps rejecting the legacy + # names with the original "Unknown variable attribute" error. + @classmethod + def _known_attrs(cls): + extra = auto_clone_constituents.extra_known_attrs() + if extra: + return cls._KNOWN_ATTRS | extra + return cls._KNOWN_ATTRS + def __init__(self, local_name: str, context: ParseContext): """Initialise with *local_name* from the ``[ name ]`` section header. @@ -552,6 +574,15 @@ def __init__(self, local_name: str, context: ParseContext): # substitutes the vertical index with `` - k + 1`` on the # host-side access expression. self.top_at_one: bool = False + # auto-clone-constituents: legacy attrs accepted only when the + # shim is enabled. Backing fields use ``None`` as the + # "not set" sentinel so the emitter can distinguish an + # explicit value from a default (optional kwargs on + # %instantiate are only passed when explicitly set). + self.default_value: Optional[float] = None + self.min_value: Optional[float] = None + self.water_species: Optional[bool] = None + self.mixing_ratio_type: Optional[str] = None self.context: ParseContext = context # Track which attributes have been explicitly set (for validation). self._set_attrs: set = set() @@ -574,7 +605,10 @@ def set_attr(self, key: str, value: str, context: ParseContext) -> None: CCPPError On unknown attribute names or invalid values. """ - if key not in self._KNOWN_ATTRS: + # auto-clone-constituents: consult the dynamic union so the + # legacy attrs (default_value/min_value/water_species/ + # mixing_ratio_type) are accepted only when the shim is on. + if key not in self._known_attrs(): raise CCPPError( "Unknown variable attribute '{}' for '{}', at {}".format( key, self.local_name, context @@ -650,6 +684,23 @@ def set_attr(self, key: str, value: str, context: ParseContext) -> None: self.molar_mass = check_molar_mass(value.strip(), None, error=True) elif key == 'top_at_one': self.top_at_one = _parse_bool(value, context) + # auto-clone-constituents: BEGIN legacy-shim attr setters. + # Only reachable when ``_known_attrs()`` returned the + # union, so the strict-mode unknown-attr error fires + # before we get here when the shim is off. + elif key == 'default_value': + self.default_value = check_default_value( + value.strip(), None, error=True) + elif key == 'min_value': + self.min_value = check_min_value( + value.strip(), None, error=True) + elif key == 'water_species': + self.water_species = check_water_species( + value.strip(), None, error=True) + elif key == 'mixing_ratio_type': + self.mixing_ratio_type = check_mixing_ratio_type( + value.strip(), None, error=True) + # auto-clone-constituents: END legacy-shim attr setters. except CCPPError as exc: # Avoid double-wrapping if the inner check already carried # the location (some helpers do; most don't). @@ -1359,6 +1410,21 @@ def flush_table_props() -> None: "appear in host, control, ddt, or suite metadata".format(key), token=key, context=ctx(lineno) ) + # auto-clone-constituents: the four legacy + # instantiate-kwarg attrs are scheme-only when the + # shim is enabled. When the shim is off they get + # rejected one layer down (unknown attribute), so + # the guard only matters in shim-on builds. + if (sec_type != SCHEME_TABLE_TYPE + and auto_clone_constituents.is_enabled() + and key in + auto_clone_constituents.extra_known_attrs()): + raise ParseSyntaxError( + "'{}' is a scheme-only constituent property " + "and cannot appear in host, control, ddt, or " + "suite metadata".format(key), + token=key, context=ctx(lineno) + ) current_var.set_attr(key, val, ctx(lineno)) continue diff --git a/capgen-ng/metadata/parse_tools/__init__.py b/capgen-ng/metadata/parse_tools/__init__.py index 9131cf34..2bb38592 100644 --- a/capgen-ng/metadata/parse_tools/__init__.py +++ b/capgen-ng/metadata/parse_tools/__init__.py @@ -19,6 +19,13 @@ check_fortran_type, check_fortran_intrinsic, check_molar_mass, + # auto-clone-constituents: legacy-shim checkers exported here so + # ``MetaVar.set_attr`` can reach them; gated by the shim's flag at + # the call site, not the import. + check_default_value, + check_min_value, + check_water_species, + check_mixing_ratio_type, FORTRAN_SCALAR_REF_RE, ) from .fortran_conditional import FORTRAN_CONDITIONAL_REGEX diff --git a/capgen-ng/metadata/parse_tools/parse_checkers.py b/capgen-ng/metadata/parse_tools/parse_checkers.py index f0621fa4..63bed4ab 100644 --- a/capgen-ng/metadata/parse_tools/parse_checkers.py +++ b/capgen-ng/metadata/parse_tools/parse_checkers.py @@ -398,6 +398,211 @@ def check_molar_mass(test_val, prop_dict, error): return test_val +# auto-clone-constituents: BEGIN legacy-shim checkers. +# These four checkers validate the metadata attributes that the +# ``--legacy-auto-clone-constituents`` shim accepts on scheme args +# (default_value, min_value, water_species, mixing_ratio_type). +# Delete this whole BEGIN..END block when the shim is removed; the +# rest of the file is unchanged. + +# Whitelist of mixing_ratio_type values accepted by the framework. +# Mirrors the canonical set used by original capgen / ccpp-physics +# hosts. Audit before extending — adding a name silently accepts it +# everywhere. +_MIXING_RATIO_TYPES = frozenset({'dry', 'wet', 'wrt_dry', 'wrt_moist'}) + + +# Anchored Fortran real-literal pattern. Captures the numeric body; +# anything that follows must be either empty or a kind suffix. +# Forms accepted by the body: +# 123 .5 1. 1.5 1.5e-3 1.5E+10 1.5d0 1.5D-12 +_FORTRAN_REAL_BODY_RE = re.compile( + r'^\s*([+-]?(?:\d+\.\d*|\.\d+|\d+)(?:[eEdD][+-]?\d+)?)' +) +# Kind suffix: ``_``. Identifier may itself +# contain underscores (``kind_phys``, ``kind_dyn``). +_FORTRAN_KIND_SUFFIX_RE = re.compile(r'^_[A-Za-z0-9_]+$') + + +def _parse_fortran_real_literal(value): + """Convert a Fortran real-literal string to a Python float. + + Legacy CAM-SIMA metadata writes constituent property values in + Fortran source form, with a kind suffix (``0.0_kind_phys``, + ``1.0e-12_kind_dyn``, ``-3.14_8``) and/or a double-precision + exponent marker (``1.0d-5`` / ``1.0D-5``). Python's :func:`float` + rejects both. This helper isolates the numeric body via an + anchored regex (so identifiers like ``not_a_number`` never match), + verifies any trailing token is a valid kind suffix, rewrites + ``d/D`` exponent markers to ``e/E``, and hands the cleaned body + to :func:`float`. + + Raises :class:`ValueError` (just like :func:`float`) on anything + that doesn't parse, so the calling check_X helper handles the + error path uniformly. + + >>> _parse_fortran_real_literal('0.0') + 0.0 + >>> _parse_fortran_real_literal('0.0_kind_phys') + 0.0 + >>> _parse_fortran_real_literal('1.0e-12_kind_dyn') + 1e-12 + >>> _parse_fortran_real_literal('-3.14_8') + -3.14 + >>> _parse_fortran_real_literal('1.0d-5') + 1e-05 + >>> _parse_fortran_real_literal('1.0D0') + 1.0 + >>> _parse_fortran_real_literal('garbage') #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: ... + >>> _parse_fortran_real_literal('1.0_bad-kind') #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ValueError: ... + """ + s = str(value).strip() + m = _FORTRAN_REAL_BODY_RE.match(s) + if not m: + raise ValueError( + "{!r} does not start with a Fortran real literal".format(value) + ) + body = m.group(1) + rest = s[m.end():] + if rest and not _FORTRAN_KIND_SUFFIX_RE.match(rest): + raise ValueError( + "{!r} has trailing junk after the numeric body".format(value) + ) + # Rewrite double-precision exponent markers (d/D) -> (e/E). + body = body.replace('d', 'e').replace('D', 'E') + return float(body) + + +def check_default_value(test_val, prop_dict, error): + """Return as a float if a valid default_value, otherwise None. + + Accepts any finite Fortran real literal: no positivity constraint + (some species use a negative sentinel as their "uninitialized" + placeholder, see CAM-SIMA cld_liq.F90). The Fortran kind suffix + (e.g. ``0.0_kind_phys``) is stripped before conversion. + + >>> check_default_value('0.0', None, True) + 0.0 + >>> check_default_value('0.0_kind_phys', None, True) + 0.0 + >>> check_default_value('-1.0e30', None, True) + -1e+30 + >>> check_default_value('1.0d-5_kind_phys', None, True) + 1e-05 + >>> check_default_value('not a number', None, False) + + >>> check_default_value('not a number', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'not a number' is not a valid default_value + """ + try: + return _parse_fortran_real_literal(test_val) + except (TypeError, ValueError): + if error: + raise CCPPError("'{}' is not a valid default_value".format(test_val)) + return None + + +def check_min_value(test_val, prop_dict, error): + """Return as a float if a valid min_value, otherwise None. + + Same shape as :func:`check_default_value`; no constraint beyond + "finite float" since min_value is a host-runtime guardrail and the + sensible range is species-dependent. Fortran kind suffix + accepted. + + >>> check_min_value('0.0', None, True) + 0.0 + >>> check_min_value('1.0e-12', None, True) + 1e-12 + >>> check_min_value('1.0e-12_kind_phys', None, True) + 1e-12 + >>> check_min_value('garbage', None, False) + + >>> check_min_value('garbage', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'garbage' is not a valid min_value + """ + try: + return _parse_fortran_real_literal(test_val) + except (TypeError, ValueError): + if error: + raise CCPPError("'{}' is not a valid min_value".format(test_val)) + return None + + +def check_water_species(test_val, prop_dict, error): + """Return ``True`` / ``False`` for a valid water_species value, + otherwise None. + + Accepts the same surface as the existing ``advected`` / + ``constituent`` flag parsing (``True``/``False``/``T``/``F``, + case-insensitive); rejects anything else. + + >>> check_water_species('True', None, True) + True + >>> check_water_species('false', None, True) + False + >>> check_water_species('maybe', None, False) + + >>> check_water_species('maybe', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'maybe' is not a valid water_species (expected True or False) + """ + if isinstance(test_val, bool): + return test_val + tok = str(test_val).strip().lower() + if tok in ('true', 't', '.true.'): + return True + if tok in ('false', 'f', '.false.'): + return False + if error: + raise CCPPError( + "'{}' is not a valid water_species " + "(expected True or False)".format(test_val) + ) + return None + + +def check_mixing_ratio_type(test_val, prop_dict, error): + """Return if a valid mixing_ratio_type, otherwise None. + + Compared case-insensitively against the canonical whitelist + (``_MIXING_RATIO_TYPES``). Returns the lowercased form on success. + + >>> check_mixing_ratio_type('dry', None, True) + 'dry' + >>> check_mixing_ratio_type('WRT_MOIST', None, True) + 'wrt_moist' + >>> check_mixing_ratio_type('bogus', None, False) + + >>> check_mixing_ratio_type('bogus', None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'bogus' is not a valid mixing_ratio_type + """ + if test_val is None: + if error: + raise CCPPError("None is not a valid mixing_ratio_type") + return None + tok = str(test_val).strip().lower() + if tok in _MIXING_RATIO_TYPES: + return tok + if error: + raise CCPPError( + "'{}' is not a valid mixing_ratio_type " + "(expected one of: {})".format( + test_val, ', '.join(sorted(_MIXING_RATIO_TYPES)) + ) + ) + return None + +# auto-clone-constituents: END legacy-shim checkers. + + def check_balanced_paren(string, start=0, error=False): """Return indices delineating a balance set of parentheses. Parentheses in character context do not count. diff --git a/doc/auto_clone_constituents.md b/doc/auto_clone_constituents.md new file mode 100644 index 00000000..0f731e24 --- /dev/null +++ b/doc/auto_clone_constituents.md @@ -0,0 +1,150 @@ +# `--legacy-auto-clone-constituents` — transient shim + +A capgen-ng CLI flag that re-enables the **auto-clone-static-constituent** +registration path the original ccpp-capgen toolchain provided to +CAM-SIMA. Off by default; turned on with a single flag and a loud +startup banner. Intended as a migration aid — every line of code it +touches is tagged so the whole feature can be removed cleanly when +legacy hosts have moved on. + +## How to enable it + +Pass the flag to both `capgen-ng` and `ccpp_validator` (the +ccpp-physics build system already does this for +`end-to-end-tests/advection_auto_clone/`): + +``` +ccpp_capgen_ng.py --legacy-auto-clone-constituents ... +ccpp_validator.py --legacy-auto-clone-constituents ... +``` + +It is **single-instance only**. If the host metadata declares the +`instance_number` + `number_of_instances` pair (capgen-ng's +multi-instance opt-in), the run aborts with a clear error before +parsing any suite. Legacy hosts predate multi-instance support, so +this restriction matches the use case. + +## What the flag does + +Same shape as original capgen's auto-clone: + +For every scheme argument flagged `advected = True`, `constituent = True`, +or `molar_mass = `, capgen-ng synthesises a `%instantiate(...)` +call into the generated host code, lifting field values straight from +the scheme metadata. The constituent ends up registered in the +per-suite dynamic-constituents buffer alongside any constituents the +host or an explicit register-phase scheme registered. The scheme +author writes no Fortran registration code. + +These metadata attributes are accepted on scheme arguments when the +flag is on (and rejected when it is off): + +| Attribute | Type | Passed to `%instantiate` as | +|----------------------|---------------------|-----------------------------| +| `default_value` | real (kind_phys) | `default_value` | +| `min_value` | real (kind_phys) | `min_value` | +| `water_species` | logical | `water_species` | +| `mixing_ratio_type` | character | `mixing_ratio_type` | + +Fortran-style literals (`0.0_kind_phys`, `1.0d-5`, `-3.14_8`, +`1.0d-5_kind_phys`) are accepted for the real-valued attrs, since +legacy metadata writes the values in source form. + +The other `%instantiate` kwargs (`std_name`, `long_name`, +`diag_name`, `units`, `vertical_dim`, `advected`, `molar_mass`) +already had accepted spellings in capgen-ng; the shim just wires them +into the synthesised call. + +## Defaults that match original capgen + +- **`long_name` is synthesised when missing.** If the scheme metadata + has no `long_name` on a constituent arg, capgen-ng builds one from + the standard name by replacing underscores with spaces and + capitalising the first character. + Example: `cloud_liquid_dry_mixing_ratio` → + `'Cloud liquid dry mixing ratio'`. +- **`diag_name` falls back to the metadata local name** when neither + `diagnostic_name` nor `diagnostic_name_fixed` is set. +- **`vertical_dim` is lifted from the scheme arg's `dimensions = (..., )`** — + whichever entry matches `vertical_layer_dimension` or + `vertical_interface_dimension`; otherwise + `vertical_layer_dimension`. + +## What's stricter than original capgen + +Capgen-ng's general rules apply even with the flag on. Two of them +trip up legacy fixtures: + +1. **Metadata args must match the Fortran subroutine signature.** + Original capgen tolerated a metadata arg-table that listed a + constituent in `_init` even when the Fortran `_init` + didn't accept it as a dummy. Capgen-ng passes the metadata args + at the call site as Fortran keyword arguments, and the validator + catches divergence. Either include the constituent as a Fortran + dummy, or remove it from the init's arg-table. +2. **Base constituents can't be `intent = out`.** A scheme arg with + `advected = True` (or `constituent = True` / `molar_mass = ...`) + on a non-`tendency_of_*` standard name has to be `intent = in` or + `intent = inout`. Only tendency args (std_name starts with + `tendency_of_`) can be `intent = out`. If the legacy scheme + wrote to the array in its init routine, change both the metadata + and the Fortran subroutine to `intent = inout` — the body is + unchanged. + +Multi-instance support is also off-limits while the flag is on (see +above). + +## Used in production CAM-SIMA + +Most CAM-SIMA atmospheric physics schemes rely on the auto-clone +path the same way original capgen ships it: a handful of schemes +register constituents explicitly (`rrtmgp_constituents`, +`musica_ccpp`, `prescribed_aerosols`, `prescribed_ozone`), and the +~16 others (`kessler`, `zm_convr`, `dadadj`, `holtslag_boville_diff`, +`state_converters`, `geopotential_temp`, `cloud_particle_sedimentation`, +…) declare `advected = True intent = inout` on their `_run` +arguments and let the framework register the constituent. Without +the flag, capgen-ng's runtime check fires for every consumer +because no source actually registered the species. The flag closes +that gap by re-creating the auto-clone behaviour from the metadata. + +Production CAM-SIMA does not use `default_value`, `min_value`, +`water_species`, or `mixing_ratio_type` in metadata; the four +extra parser attributes exist to support the advection test (and +any future legacy host that needs the broader kwarg surface on +`%instantiate`). + +## `end-to-end-tests/advection_auto_clone/` — what changed + +The fixture is a port of CAM-SIMA's `ccpp_framework/test/advection_test`. +It exercises the full legacy attr surface (`default_value`, +`diagnostic_name`, `advected`) and the unusual init-phase +`intent = out`-on-base-constituent pattern. Three small edits were +needed to make the port build under capgen-ng: + +1. **`cld_liq.meta`** — in `cld_liq_init`'s `[ cld_liq_array ]` + block, change `intent = out` to `intent = inout`. +2. **`cld_liq.F90`** — in `cld_liq_init`, change + `real(kind=kind_phys), intent(out) :: cld_liq_array(:, :)` to + `intent(inout)`. The body still does + `cld_liq_array = 0.0_kind_phys`; inout is a strict superset. +3. **`cld_ice.meta`** — delete the `[ cld_ice_array ]` block inside + `cld_ice_init`. Fortran `cld_ice_init` only takes + `(tfreeze, errmsg, errflg)`; the run phase already triggers + auto-clone registration of `cloud_ice_dry_mixing_ratio`. + +No `long_name` additions to the metadata were necessary — capgen-ng +synthesises the long_name from the standard name automatically +(see the defaults section above). + +The CTest target `test_advection_auto_clone` passes after these edits. + +## When to retire the flag + +When all consumers have been moved to capgen-ng's explicit +registration model — either by declaring constituents in the host's +`host_constituents(:)` array, or by writing a register-phase scheme +with a `ccpp_constituent_properties_t(:), intent=out` argument that +calls `%instantiate` directly. At that point the flag is no longer +needed by any host, and the shim can be removed in one cleanup +pass. diff --git a/end-to-end-tests/CMakeLists.txt b/end-to-end-tests/CMakeLists.txt index 84f4b984..76109f2d 100644 --- a/end-to-end-tests/CMakeLists.txt +++ b/end-to-end-tests/CMakeLists.txt @@ -81,3 +81,6 @@ add_subdirectory(capgen_ng) add_subdirectory(var_compat) add_subdirectory(advection) add_subdirectory(instances_advection) +# auto-clone-constituents: Remove this test when the legacy switch to support +# auto-clone constituents is removed (i.e. after CAM-SIMA updated its physics) +add_subdirectory(advection_auto_clone) diff --git a/end-to-end-tests/advection_auto_clone/CMakeLists.txt b/end-to-end-tests/advection_auto_clone/CMakeLists.txt new file mode 100644 index 00000000..c04bd7cc --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/CMakeLists.txt @@ -0,0 +1,93 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "cld_liq" "cld_ice" "apply_constituent_tendencies" "const_indices") +set(HOST_FILES "test_host_data" "test_host_mod" "test_host") +set(SUITE_FILES "cld_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Enable legacy flags for the capgen call +set(EXTRA_FLAGS + "--legacy-mode" + "--legacy-auto-clone-constituents" +) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + EXTRA_FLAGS ${EXTRA_FLAGS}) +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + EXTRA_FLAGS ${EXTRA_FLAGS}) + +# Enable trace output in auto-generated caps +set(CCPP_TRACE ON) + +# Enable legacy flags for the capgen call +set(EXTRA_FLAGS + "--legacy-mode" + "--legacy-auto-clone-constituents" +) + +# Run ccpp_capgen +ccpp_capgen(TRACE ${CCPP_TRACE} + VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + EXTRA_FLAGS ${EXTRA_FLAGS} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +# Add extra files needed for testing +set(EXTRA_FILES + ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 +) + +add_executable(test_advection_auto_clone.x + ${EXTRA_FILES} + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} + test_advection_host_integration.F90 +) +target_link_libraries(test_advection_auto_clone.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_advection_auto_clone.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_advection_auto_clone.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_advection_auto_clone + COMMAND test_advection_auto_clone.x) diff --git a/end-to-end-tests/advection_auto_clone/README.md b/end-to-end-tests/advection_auto_clone/README.md new file mode 100644 index 00000000..616b9036 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/README.md @@ -0,0 +1,10 @@ +# Advection Test + +Contains tests to exercise the capabilities of the constituents object, including: +- Adding build-time constituents via metadata property +- Adding run-time constituents from schemes via a register phase + - Also tests that trying to add a constituent outside of the register phase errors as expected +- Passing around and modifying the constituent array +- Accessing and modifying a constituent tendency variable +- Passing around the constituent tendency array +- Dimensions are case-insensitive diff --git a/end-to-end-tests/advection_auto_clone/advection_test_reports.py b/end-to-end-tests/advection_auto_clone/advection_test_reports.py new file mode 100644 index 00000000..4fbe8e68 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/advection_test_reports.py @@ -0,0 +1,127 @@ +#! /usr/bin/env python3 +""" +----------------------------------------------------------------------- + Description: Test advection database report python interface + + Assumptions: + + Command line arguments: build_dir database_filepath + + Usage: python test_reports +----------------------------------------------------------------------- +""" +import os +import unittest + +from test_stub import BaseTests + +_BUILD_DIR = os.path.join(os.path.abspath(os.environ['BUILD_DIR']), "test", "advection_test") +_DATABASE = os.path.abspath(os.path.join(_BUILD_DIR, "ccpp", "datatable.xml")) + +_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +_FRAMEWORK_DIR = os.path.abspath(os.path.join(_TEST_DIR, os.pardir, os.pardir)) +_SCRIPTS_DIR = os.path.abspath(os.path.join(_FRAMEWORK_DIR, "scripts")) + +# Check data +_HOST_FILES = [os.path.join(_BUILD_DIR, "ccpp", "test_host_ccpp_cap.F90")] +_SUITE_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_cld_suite_cap.F90")] +_UTILITY_FILES = [os.path.join(_BUILD_DIR, "ccpp", "ccpp_kinds.F90"), + os.path.join(_FRAMEWORK_DIR, "src", + "ccpp_constituent_prop_mod.F90"), + os.path.join(_FRAMEWORK_DIR, "src", + "ccpp_scheme_utils.F90"), + os.path.join(_FRAMEWORK_DIR, "src", "ccpp_hashable.F90"), + os.path.join(_FRAMEWORK_DIR, "src", "ccpp_hash_table.F90")] +_CCPP_FILES = _UTILITY_FILES + _HOST_FILES + _SUITE_FILES +_DEPENDENCIES = [""] +_PROCESS_LIST = [""] +_MODULE_LIST = ["cld_ice", "cld_liq", "const_indices", "apply_constituent_tendencies"] +_SUITE_LIST = ["cld_suite"] +_REQUIRED_VARS_CLD = ["ccpp_error_code", "ccpp_error_message", + "horizontal_loop_begin", "horizontal_loop_end", + "surface_air_pressure", "temperature", + "tendency_of_cloud_liquid_dry_mixing_ratio", + "time_step_for_physics", "water_temperature_at_freezing", + "water_vapor_specific_humidity", + "cloud_ice_dry_mixing_ratio", + "cloud_liquid_dry_mixing_ratio", + "ccpp_constituents", + "ccpp_constituent_tendencies", + "number_of_ccpp_constituents", + "dynamic_constituents_for_cld_ice", + "dynamic_constituents_for_cld_liq", + "test_banana_constituent_indices", "test_banana_name", + "banana_array_dim", + "test_banana_name_array", + "test_banana_constituent_index", + # Added by --debug option + "horizontal_dimension", + "vertical_layer_dimension"] +_INPUT_VARS_CLD = ["surface_air_pressure", "temperature", + "horizontal_loop_begin", "horizontal_loop_end", + "time_step_for_physics", "water_temperature_at_freezing", + "water_vapor_specific_humidity", + "cloud_ice_dry_mixing_ratio", + "cloud_liquid_dry_mixing_ratio", + "tendency_of_cloud_liquid_dry_mixing_ratio", + "ccpp_constituents", + "ccpp_constituent_tendencies", + "number_of_ccpp_constituents", + "banana_array_dim", + "test_banana_name_array", "test_banana_name", + # Added by --debug option + "horizontal_dimension", + "vertical_layer_dimension"] +_OUTPUT_VARS_CLD = ["ccpp_error_code", "ccpp_error_message", + "water_vapor_specific_humidity", "temperature", + "tendency_of_cloud_liquid_dry_mixing_ratio", + "cloud_ice_dry_mixing_ratio", + "ccpp_constituents", + "ccpp_constituent_tendencies", + "cloud_liquid_dry_mixing_ratio", + "dynamic_constituents_for_cld_ice", + "dynamic_constituents_for_cld_liq", + "dynamic_constituents_for_cld_liq", + "test_banana_constituent_indices", + "test_banana_constituent_index"] + + +class TestAdvectionHostDataTables(unittest.TestCase, BaseTests.TestHostDataTables): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + +class CommandLineAdvectionHostDatafileRequiredFiles(unittest.TestCase, BaseTests.TestHostCommandLineDataFiles): + database = _DATABASE + host_files = _HOST_FILES + suite_files = _SUITE_FILES + utility_files = _UTILITY_FILES + ccpp_files = _CCPP_FILES + process_list = _PROCESS_LIST + module_list = _MODULE_LIST + dependencies = _DEPENDENCIES + suite_list = _SUITE_LIST + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" + + +class TestCapgenCldSuite(unittest.TestCase, BaseTests.TestSuite): + database = _DATABASE + required_vars = _REQUIRED_VARS_CLD + input_vars = _INPUT_VARS_CLD + output_vars = _OUTPUT_VARS_CLD + suite_name = "cld_suite" + + +class CommandLineCapgenDdtSuite(unittest.TestCase, BaseTests.TestSuiteCommandLine): + database = _DATABASE + required_vars = _REQUIRED_VARS_CLD + input_vars = _INPUT_VARS_CLD + output_vars = _OUTPUT_VARS_CLD + suite_name = "cld_suite" + datafile_script = f"{_SCRIPTS_DIR}/ccpp_datafile.py" diff --git a/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.F90 b/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.F90 new file mode 100644 index 00000000..63a1881c --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.F90 @@ -0,0 +1,39 @@ +module apply_constituent_tendencies + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: apply_constituent_tendencies_run + +contains + + !> \section arg_table_apply_constituent_tendencies_run Argument Table + !!! \htmlinclude apply_constituent_tendencies_run.html + subroutine apply_constituent_tendencies_run(const_tend, const, errcode, errmsg) + ! Dummy arguments + real(kind=kind_phys), intent(inout) :: const_tend(:, :, :) ! constituent tendency array + real(kind=kind_phys), intent(inout) :: const(:, :, :) ! constituent state array + integer, intent(out) :: errcode + character(len=512), intent(out) :: errmsg + + ! Local variables + integer :: klev, jcnst, icol + + errcode = 0 + errmsg = '' + + do icol = 1, size(const_tend, 1) + do klev = 1, size(const_tend, 2) + do jcnst = 1, size(const_tend, 3) + const(icol, klev, jcnst) = const(icol, klev, jcnst) + const_tend(icol, klev, jcnst) + end do + end do + end do + + const_tend = 0._kind_phys + + end subroutine apply_constituent_tendencies_run + +end module apply_constituent_tendencies diff --git a/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.meta b/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.meta new file mode 100644 index 00000000..ac02e5e4 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/apply_constituent_tendencies.meta @@ -0,0 +1,36 @@ +##################################################################### +[ccpp-table-properties] + name = apply_constituent_tendencies + type = scheme +[ccpp-arg-table] + name = apply_constituent_tendencies_run + type = scheme +[ const_tend ] + standard_name = ccpp_constituent_tendencies + long_name = ccpp constituent tendencies + units = none + type = real | kind = kind_phys + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + intent = inout +[ const ] + standard_name = ccpp_constituents + long_name = ccpp constituents + units = none + type = real | kind = kind_phys + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_ccpp_constituents) + intent = inout +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + type = integer + dimensions = () + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + type = character | kind = len=512 + dimensions = () + intent = out +######################################################### diff --git a/end-to-end-tests/advection_auto_clone/cld_ice.F90 b/end-to-end-tests/advection_auto_clone/cld_ice.F90 new file mode 100644 index 00000000..bf19b979 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/cld_ice.F90 @@ -0,0 +1,125 @@ +! Test parameterization with advected species +! + +module cld_ice + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: cld_ice_register + public :: cld_ice_init + public :: cld_ice_run + public :: cld_ice_final + + real(kind=kind_phys), private :: tcld = huge(1.0_kind_phys) + +contains + + !> \section arg_table_cld_ice_register Argument Table + !! \htmlinclude arg_table_cld_ice_register.html + !! + subroutine cld_ice_register(dyn_const_ice, errmsg, errcode) + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const_ice(:) + integer, intent(out) :: errcode + character(len=512), intent(out) :: errmsg + + errmsg = '' + errcode = 0 + allocate(dyn_const_ice(2), stat=errcode) + if (errcode /= 0) then + errmsg = 'Error allocating dyn_const in cld_ice_dynamic_constituents' + return + end if + call dyn_const_ice(1)%instantiate(std_name='dyn_const1', long_name='dyn const1', & + diag_name='DYNCONST1', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + min_value=1000._kind_phys, water_species=.true., mixing_ratio_type='wet', & + errcode=errcode, errmsg=errmsg) + call dyn_const_ice(2)%instantiate(std_name='dyn_const2_wrt_moist_air', long_name='dyn const2', & + diag_name='DYNCONST2', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + water_species=.false., errcode=errcode, errmsg=errmsg) + + end subroutine cld_ice_register + + !> \section arg_table_cld_ice_run Argument Table + !! \htmlinclude arg_table_cld_ice_run.html + !! + subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & + errmsg, errflg) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(inout) :: temp(:, :) + real(kind=kind_phys), intent(inout) :: qv(:, :) + real(kind=kind_phys), intent(in) :: ps(:) + real(kind=kind_phys), intent(inout) :: cld_ice_array(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: icol + integer :: ilev + real(kind=kind_phys) :: frz + + errmsg = '' + errflg = 0 + + ! Apply state-of-the-art thermodynamics :) + do icol = 1, ncol + do ilev = 1, size(temp, 2) + if (temp(icol, ilev) < tcld) then + frz = max(qv(icol, ilev) - 0.5_kind_phys, 0.0_kind_phys) + cld_ice_array(icol, ilev) = cld_ice_array(icol, ilev) + frz + qv(icol, ilev) = qv(icol, ilev) - frz + if (frz > 0.0_kind_phys) then + temp(icol, ilev) = temp(icol, ilev) + 1.0_kind_phys + end if + end if + end do + end do + + end subroutine cld_ice_run + + !> \section arg_table_cld_ice_init Argument Table + !! \htmlinclude arg_table_cld_ice_init.html + !! + subroutine cld_ice_init(tfreeze, errmsg, errflg) + + real(kind=kind_phys), intent(in) :: tfreeze + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + tcld = tfreeze - 20.0_kind_phys + + end subroutine cld_ice_init + + !> \section arg_table_cld_ice_final Argument Table + !! \htmlinclude arg_table_cld_ice_final.html + !! + + !> @{ + !! This routine does nothing, but it tests if blank + !! lines and doxygen comments between metadata hooks + !! and the subroutine are parsed correctly. + !! @{ + + subroutine cld_ice_final(errmsg, errflg) + + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + end subroutine cld_ice_final + + !! @} + !! @} + +end module cld_ice diff --git a/end-to-end-tests/advection_auto_clone/cld_ice.meta b/end-to-end-tests/advection_auto_clone/cld_ice.meta new file mode 100644 index 00000000..df0a925d --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/cld_ice.meta @@ -0,0 +1,137 @@ +# cld_ice is a scheme that produces a cloud ice amount +[ccpp-table-properties] + name = cld_ice + type = scheme + +[ccpp-arg-table] + name = cld_ice_register + type = scheme +[ dyn_const_ice ] + standard_name = dynamic_constituents_for_cld_ice + units = none + dimensions = (:) + allocatable = True + type = ccpp_constituent_properties_t + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errcode ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_ice_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ temp ] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ qv ] + standard_name = water_vapor_specific_humidity + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) + intent = in +[ cld_ice_array ] + standard_name = cloud_ice_dry_mixing_ratio + advected = .true. + default_value = 0.0_kind_phys + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_ice_init + type = scheme +[ tfreeze ] + standard_name = water_temperature_at_freezing + long_name = Freezing temperature of water at sea level + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_ice_final + type = scheme +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/advection_auto_clone/cld_liq.F90 b/end-to-end-tests/advection_auto_clone/cld_liq.F90 new file mode 100644 index 00000000..0eaffbdb --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/cld_liq.F90 @@ -0,0 +1,102 @@ +! Test parameterization with advected species +! + +module cld_liq + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + private + + public :: cld_liq_register + public :: cld_liq_init + public :: cld_liq_run + +contains + + !> \section arg_table_cld_liq_register Argument Table + !! \htmlinclude arg_table_cld_liq_register.html + !! + subroutine cld_liq_register(dyn_const, errmsg, errflg) + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + allocate(dyn_const(1), stat=errflg) + if (errflg /= 0) then + errmsg = 'Error allocating dyn_const in cld_liq_register' + return + end if + call dyn_const(1)%instantiate(std_name="dyn_const3_wrt_moist_air_and_condensed_water", long_name='dyn const3', & + diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + water_species=.true., mixing_ratio_type='dry', & + errcode=errflg, errmsg=errmsg) + + end subroutine cld_liq_register + + !> \section arg_table_cld_liq_run Argument Table + !! \htmlinclude arg_table_cld_liq_run.html + !! + subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & + cld_liq_tend, errmsg, errflg) + + integer, intent(in) :: ncol + real(kind=kind_phys), intent(in) :: timestep + real(kind=kind_phys), intent(in) :: tcld + real(kind=kind_phys), intent(inout) :: temp(:, :) + real(kind=kind_phys), intent(inout) :: qv(:, :) + real(kind=kind_phys), intent(in) :: ps(:) + real(kind=kind_phys), intent(out) :: cld_liq_tend(:, :) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: icol + integer :: ilev + real(kind=kind_phys) :: cond + + errmsg = '' + errflg = 0 + + ! Apply state-of-the-art thermodynamics :) + do icol = 1, ncol + do ilev = 1, size(temp, 2) + if ((qv(icol, ilev) > 0.0_kind_phys) .and. & + (temp(icol, ilev) <= tcld)) then + cond = min(qv(icol, ilev), 0.1_kind_phys) + cld_liq_tend(icol, ilev) = cond + qv(icol, ilev) = qv(icol, ilev) - cond + if (cond > 0.0_kind_phys) then + temp(icol, ilev) = temp(icol, ilev) + (cond * 5.0_kind_phys) + end if + end if + end do + end do + + end subroutine cld_liq_run + + !> \section arg_table_cld_liq_init Argument Table + !! \htmlinclude arg_table_cld_liq_init.html + !! + subroutine cld_liq_init(tfreeze, cld_liq_array, tcld, errmsg, errflg) + + real(kind=kind_phys), intent(in) :: tfreeze + real(kind=kind_phys), intent(inout) :: cld_liq_array(:, :) + real(kind=kind_phys), intent(out) :: tcld + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! This routine currently does nothing + + errmsg = '' + errflg = 0 + cld_liq_array = 0.0_kind_phys + tcld = tfreeze - 20.0_kind_phys + + end subroutine cld_liq_init + +end module cld_liq diff --git a/end-to-end-tests/advection_auto_clone/cld_liq.meta b/end-to-end-tests/advection_auto_clone/cld_liq.meta new file mode 100644 index 00000000..0b73cfa0 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/cld_liq.meta @@ -0,0 +1,137 @@ +# cld_liq is a scheme that produces a cloud liquid amount +[ccpp-table-properties] + name = cld_liq + type = scheme +[ccpp-arg-table] + name = cld_liq_register + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_cld_liq + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out + allocatable = true +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_liq_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + type = integer + units = count + dimensions = () + intent = in +[ timestep ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real + kind = kind_phys + intent = in +[ tcld] + standard_name = minimum_temperature_for_cloud_liquid + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ temp ] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_LAYER_dimension) + type = real + kind = kind_phys + intent = inout +[ qv ] + standard_name = water_vapor_specific_humidity + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = hPa + dimensions = (horizontal_dimension) + intent = in +[ cld_liq_tend ] + standard_name = tendency_of_cloud_liquid_dry_mixing_ratio + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out + constituent = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = cld_liq_init + type = scheme +[ tfreeze] + standard_name = water_temperature_at_freezing + long_name = Freezing temperature of water at sea level + units = K + dimensions = () + type = real | kind = kind_phys + intent = in +[ cld_liq_array ] + standard_name = cloud_liquid_dry_mixing_ratio + diagnostic_name = CLDLIQ + advected = .true. + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + # Advected species that needs to be promoted from suite. + # Note that in capgen-ng, intent 'out' is no longer permitted + intent = inout +[ tcld] + standard_name = minimum_temperature_for_cloud_liquid + units = K + dimensions = () + type = real | kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/advection_auto_clone/cld_suite.xml b/end-to-end-tests/advection_auto_clone/cld_suite.xml new file mode 100644 index 00000000..fac613e8 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/cld_suite.xml @@ -0,0 +1,11 @@ + + + + + const_indices + cld_liq + apply_constituent_tendencies + cld_ice + apply_constituent_tendencies + + diff --git a/end-to-end-tests/advection_auto_clone/cld_suite_error.xml b/end-to-end-tests/advection_auto_clone/cld_suite_error.xml new file mode 100644 index 00000000..80acac91 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/cld_suite_error.xml @@ -0,0 +1,9 @@ + + + + + dlc_liq + cld_liq + cld_ice + + diff --git a/end-to-end-tests/advection_auto_clone/const_indices.F90 b/end-to-end-tests/advection_auto_clone/const_indices.F90 new file mode 100644 index 00000000..bc3b46a7 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/const_indices.F90 @@ -0,0 +1,95 @@ +! Test collection of constituent indices +! + +module const_indices + + use ccpp_kinds, only: kind_phys + + implicit none + private + + public :: const_indices_init + public :: const_indices_run + +contains + + !> \section arg_table_const_indices_run Argument Table + !! \htmlinclude arg_table_const_indices_run.html + !! + subroutine const_indices_run(const_std_name, num_consts, test_stdname_array, & + const_index, const_inds, errmsg, errflg) + use ccpp_constituent_prop_mod, only: int_unassigned + use ccpp_scheme_utils, only: ccpp_constituent_index + use ccpp_scheme_utils, only: ccpp_constituent_indices + + character(len=*), intent(in) :: const_std_name + integer, intent(in) :: num_consts + character(len=*), intent(in) :: test_stdname_array(:) + integer, intent(out) :: const_index + integer, intent(out) :: const_inds(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: indx + integer :: test_indx + + errmsg = '' + errflg = 0 + + ! Find the constituent index for + call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) + if (errflg == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) + end if + ! Check that a non-registered constituent is detectable but + ! does not cause an error + if (errflg == 0) then + call ccpp_constituent_index('unobtainium', test_indx, errflg, errmsg) + if (test_indx /= int_unassigned) then + if (errflg == 0) then + ! Do not add an error if one is already reported + errflg = 2 + write(errmsg, '(2a,i0,a,i0)') "ccpp_constituent_index called for ", & + "'unobtainium' returned an index of ", test_indx, ", not ", & + int_unassigned + end if + end if + end if + + end subroutine const_indices_run + + !> \section arg_table_const_indices_init Argument Table + !! \htmlinclude arg_table_const_indices_init.html + !! + subroutine const_indices_init(const_std_name, num_consts, test_stdname_array, & + const_index, const_inds, errmsg, errflg) + use ccpp_scheme_utils, only: ccpp_constituent_index, & + ccpp_constituent_indices + + character(len=*), intent(in) :: const_std_name + integer, intent(in) :: num_consts + character(len=*), intent(in) :: test_stdname_array(:) + integer, intent(out) :: const_index + integer, intent(out) :: const_inds(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + !---------------------------------------------------------------- + + integer :: indx + + errmsg = '' + errflg = 0 + + ! Find the constituent index for + call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) + if (errflg == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) + end if + + end subroutine const_indices_init + + !! @} + !! @} + +end module const_indices diff --git a/end-to-end-tests/advection_auto_clone/const_indices.meta b/end-to-end-tests/advection_auto_clone/const_indices.meta new file mode 100644 index 00000000..a4cc98e2 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/const_indices.meta @@ -0,0 +1,108 @@ +# const_indices just returns some constituent indices as a test +[ccpp-table-properties] + name = const_indices + type = scheme +[ccpp-arg-table] + name = const_indices_run + type = scheme +[ const_std_name ] + standard_name = test_banana_name + type = character | kind = len=* + units = 1 + dimensions = () + protected = true + intent = in +[ num_consts ] + standard_name = banana_array_dim + long_name = Size of test_banana_name_array + units = 1 + dimensions = () + type = integer + intent = in +[ test_stdname_array ] + standard_name = test_banana_name_array + type = character | kind = len=* + units = count + dimensions = (banana_array_dim) + intent = in +[ const_index ] + standard_name = test_banana_constituent_index + long_name = Constituent index + units = 1 + dimensions = () + type = integer + intent = out +[ const_inds ] + standard_name = test_banana_constituent_indices + long_name = Array of constituent indices + units = 1 + dimensions = (banana_array_dim) + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out +[ccpp-arg-table] + name = const_indices_init + type = scheme +[ const_std_name ] + standard_name = test_banana_name + type = character | kind = len=* + units = 1 + dimensions = () + protected = true + intent = in +[ num_consts ] + standard_name = banana_array_dim + long_name = Size of test_banana_name_array + units = 1 + dimensions = () + type = integer + intent = in +[ test_stdname_array ] + standard_name = test_banana_name_array + type = character | kind = len=* + units = count + dimensions = (banana_array_dim) + intent = in +[ const_index ] + standard_name = test_banana_constituent_index + long_name = Constituent index + units = 1 + dimensions = () + type = integer + intent = out +[ const_inds ] + standard_name = test_banana_constituent_indices + long_name = Array of constituent indices + units = 1 + dimensions = (banana_array_dim) + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/advection_auto_clone/dlc_liq.F90 b/end-to-end-tests/advection_auto_clone/dlc_liq.F90 new file mode 100644 index 00000000..20ff4b7b --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/dlc_liq.F90 @@ -0,0 +1,41 @@ +! Test parameterization with a runtime constituents +! properties object outside of the register phase + +module dlc_liq + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + private + + public :: dlc_liq_init + +contains + + !> \section arg_table_dlc_liq_init Argument Table + !! \htmlinclude arg_table_dlc_liq_init.html + !! + subroutine dlc_liq_init(dyn_const, errmsg, errflg) + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errflg + + character(len=256) :: stdname + + errmsg = '' + errflg = 0 + allocate(dyn_const(1), stat=errflg) + if (errflg /= 0) then + errmsg = 'Error allocating dyn_const in dlc_liq_init' + return + end if + call dyn_const(1)%instantiate(std_name="dyn_const3", long_name='dyn const3', & + diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + errcode=errflg, errmsg=errmsg) + call dyn_const(1)%standard_name(stdname, errcode=errflg, errmsg=errmsg) + + end subroutine dlc_liq_init + +end module dlc_liq diff --git a/end-to-end-tests/advection_auto_clone/dlc_liq.meta b/end-to-end-tests/advection_auto_clone/dlc_liq.meta new file mode 100644 index 00000000..fedb6243 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/dlc_liq.meta @@ -0,0 +1,29 @@ +# dlc_liq is a scheme that has a ccpp_constituent_properties_t variable +# outside of the register phase +[ccpp-table-properties] + name = dlc_liq + type = scheme +[ccpp-arg-table] + name = dlc_liq_init + type = scheme +[ dyn_const ] + standard_name = dynamic_constituents_for_dlc_liq + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out + allocatable = true +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/advection_auto_clone/test_advection_host_integration.F90 b/end-to-end-tests/advection_auto_clone/test_advection_host_integration.F90 new file mode 100644 index 00000000..f1f73576 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_advection_host_integration.F90 @@ -0,0 +1,79 @@ +program test + use test_prog, only: test_host, & + suite_info, & + cm, & + cs + + implicit none + + character(len=cs), target :: test_parts1(1) + character(len=cm), target :: test_invars1(11) + character(len=cm), target :: test_outvars1(13) + character(len=cm), target :: test_reqvars1(18) + + type(suite_info) :: test_suites(1) + logical :: run_okay + + test_parts1 = (/ 'physics '/) + test_invars1 = (/ & + 'banana_array_dim ', & + 'cloud_ice_dry_mixing_ratio ', & + 'cloud_liquid_dry_mixing_ratio ', & + 'surface_air_pressure ', & + 'temperature ', & + 'time_step_for_physics ', & + 'water_temperature_at_freezing ', & + 'ccpp_constituent_tendencies ', & + 'ccpp_constituents ', & + 'number_of_ccpp_constituents ', & + 'water_vapor_specific_humidity ' /) + test_outvars1 = (/ & + 'ccpp_error_message ', & + 'ccpp_error_code ', & + 'temperature ', & + 'water_vapor_specific_humidity ', & + 'cloud_liquid_dry_mixing_ratio ', & + 'ccpp_constituent_tendencies ', & + 'ccpp_constituents ', & + 'dynamic_constituents_for_cld_liq ', & + 'dynamic_constituents_for_cld_ice ', & + 'tendency_of_cloud_liquid_dry_mixing_ratio', & + 'test_banana_constituent_index ', & + 'test_banana_constituent_indices ', & + 'cloud_ice_dry_mixing_ratio ' /) + test_reqvars1 = (/ & + 'banana_array_dim ', & + 'surface_air_pressure ', & + 'temperature ', & + 'time_step_for_physics ', & + 'cloud_liquid_dry_mixing_ratio ', & + 'tendency_of_cloud_liquid_dry_mixing_ratio', & + 'cloud_ice_dry_mixing_ratio ', & + 'dynamic_constituents_for_cld_liq ', & + 'dynamic_constituents_for_cld_ice ', & + 'water_temperature_at_freezing ', & + 'ccpp_constituent_tendencies ', & + 'ccpp_constituents ', & + 'number_of_ccpp_constituents ', & + 'test_banana_constituent_index ', & + 'test_banana_constituent_indices ', & + 'water_vapor_specific_humidity ', & + 'ccpp_error_message ', & + 'ccpp_error_code ' /) + + ! Setup expected test suite info + test_suites(1)%suite_name = 'cld_suite' + test_suites(1)%suite_parts => test_parts1 + test_suites(1)%suite_input_vars => test_invars1 + test_suites(1)%suite_output_vars => test_outvars1 + test_suites(1)%suite_required_vars => test_reqvars1 + + call test_host(run_okay, test_suites) + + if (run_okay) then + stop 0 + else + stop -1 + end if + +end program test diff --git a/end-to-end-tests/advection_auto_clone/test_host.F90 b/end-to-end-tests/advection_auto_clone/test_host.F90 new file mode 100644 index 00000000..cd41f0f7 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_host.F90 @@ -0,0 +1,1160 @@ +module test_prog + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + private + + public test_host + + ! Public data and interfaces + integer, public, parameter :: cs = 16 + integer, public, parameter :: cm = 41 + + !> \section arg_table_suite_info Argument Table + !! \htmlinclude arg_table_suite_info.html + !! + type, public :: suite_info + character(len=cs) :: suite_name = '' + character(len=cs), pointer :: suite_parts(:) => null() + character(len=cm), pointer :: suite_input_vars(:) => null() + character(len=cm), pointer :: suite_output_vars(:) => null() + character(len=cm), pointer :: suite_required_vars(:) => null() + end type suite_info + + type(ccpp_constituent_properties_t), private, target, allocatable :: host_constituents(:) + + private :: check_suite + private :: advect_constituents ! Move data around + private :: check_errflg + +contains + + subroutine check_errflg(subname, errflg, errmsg, errflg_final) + ! If errflg is not zero, print an error message + character(len=*), intent(in) :: subname + integer, intent(in) :: errflg + character(len=*), intent(in) :: errmsg + + integer, intent(out) :: errflg_final + + if (errflg /= 0) then + write(6, '(a,i0,4a)') "Error ", errflg, " from ", trim(subname), & + ':', trim(errmsg) + !Notify test script that a failure occurred: + errflg_final = -1 !Notify test script that a failure occured + end if + + end subroutine check_errflg + + logical function check_suite(test_suite) + use test_host_ccpp_cap, only: ccpp_physics_suite_part_list + use test_host_ccpp_cap, only: ccpp_physics_suite_variables + use test_utils, only: check_list + + ! Dummy argument + type(suite_info), intent(in) :: test_suite + ! Local variables + logical :: check + integer :: errflg + character(len=512) :: errmsg + character(len=128), allocatable :: test_list(:) + + check_suite = .true. + ! First, check the suite parts + call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_parts, 'part names', & + suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the input variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.true., output_vars=.false.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_input_vars, & + 'input variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check the output variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg, input_vars=.false., output_vars=.true.) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_output_vars, & + 'output variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + ! Check all required variables + call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & + errmsg, errflg) + if (errflg == 0) then + check = check_list(test_list, test_suite%suite_required_vars, & + 'required variable names', suite_name=test_suite%suite_name) + else + check = .false. + write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + end if + check_suite = check_suite .and. check + if (allocated(test_list)) then + deallocate(test_list) + end if + end function check_suite + + subroutine advect_constituents() + use test_host_mod, only: phys_state, & + ncnst + use test_host_mod, only: twist_array + + ! Local variables + integer :: q_ind ! Constituent index + + do q_ind = 1, ncnst ! Skip checks, they were done in constituents_in + call twist_array(phys_state%q(:, :, q_ind)) + end do + end subroutine advect_constituents + + !> \section arg_table_test_host Argument Table + !! \htmlinclude arg_table_test_host.html + !! + subroutine test_host(retval, test_suites) + + use ccpp_constituent_prop_mod, only: ccpp_constituent_prop_ptr_t + use test_host_mod, only: num_time_steps + use test_host_mod, only: init_data, & + compare_data + use test_host_mod, only: ncols, & + pver + use test_host_data, only: num_consts, & + std_name_array, & + const_std_name + use test_host_data, only: check_constituent_indices + use test_host_ccpp_cap, only: ccpp_deallocate_dynamic_constituents + use test_host_ccpp_cap, only: ccpp_register_constituents + use test_host_ccpp_cap, only: ccpp_is_scheme_constituent + use test_host_ccpp_cap, only: ccpp_initialize_constituents + use test_host_ccpp_cap, only: ccpp_number_constituents + use test_host_ccpp_cap, only: ccpp_constituents_array + use test_host_ccpp_cap, only: ccpp_register + use test_host_ccpp_cap, only: ccpp_init + use test_host_ccpp_cap, only: ccpp_physics_init + use test_host_ccpp_cap, only: ccpp_physics_timestep_init + use test_host_ccpp_cap, only: ccpp_physics_run + use test_host_ccpp_cap, only: ccpp_physics_timestep_final + use test_host_ccpp_cap, only: ccpp_physics_final + use test_host_ccpp_cap, only: ccpp_final + use test_host_ccpp_cap, only: ccpp_physics_suite_list + use test_host_ccpp_cap, only: ccpp_const_get_index + use test_host_ccpp_cap, only: ccpp_model_const_properties + use test_utils, only: check_list + + type(suite_info), intent(in) :: test_suites(:) + logical, intent(out) :: retval + + logical :: check + integer :: col_start, col_end + integer :: index, sind + integer :: index_liq, index_ice + integer :: index_dyn1, index_dyn2, index_dyn3 + integer :: time_step + integer :: num_suites + integer :: num_advected ! Num advected species + logical :: const_log + logical :: is_constituent + logical :: has_default + integer :: test_scalar_const_index + integer :: test_const_indices(num_consts) + character(len=128), allocatable :: suite_names(:) + character(len=256) :: const_str + character(len=512) :: errmsg + character(len=512) :: expected_error + integer :: errflg + integer :: errflg_final ! Used to notify testing script of test failure + real(kind=kind_phys), pointer :: const_ptr(:, :, :) + real(kind=kind_phys) :: default_value + real(kind=kind_phys) :: check_value + type(ccpp_constituent_prop_ptr_t), pointer :: const_props(:) + character(len=*), parameter :: subname = 'test_host' + + ! Initialized "final" error flag used to report a failure to the larged + ! testing script: + errflg_final = 0 + + ! Gather and test the inspection routines + num_suites = size(test_suites) + call ccpp_physics_suite_list(suite_names) + retval = check_list(suite_names, test_suites(:)%suite_name, & + 'suite names') + write(6, *) 'Available suites are:' + do index = 1, size(suite_names) + do sind = 1, num_suites + if (trim(test_suites(sind)%suite_name) == & + trim(suite_names(index))) then + exit + end if + end do + write(6, '(i0,3a,i0,a)') index, ') ', trim(suite_names(index)), & + ' = test_suites(', sind, ')' + end do + if (retval) then + do sind = 1, num_suites + check = check_suite(test_suites(sind)) + retval = retval .and. check + end do + end if + !!! Return here if any check failed + if (.not. retval) then + return + end if + + errflg = 0 + errmsg = '' + + ! Check that is_scheme_constituent works as expected + call ccpp_is_scheme_constituent('specific_humidity', & + is_constituent, errflg, errmsg) + call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & + errmsg, errflg_final) + ! specific_humidity should not be an existing constituent + if (is_constituent) then + write(6, *) "ERROR: specific humidity is already a constituent" + errflg_final = -1 ! Notify test script that a failure occurred + end if + call ccpp_is_scheme_constituent('cloud_ice_dry_mixing_ratio', & + is_constituent, errflg, errmsg) + call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & + errmsg, errflg_final) + ! cloud_ice_dry_mixing_ratio should be an existing constituent + if (.not. is_constituent) then + write(6, *) "ERROR: cloud_ice_dry_mixing ratio not found in ", & + "host cap constituent list" + errflg_final = -1 ! Notify test script that a failure occurred + end if + + ! Use the suite information to call the register phase + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_register(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in register of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + + ! Register the constituents to find out what needs advecting + ! DO A COUPLE OF TESTS FIRST + + ! First confirm the correct error occurs if you try to add an + ! incompatible constituent with the same standard name + expected_error = 'ccp_model_const_add_metadata ERROR: Trying to add ' //& + 'constituent specific_humidity but an incompatible ' // & + 'constituent with this name already exists' + allocate(host_constituents(2)) + call host_constituents(1)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errflg, errmsg=errmsg) + call host_constituents(2)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errflg, errmsg=errmsg) + call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) + if (errflg == 0) then + call ccpp_register_constituents(host_constituents, & + errmsg=errmsg, errflg=errflg) + end if + ! Check the error + if (errflg == 0) then + write(6, '(2a)') 'ERROR register_constituents: expected this error: ', & + trim(expected_error) + else + if (trim(errmsg) /= trim(expected_error)) then + write(6, '(4a)') 'ERROR register_constituents: expected this error: ', & + trim(expected_error), ' Got: ', trim(errmsg) + end if + end if + + ! Now try again but with a compatible constituent - should be ignored when + ! the constituents object is created + ! Use the suite information to call the register phase + errflg = 0 + call ccpp_deallocate_dynamic_constituents() + deallocate(host_constituents) + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_register(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in register of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + allocate(host_constituents(2)) + call host_constituents(1)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errflg, errmsg=errmsg) + call host_constituents(2)%instantiate(std_name="specific_humidity", & + long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & + vertical_dim="vertical_layer_dimension", advected=.true., & + min_value=1000._kind_phys, molar_mass=2000._kind_phys, & + errcode=errflg, errmsg=errmsg) + call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) + if (errflg == 0) then + call ccpp_register_constituents(host_constituents, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(2a)') 'ERROR register_constituents: ', trim(errmsg) + retval = .false. + return + end if + ! Check number of advected constituents + if (errflg == 0) then + call ccpp_number_constituents(num_advected, errmsg=errmsg, & + errflg=errflg) + call check_errflg(subname // ".num_advected", errflg, errmsg, errflg_final) + end if + if (num_advected /= 6) then + write(6, '(a,i0)') "ERROR: num advected constituents = ", num_advected + retval = .false. + return + end if + ! Initialize constituent data + call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, errflg=errflg, errmsg=errmsg) + + ! Stop tests here if initialization failed (as all other tests will likely + ! fail as well: + if (errflg /= 0) then + retval = .false. + return + end if + + ! Initialize our 'data' + const_ptr => ccpp_constituents_array() + + ! Check if the specific humidity index can be found: + call ccpp_const_get_index('specific_humidity', const_index=index, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_specific_humidity", errflg, errmsg, & + errflg_final) + + ! Check if the cloud liquid index can be found: + call ccpp_const_get_index(stdname='cloud_liquid_dry_mixing_ratio', & + const_index=index_liq, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_cld_liq", errflg, errmsg, & + errflg_final) + + ! Check if the cloud ice index can be found: + call ccpp_const_get_index(stdname='cloud_ice_dry_mixing_ratio', & + const_index=index_ice, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_cld_ice", errflg, errmsg, & + errflg_final) + + ! Check if the dynamic constituents indices can be found + call ccpp_const_get_index(stdname='dyn_const1', const_index=index_dyn1, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_dyn_const1", errflg, errmsg, & + errflg_final) + call ccpp_const_get_index(stdname='dyn_const2_wrt_moist_air', const_index=index_dyn2, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_dyn_const2", errflg, errmsg, & + errflg_final) + call ccpp_const_get_index(stdname='dyn_const3_wrt_moist_air_and_condensed_water', const_index=index_dyn3, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // ".index_dyn_const3", errflg, errmsg, & + errflg_final) + + ! Load up the test array indices + call ccpp_const_get_index(stdname=const_std_name, const_index=test_scalar_const_index, errflg=errflg, errmsg=errmsg) + call check_errflg(subname // "." // const_std_name, errflg, errmsg, & + errflg_final) + do sind = 1, num_consts + call ccpp_const_get_index(stdname=std_name_array(sind), & + const_index=test_const_indices(sind), errflg=errflg, errmsg=errmsg) + call check_errflg(subname // "." // std_name_array(sind), errflg, errmsg, & + errflg_final) + end do + + ! Stop tests here if the index checks failed, as all other tests will + ! likely fail as well: + if (errflg_final /= 0) then + retval = .false. + return + end if + + call init_data(const_ptr, index, index_liq, index_ice, index_dyn3) + + ! Check some constituent properties + ! ++++++++++++++++++++++++++++++++++ + + const_props => ccpp_model_const_properties() + + ! Standard name: + call const_props(index)%standard_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get standard_name for specific_humidity, index = ", & + index, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'specific_humidity') then + write(6, *) "ERROR: standard name, '", trim(const_str), & + "' should be 'specific_humidity'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check standard name for a dynamic constituent + call const_props(index_dyn2)%standard_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get standard_name for dyn_const2, index = ", & + index_dyn2, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'dyn_const2_wrt_moist_air') then + write(6, *) "ERROR: standard name, '", trim(const_str), & + "' should be 'dyn_const2_wrt_moist_air'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Long name: + call const_props(index_liq)%long_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get long_name for cld_liq index = ", & + index_liq, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'Cloud liquid dry mixing ratio') then + write(6, *) "ERROR: long name, '", trim(const_str), & + "' should be 'Cloud liquid dry mixing ratio'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check long name for a dynamic constituent + call const_props(index_dyn1)%long_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get long_name for dyn_const1 index = ", & + index_dyn1, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'dyn const1') then + write(6, *) "ERROR: long name, '", trim(const_str), & + "' should be 'dyn const1'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Diagnostic name: + call const_props(index_liq)%diagnostic_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get diagnostic name for cld_liq index = ", & + index_liq, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'CLDLIQ') then + write(6, *) "ERROR: diagnostic name, '", trim(const_str), & + "' should be 'CLDLIQ'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check default diagnostic name is set correctly + call const_props(index_ice)%diagnostic_name(const_str, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get diagnostic name for cld_ice index = ", & + index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'cld_ice_array') then + write(6, *) "ERROR: diagnostic name, '", trim(const_str), & + "' should be 'cld_ice_array'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check diagnostic name of a dynamic constituent + call const_props(index_dyn2)%diagnostic_name(const_str, errflg, & + errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get diagnostic name for dyn_const2 index = ", & + index_dyn2, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (trim(const_str) /= 'DYNCONST2') then + write(6, *) "ERROR: diagnostic name, '", trim(const_str), & + "' should be 'DYNCONST2'" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Mass mixing ratio: + call const_props(index_ice)%is_mass_mixing_ratio(const_log, errflg, & + errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get mass mixing ratio prop for cld_ice index = ", & + index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: cloud ice is not a mass mixing_ratio" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check mass mixing ratio for a dynamic constituent + call const_props(index_dyn2)%is_mass_mixing_ratio(const_log, errflg, & + errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get mass mixing ratio prop for dyn_const2 index = ", & + index_dyn2, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occured + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const2 is not a mass mixing_ratio" + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Dry mixing ratio: + call const_props(index_ice)%is_dry(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get dry prop for cld_ice index = ", index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: cloud ice mass_mixing_ratio is not dry" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check wet mixing ratio for dynamic constituent 1 + call const_props(index_dyn1)%is_dry(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get dry prop for dyn_const1 index = ", index_dyn1, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (const_log) then + write(6, *) "ERROR: dyn_const1 is dry and should be wet" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + call const_props(index_dyn1)%is_wet(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get wet prop for dyn_const1 index = ", index_dyn1, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const1 is not wet but should be" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check moist mixing ratio for dynamic constituent 2 + call const_props(index_dyn2)%is_dry(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get dry prop for dyn_const2 index = ", index_dyn2, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (const_log) then + write(6, *) "ERROR: dyn_const2 is dry and should be moist" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + call const_props(index_dyn2)%is_moist(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get moist prop for dyn_const2 index = ", index_dyn2, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const2 is not moist but should be" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! Check dry mixing ratio for dynamic constituent 3 + call const_props(index_dyn3)%is_dry(const_log, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get dry prop for dyn_const3 index = ", index_dyn3, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (.not. const_log) then + write(6, *) "ERROR: dyn_const3 is not dry and should be" + errflg_final = -1 + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! ------------------- + + ! ------------------- + ! minimum value tests: + ! ------------------- + + ! Check that a constituent's minimum value defaults to zero: + call const_props(index_dyn2)%minimum(check_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get minimum value for dyn_const2 index = ", index_dyn2, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (check_value /= 0._kind_phys) then ! Should be zero + write(6, *) "ERROR: 'minimum' should default to zero for all ", & + "constituents unless set by host model or scheme metadata." + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that a constituent instantiated with a specified minimum value + ! actually contains that minimum value property: + call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get minimum value for dyn_const1 index = ", index_dyn1, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (check_value /= 1000._kind_phys) then !Should be 1000 + write(6, *) "ERROR: 'minimum' should give a value of 1000 ", & + "for dyn_const1, as was set during instantiation." + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that setting a constituent's minimum value works + ! as expected: + call const_props(index_dyn1)%set_minimum(1._kind_phys, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to set minimum value for dyn_const1 index = ", index_dyn1, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + " trying to get minimum value for dyn_const1 index = ", & + index_dyn1, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errflg == 0) then + if (check_value /= 1._kind_phys) then ! Should now be one + write(6, *) "ERROR: 'set_minimum' did not set constituent", & + " minimum value correctly." + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! ---------------------- + ! molecular weight tests: + ! ---------------------- + + ! Check that a constituent instantiated with a specified molecular + ! weight actually contains that molecular weight property value: + call const_props(index)%molar_mass(check_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get molecular weight for specific humidity index = ", & + index, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (check_value /= 2000._kind_phys) then ! Should be 2000 + write(6, *) "ERROR: 'molar_mass' should give a value of 2000 ", & + "for specific humidity, as was set during instantiation." + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that setting a constituent's molecular weight works + ! as expected: + call const_props(index_ice)%set_molar_mass(1._kind_phys, errflg, & + errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to set molecular weight for cld_ice index = ", index_ice, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + call const_props(index_ice)%molar_mass(check_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + " trying to get molecular weight for cld_ice index = ", & + index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errflg == 0) then + if (check_value /= 1._kind_phys) then ! Should be equal to one + write(6, *) "ERROR: 'set_molar_mass' did not set constituent", & + " molecular weight value correctly." + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! ------------------- + ! thermo-active tests: + ! ------------------- + + ! Check that being thermodynamically active defaults to False: + call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get thermo_active prop for cld_ice index = ", index_ice, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (check) then ! Should be False + write(6, *) "ERROR: 'is_thermo_active' should default to False ", & + "for all constituents unless set by host model." + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that setting a constituent to be thermodynamically active works + ! as expected: + call const_props(index_ice)%set_thermo_active(.true., errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to set thermo_active prop for cld_ice index = ", index_ice, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + " trying to get thermo_active prop for cld_ice index = ", & + index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errflg == 0) then + if (.not. check) then ! Should now be True + write(6, *) "ERROR: 'set_thermo_active' did not set", & + " thermo_active constituent property correctly." + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! ------------------- + + ! ------------------- + ! water-species tests: + ! ------------------- + + ! Check that being a water species defaults to False: + call const_props(index_liq)%is_water_species(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to get water_species prop for cld_liq index = ", index_liq, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (check) then ! Should be False + write(6, *) "ERROR: 'is_water_species' should default to False ", & + "for all constituents unless set by host model." + errflg_final = -1 ! Notify test script that a failure occured + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that setting a constituent to be a water species works + ! as expected: + call const_props(index_liq)%set_water_species(.true., errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to set water_species prop for cld_liq index = ", index_liq, & + trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + call const_props(index_liq)%is_water_species(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + " trying to get water_species prop for cld_liq index = ", & + index_liq, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + end if + if (errflg == 0) then + if (.not. check) then ! Should now be True + write(6, *) "ERROR: 'set_water_species' did not set", & + " water_species constituent property correctly." + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + + ! Check that setting a constituent to be a water species via the + ! instantiate call works as expected + call const_props(index_dyn1)%is_water_species(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + "trying to get water_species prop for dyn_const1 index = ", & + index_dyn1, trim(errmsg) + end if + if (errflg == 0) then + if (.not. check) then ! Should now be True + write(6, *) "ERROR: 'water_species=.true. did not set", & + " water_species constituent property correctly" + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + call const_props(index_dyn2)%is_water_species(check, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + "trying to get water_species prop for dyn_const2 index = ", & + index_dyn2, trim(errmsg) + end if + if (errflg == 0) then + if (check) then ! Should now be False + write(6, *) "ERROR: 'water_species=.false. did not set", & + " water_species constituent property correctly" + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! ------------------- + + ! Check that setting a constituent's default value works as expected + call const_props(index_liq)%has_default(has_default, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to check for default for cld_liq index = ", index_liq, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (has_default) then + write(6, *) "ERROR: cloud liquid mass_mixing_ratio should not have default but does" + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + call const_props(index_ice)%has_default(has_default, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to check for default for cld_ice index = ", index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (.not. has_default) then + write(6, *) "ERROR: cloud ice_dry_mixing_ratio should have default but doesn't" + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + call const_props(index_ice)%default_value(default_value, errflg, errmsg) + if (errflg /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + "to grab default for cld_ice index = ", index_ice, trim(errmsg) + errflg_final = -1 ! Notify test script that a failure occurred + end if + if (errflg == 0) then + if (default_value /= 0.0_kind_phys) then + write(6, *) "ERROR: cloud ice mass_mixing_ratio default is ", default_value, & + " but should be 0.0" + errflg_final = -1 ! Notify test script that a failure occurred + end if + else + ! Reset error flag to continue testing other properties: + errflg = 0 + end if + ! ++++++++++++++++++++++++++++++++++ + + ! Set error flag to the "final" value, because any error + ! above will likely result in a large number of failures + ! below: + errflg = errflg_final + + ! Call ccpp_init + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_init(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + + ! Call ccpp_physics_init + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_physics_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(4a)') 'ERROR in initialize of ', & + trim(test_suites(sind)%suite_name), ': ', trim(errmsg) + exit + end if + end if + end do + + ! Check indices + call check_constituent_indices(test_scalar_const_index, test_const_indices, & + errmsg, errflg) + call check_errflg(subname // " check suite indices", errflg, errmsg, & + errflg_final) + + ! Loop over time steps + do time_step = 1, num_time_steps + ! Initialize the timestep + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_physics_timestep_init( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + end if + end if + end do + + do col_start = 1, ncols, 5 + if (errflg /= 0) then + continue + end if + col_end = min(col_start + 4, ncols) + + do sind = 1, num_suites + do index = 1, size(test_suites(sind)%suite_parts) + if (errflg == 0) then + call ccpp_physics_run( & + suite_name=test_suites(sind)%suite_name, & + group_name=test_suites(sind)%suite_parts(index), & + col_start=col_start, col_end=col_end, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(5a)') trim(test_suites(sind)%suite_name), & + '/', trim(test_suites(sind)%suite_parts(index)),& + ': ', trim(errmsg) + exit + end if + end if + end do + end do + end do + ! Check indices + call check_constituent_indices(test_scalar_const_index, test_const_indices, & + errmsg, errflg) + call check_errflg(subname // " check suite indices", errflg, errmsg, & + errflg_final) + + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_physics_timestep_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + end if + if (errflg /= 0) then + write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & + trim(errmsg) + exit + end if + end do + + ! Run "dycore" + if (errflg == 0) then + call advect_constituents() + end if + end do ! End time step loop + + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_physics_final( & + suite_name=test_suites(sind)%suite_name, & + group_name='all', col_start=1, col_end=ncols, & + thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & + 'Exiting...' + exit + end if + end if + end do + + do sind = 1, num_suites + if (errflg == 0) then + call ccpp_final(suite_name=test_suites(sind)%suite_name, & + errmsg=errmsg, errflg=errflg) + if (errflg /= 0) then + write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & + trim(errmsg) + write(6, '(2a)') 'An error occurred in ccpp_final, ', & + 'Exiting...' + exit + end if + end if + end do + + call ccpp_deallocate_dynamic_constituents() + deallocate(host_constituents) + + if (errflg == 0) then + ! Run finished without error, check answers + if (compare_data(num_advected)) then + write(6, *) 'Answers are correct!' + errflg = 0 + else + write(6, *) 'Answers are not correct!' + errflg = -1 + end if + end if + + ! Make sure "final" flag is non-zero if "errflg" is: + if (errflg /= 0) then + errflg_final = -1 ! Notify test script that a failure occured + end if + + ! Set return value to False if any errors were found: + retval = errflg_final == 0 + + end subroutine test_host + +end module test_prog diff --git a/end-to-end-tests/advection_auto_clone/test_host.meta b/end-to-end-tests/advection_auto_clone/test_host.meta new file mode 100644 index 00000000..ab33172f --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_host.meta @@ -0,0 +1,70 @@ +[ccpp-table-properties] + name = suite_info + type = ddt +[ccpp-arg-table] + name = suite_info + type = ddt + +[ccpp-table-properties] + name = test_host + type = control +[ccpp-arg-table] + name = test_host + type = control +[ col_start ] + standard_name = horizontal_loop_begin + type = integer + units = count + dimensions = () + protected = True +[ col_end ] + standard_name = horizontal_loop_end + type = integer + units = count + dimensions = () + protected = True +[ errmsg ] + standard_name = ccpp_error_message + long_name = Error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = Error flag for error handling in CCPP + units = 1 + dimensions = () + type = integer +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer diff --git a/end-to-end-tests/advection_auto_clone/test_host_data.F90 b/end-to-end-tests/advection_auto_clone/test_host_data.F90 new file mode 100644 index 00000000..f360ad79 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_host_data.F90 @@ -0,0 +1,96 @@ +module test_host_data + + use ccpp_kinds, only: kind_phys + + implicit none + + !> \section arg_table_physics_state Argument Table + !! \htmlinclude arg_table_physics_state.html + type physics_state + real(kind=kind_phys), allocatable :: ps(:) ! surface pressure + real(kind=kind_phys), allocatable :: temp(:, :) ! temperature + real(kind=kind_phys), dimension(:, :, :), pointer :: q => null() ! constituent array + end type physics_state + + !> \section arg_table_test_host_data Argument Table + !! \htmlinclude arg_table_test_host_data.html + integer, public, parameter :: num_consts = 3 + character(len=32), public, parameter :: std_name_array(num_consts) = (/ & + 'specific_humidity ', & + 'cloud_liquid_dry_mixing_ratio', & + 'cloud_ice_dry_mixing_ratio ' /) + character(len=32), public, parameter :: const_std_name = std_name_array(1) + + integer :: const_inds(num_consts) = -1 ! test array access from suite + integer :: const_index = -1 ! test scalar access from suite + + public :: allocate_physics_state + public :: check_constituent_indices + +contains + + subroutine check_constituent_indices(test_index, test_indices, errmsg, errflg) + ! Check constituent indices against what was found by suite + ! indices are passed in rather than looked up to avoid a dependency loop + ! Dummy arguments + integer, intent(in) :: test_index ! scalar const index from host + integer, intent(in) :: test_indices(:) ! array_test_indices from host + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + + ! Local variable + integer :: indx + integer :: emstrt + + errflg = 0 + errmsg = '' + if (test_index /= const_index) then + emstrt = len_trim(errmsg) + 1 + write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_index_check for ', & + const_std_name, test_index, ' /= ', const_index + errflg = errflg + 1 + end if + do indx = 1, num_consts + if (test_indices(indx) /= const_inds(indx)) then + emstrt = len_trim(errmsg) + 1 + if (len_trim(errmsg) > 0) then + write(errmsg(emstrt:), '(", ")') + emstrt = emstrt + 2 + end if + write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_indices_check for ', & + std_name_array(indx), test_indices(indx), ' /= ', const_inds(indx) + errflg = errflg + 1 + end if + end do + + ! Reset for next test + const_index = -1 + const_inds = -1 + + end subroutine check_constituent_indices + + subroutine allocate_physics_state(cols, levels, constituents, state) + integer, intent(in) :: cols + integer, intent(in) :: levels + real(kind=kind_phys), pointer :: constituents(:, :, :) + type(physics_state), intent(out) :: state + + if (allocated(state%ps)) then + deallocate(state%ps) + end if + allocate(state%ps(cols)) + state%ps = 0.0_kind_phys + if (allocated(state%temp)) then + deallocate(state%temp) + end if + allocate(state%temp(cols, levels)) + if (associated(state%q)) then + ! Do not deallocate (we do not own this array) + nullify(state%q) + end if + ! Point to the advected constituents array + state%q => constituents + + end subroutine allocate_physics_state + +end module test_host_data diff --git a/end-to-end-tests/advection_auto_clone/test_host_data.meta b/end-to-end-tests/advection_auto_clone/test_host_data.meta new file mode 100644 index 00000000..960ce33e --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_host_data.meta @@ -0,0 +1,67 @@ +[ccpp-table-properties] + name = physics_state + type = ddt +[ccpp-arg-table] + name = physics_state + type = ddt +[ ps ] + standard_name = surface_air_pressure + type = real + kind = kind_phys + units = Pa + dimensions = (horizontal_dimension) +[ Temp ] + standard_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys +[ q ] + standard_name = constituent_mixing_ratio + type = real + kind = kind_phys + units = kg kg-1 moist or dry air depending on type + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) +[ q(:,:,index_of_water_vapor_specific_humidity) ] + standard_name = water_vapor_specific_humidity + type = real + kind = kind_phys + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + +[ccpp-table-properties] + name = test_host_data + type = host +[ccpp-arg-table] + name = test_host_data + type = host +[ num_consts ] + standard_name = banana_array_dim + long_name = Size of test_banana_name_array + units = 1 + dimensions = () + type = integer +[ std_name_array ] + standard_name = test_banana_name_array + type = character | kind = len=32 + units = count + dimensions = (banana_array_dim) + protected = true +[ const_std_name ] + standard_name = test_banana_name + type = character | kind = len=32 + units = 1 + dimensions = () + protected = true +[ const_inds ] + standard_name = test_banana_constituent_indices + long_name = Array of constituent indices + units = 1 + dimensions = (banana_array_dim) + protected = true + type = integer +[ const_index ] + standard_name = test_banana_constituent_index + long_name = Constituent index + units = 1 + dimensions = () + type = integer diff --git a/end-to-end-tests/advection_auto_clone/test_host_mod.F90 b/end-to-end-tests/advection_auto_clone/test_host_mod.F90 new file mode 100644 index 00000000..5099b9c1 --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_host_mod.F90 @@ -0,0 +1,176 @@ +module test_host_mod + + use ccpp_kinds, only: kind_phys + use test_host_data, only: physics_state, & + allocate_physics_state + + implicit none + public + + integer, parameter :: num_time_steps = 2 + real(kind=kind_phys), parameter :: tolerance = 1.0e-13_kind_phys + + !> \section arg_table_test_host_mod Argument Table + !! \htmlinclude arg_table_test_host_mod.html + !! + integer, parameter :: ncols = 10 + integer, parameter :: pver = 5 + integer, parameter :: pverp = pver + 1 + integer, protected :: ncnst = -1 + integer, protected :: index_qv = -1 + real(kind=kind_phys) :: dt + real(kind=kind_phys), parameter :: tfreeze = 273.15_kind_phys + type(physics_state) :: phys_state + integer :: num_model_times = -1 + integer, allocatable :: model_times(:) + + public :: init_data + public :: compare_data + public :: twist_array + + real(kind=kind_phys), private, allocatable :: check_vals(:, :, :) + real(kind=kind_phys), private :: check_temp(ncols, pver) + integer, private :: ind_liq = -1 + integer, private :: ind_ice = -1 + +contains + + subroutine init_data(constituent_array, index_qv_use, index_liq, index_ice, index_dyn) + + ! Dummy arguments + real(kind=kind_phys), pointer :: constituent_array(:, :, :) ! From host & suites + integer, intent(in) :: index_qv_use + integer, intent(in) :: index_liq + integer, intent(in) :: index_ice + integer, intent(in) :: index_dyn + + ! Local variables + integer :: col + integer :: lev + integer :: cind + integer :: itime + real(kind=kind_phys) :: qmax + real(kind=kind_phys), parameter :: inc = 0.1_kind_phys + + ! Allocate and initialize state + ! Temperature starts above freezing and decreases to -30C + ! water vapor is initialized in odd columns to different amounts + ncnst = size(constituent_array, 3) + call allocate_physics_state(ncols, pver, constituent_array, phys_state) + index_qv = index_qv_use + ind_liq = index_liq + ind_ice = index_ice + allocate(check_vals(ncols, pver, ncnst)) + check_vals(:, :, :) = 0.0_kind_phys + check_vals(:, :, index_dyn) = 1.0_kind_phys + do lev = 1, pver + phys_state%temp(:, lev) = tfreeze + (10.0_kind_phys * (lev - 3)) + qmax = real(lev, kind_phys) + do col = 1, ncols + if (mod(col, 2) == 1) then + phys_state%q(col, lev, index_qv) = qmax + else + phys_state%q(col, lev, index_qv) = 0.0_kind_phys + end if + end do + end do + check_vals(:, :, index_qv) = phys_state%q(:, :, index_qv) + check_temp(:, :) = phys_state%temp(:, :) + ! Do timestep 1 + do col = 1, ncols, 2 + check_temp(col, 1) = check_temp(col, 1) + 0.5_kind_phys + check_vals(col, 1, index_qv) = check_vals(col, 1, index_qv) - inc + check_vals(col, 1, ind_liq) = check_vals(col, 1, ind_liq) + inc + end do + do itime = 1, num_time_steps + do cind = 1, ncnst + call twist_array(check_vals(:, :, cind)) + end do + end do + + end subroutine init_data + + subroutine twist_array(array) + ! Dummy argument + real(kind=kind_phys), intent(inout) :: array(:, :) + + ! Local variables + integer :: icol, ilev ! Field coordinates + integer :: idir ! 'w' sign + integer :: levb, leve ! Starting and ending level indices + real(kind=kind_phys) :: last_val, next_val + + idir = 1 + leve = (pver * mod(ncols, 2)) + mod(ncols - 1, 2) + last_val = array(ncols, leve) + do icol = 1, ncols + levb = ((pver * (1 - idir)) + (1 + idir)) / 2 + leve = ((pver * (1 + idir)) + (1 - idir)) / 2 + do ilev = levb, leve, idir + next_val = array(icol, ilev) + array(icol, ilev) = last_val + last_val = next_val + end do + idir = -1 * idir + end do + + end subroutine twist_array + + logical function compare_data(ncnst) + + integer, intent(in) :: ncnst + + integer :: col + integer :: lev + integer :: cind + logical :: need_header + real(kind=kind_phys) :: check + real(kind=kind_phys) :: denom + + compare_data = .true. + + need_header = .true. + do lev = 1, pver + do col = 1, ncols + check = check_temp(col, lev) + if (abs((phys_state%temp(col, lev) - check) / check) > & + tolerance) then + if (need_header) then + write(6, '(" COL LEV T MIDPOINTS EXPECTED")') + need_header = .false. + end if + write(6, '(2i5,2(3x,es15.7))') col, lev, & + phys_state%temp(col, lev), check + compare_data = .false. + end if + end do + end do + ! Check constituents + need_header = .true. + do cind = 1, ncnst + do lev = 1, pver + do col = 1, ncols + check = check_vals(col, lev, cind) + if (check < tolerance) then + denom = 1.0_kind_phys + else + denom = check + end if + if (abs((phys_state%q(col, lev, cind) - check) / denom) > & + tolerance) then + if (need_header) then + write(6, '(2(2x,a),3x,a,10x,a,14x,a)') & + 'COL', 'LEV', 'C#', 'Q', 'EXPECTED' + need_header = .false. + end if + write(6, '(3i5,2(3x,es15.7))') col, lev, cind, & + phys_state%q(col, lev, cind), check + compare_data = .false. + end if + end do + end do + end do + + end function compare_data + +end module test_host_mod diff --git a/end-to-end-tests/advection_auto_clone/test_host_mod.meta b/end-to-end-tests/advection_auto_clone/test_host_mod.meta new file mode 100644 index 00000000..6c3d15eb --- /dev/null +++ b/end-to-end-tests/advection_auto_clone/test_host_mod.meta @@ -0,0 +1,64 @@ +[ccpp-table-properties] + name = test_host_mod + type = host +[ccpp-arg-table] + name = test_host_mod + type = host +[ ncols] + standard_name = horizontal_dimension + units = count + type = integer + protected = True + dimensions = () +[ pver ] + standard_name = vertical_layer_dimension + units = count + type = integer + protected = True + dimensions = () +[ pverP ] + standard_name = vertical_interface_dimension + type = integer + units = count + protected = True + dimensions = () +[ ncnst ] + standard_name = number_of_tracers + type = integer + units = count + protected = True + dimensions = () +[ index_qv ] + standard_name = index_of_water_vapor_specific_humidity + units = index + type = integer + protected = True + dimensions = () +[ dt ] + standard_name = time_step_for_physics + long_name = time step + units = s + dimensions = () + type = real | kind = kind_phys +[ tfreeze ] + standard_name = water_temperature_at_freezing + long_name = Freezing temperature of water at sea level + units = K + dimensions = () + type = real | kind = kind_phys +[ phys_state ] + standard_name = physics_state_derived_type + long_name = Physics State DDT + type = physics_state + dimensions = () +[ num_model_times ] + standard_name = number_of_model_times + type = integer + units = count + dimensions = () +[ model_times ] + standard_name = model_times + units = seconds + dimensions = (number_of_model_times) + type = integer + allocatable = True diff --git a/end-to-end-tests/cmake/ccpp_capgen.cmake b/end-to-end-tests/cmake/ccpp_capgen.cmake index e344bf9b..76bca88f 100644 --- a/end-to-end-tests/cmake/ccpp_capgen.cmake +++ b/end-to-end-tests/cmake/ccpp_capgen.cmake @@ -5,7 +5,7 @@ function(ccpp_validator) set(optionalArgs) set(oneValueArgs VERBOSITY) - set(multi_value_keywords SOURCE_FILES METADATA_FILES) + set(multi_value_keywords SOURCE_FILES METADATA_FILES EXTRA_FLAGS) cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN}) # Error if script file not found. @@ -33,6 +33,10 @@ function(ccpp_validator) list(APPEND CCPP_VALIDATOR_CMD_LIST ${VERBOSE_PARAMS}) endif() + if(DEFINED arg_EXTRA_FLAGS) + list(APPEND CCPP_VALIDATOR_CMD_LIST ${arg_EXTRA_FLAGS}) + endif() + message(STATUS "Running ccpp_validator.py from ${CMAKE_CURRENT_SOURCE_DIR}") unset(VALIDATOR_OUT) @@ -71,7 +75,7 @@ endfunction() function(ccpp_capgen) set(optionalArgs TRACE) set(oneValueArgs HOST_NAME OUTPUT_ROOT VERBOSITY KIND_SPECS) - set(multi_value_keywords HOSTFILES SCHEMEFILES SUITES) + set(multi_value_keywords HOSTFILES SCHEMEFILES SUITES EXTRA_FLAGS) cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN}) @@ -132,6 +136,10 @@ function(ccpp_capgen) list(APPEND CCPP_CAPGEN_CMD_LIST "--trace") endif() + if(DEFINED arg_EXTRA_FLAGS) + list(APPEND CCPP_CAPGEN_CMD_LIST ${arg_EXTRA_FLAGS}) + endif() + message(STATUS "Running ccpp_capgen.py from ${CMAKE_CURRENT_SOURCE_DIR}") # Unset CAPGEN_OUT to prevent incorrect output on subsequent ccpp_capgen(...) calls diff --git a/unit-tests/sample_files/scheme_auto_clone_consumer.meta b/unit-tests/sample_files/scheme_auto_clone_consumer.meta new file mode 100644 index 00000000..625262e1 --- /dev/null +++ b/unit-tests/sample_files/scheme_auto_clone_consumer.meta @@ -0,0 +1,48 @@ +# auto-clone-constituents: fixture for the legacy auto-clone shim +# test. Two is_constituent consumer args (no register-phase source) +# that exercise the full set of legacy attrs the shim accepts: +# default_value, min_value, water_species, mixing_ratio_type. + +[ccpp-table-properties] + name = auto_clone_consumer + type = scheme + +[ccpp-arg-table] + name = auto_clone_consumer_run + type = scheme +[ qv ] + standard_name = water_vapor_specific_humidity + long_name = base constituent water vapor + diagnostic_name = QV + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = inout + advected = .true. + default_value = 1.0e-12 + min_value = 0.0 + water_species = .true. + mixing_ratio_type = wrt_moist +[ qc ] + standard_name = cloud_liquid_dry_mixing_ratio + long_name = cloud liquid water + diagnostic_name = CLDLIQ + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in + advected = .true. + default_value = 0.0 +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/sample_suite_files/suite_auto_clone.xml b/unit-tests/sample_suite_files/suite_auto_clone.xml new file mode 100644 index 00000000..4f0959a2 --- /dev/null +++ b/unit-tests/sample_suite_files/suite_auto_clone.xml @@ -0,0 +1,9 @@ + + + + + auto_clone_consumer + + diff --git a/unit-tests/test_auto_clone_constituents.py b/unit-tests/test_auto_clone_constituents.py new file mode 100644 index 00000000..b9c586f4 --- /dev/null +++ b/unit-tests/test_auto_clone_constituents.py @@ -0,0 +1,693 @@ +"""Tests for the transient auto-clone-constituents legacy shim. + +This whole file is part of the auto-clone-constituents shim and should +be deleted alongside ``metadata/auto_clone_constituents.py`` when the +shim is retired. Search ``auto-clone-constituents`` to find every +touchpoint. +""" + +from __future__ import annotations + +import doctest +import io +import logging +import os +import sys +import unittest + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen-ng') +if _CAPGEN_DIR not in sys.path: + sys.path.insert(0, _CAPGEN_DIR) + +from metadata import auto_clone_constituents # noqa: E402 +from metadata.metadata_table import ( # noqa: E402 + MetaVar, parse_metadata_file, _parse_lines, +) +from metadata.parse_tools import CCPPError, ParseContext # noqa: E402 +from generator.suite_resolver import ( # noqa: E402 + AutoCloneEntry, + _make_auto_clone_entry, + _collect_auto_clone_entries, + _vertical_dim_of, + _synthesised_long_name_from_std, +) +from generator.suite_cap import ( # noqa: E402 + _emit_auto_clone_instantiate, + _fmt_kind_phys_real, + _esc_fortran_char, +) + + +_SAMPLES_DIR = os.path.join(_TESTS_DIR, 'sample_files') + + +def _sf(name): + return os.path.join(_SAMPLES_DIR, name) + + +def _ctx(): + return ParseContext(linenum=1, filename='auto_clone_test.meta') + + +class _AutoCloneFixture(unittest.TestCase): + """Mixin that flips the shim on for the duration of a test and + guarantees it goes back off afterwards (the flag is process state). + """ + + def setUp(self): + # Sanity: never start a test with the flag set by an earlier + # test that crashed before cleanup. + auto_clone_constituents.disable() + + def tearDown(self): + auto_clone_constituents.disable() + + +######################################################################## +# Module surface +######################################################################## + +class TestExtraKnownAttrs(_AutoCloneFixture): + + def test_empty_when_disabled(self): + self.assertFalse(auto_clone_constituents.is_enabled()) + self.assertEqual(auto_clone_constituents.extra_known_attrs(), + frozenset()) + + def test_populated_when_enabled(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + extra = auto_clone_constituents.extra_known_attrs() + self.assertEqual( + extra, + frozenset({ + 'default_value', 'min_value', + 'water_species', 'mixing_ratio_type', + }), + ) + + +class TestEnableDisable(_AutoCloneFixture): + + def test_enable_flips_flag_and_writes_banner(self): + sink = io.StringIO() + auto_clone_constituents.enable(_stream=sink) + self.assertTrue(auto_clone_constituents.is_enabled()) + out = sink.getvalue() + self.assertIn('LEGACY AUTO-CLONE-CONSTITUENTS ENABLED', out) + for attr in ('default_value', 'min_value', + 'water_species', 'mixing_ratio_type'): + self.assertIn(attr, out) + self.assertIn('single-instance', out) + self.assertIn('TRANSIENT', out) + self.assertIn('REMOVED', out) + self.assertGreaterEqual(out.count('*' * 10), 2) + + def test_enable_is_idempotent(self): + sink1 = io.StringIO() + auto_clone_constituents.enable(_stream=sink1) + first = sink1.getvalue() + sink2 = io.StringIO() + auto_clone_constituents.enable(_stream=sink2) + self.assertEqual(sink2.getvalue(), '') + self.assertIn('AUTO-CLONE', first) + + def test_disable_resets(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + self.assertTrue(auto_clone_constituents.is_enabled()) + auto_clone_constituents.disable() + self.assertFalse(auto_clone_constituents.is_enabled()) + + def test_logger_receives_warning(self): + logger = logging.getLogger('auto_clone_test_logger') + records = [] + + class _Capture(logging.Handler): + def emit(self, record): + records.append(record) + + handler = _Capture(level=logging.WARNING) + logger.addHandler(handler) + try: + auto_clone_constituents.enable( + logger=logger, _stream=io.StringIO()) + self.assertEqual(len(records), 1) + self.assertEqual(records[0].levelno, logging.WARNING) + msg = records[0].getMessage() + for attr in ('default_value', 'min_value', + 'water_species', 'mixing_ratio_type'): + self.assertIn(attr, msg) + finally: + logger.removeHandler(handler) + + +class TestSingleInstanceGuard(_AutoCloneFixture): + + def test_noop_when_disabled(self): + # Even a multi-instance host should not raise when the shim is off. + host_dict = {'instance_number': object(), + 'number_of_instances': object()} + auto_clone_constituents.require_single_instance_host(host_dict) # no raise + + def test_single_instance_passes(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + # No instance pair → fine. + auto_clone_constituents.require_single_instance_host({}) + + def test_instance_number_alone_raises(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + with self.assertRaises(CCPPError) as cm: + auto_clone_constituents.require_single_instance_host( + {'instance_number': object()} + ) + self.assertIn('single-instance', str(cm.exception)) + self.assertIn('instance_number', str(cm.exception)) + + def test_number_of_instances_alone_raises(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + with self.assertRaises(CCPPError) as cm: + auto_clone_constituents.require_single_instance_host( + {'number_of_instances': object()} + ) + self.assertIn('number_of_instances', str(cm.exception)) + + +######################################################################## +# Parser: conditional _KNOWN_ATTRS extension +######################################################################## + +class TestParserShimOff(_AutoCloneFixture): + """When the shim is OFF, the four legacy attrs are rejected with + the standard "Unknown variable attribute" error.""" + + def _make_var(self): + return MetaVar('q', _ctx()) + + def test_default_value_rejected(self): + v = self._make_var() + with self.assertRaises(CCPPError) as cm: + v.set_attr('default_value', '0.0', _ctx()) + self.assertIn('Unknown variable attribute', str(cm.exception)) + self.assertIn('default_value', str(cm.exception)) + + def test_min_value_rejected(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('min_value', '0.0', _ctx()) + + def test_water_species_rejected(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('water_species', 'True', _ctx()) + + def test_mixing_ratio_type_rejected(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('mixing_ratio_type', 'dry', _ctx()) + + +class TestParserShimOn(_AutoCloneFixture): + """When the shim is ON, the four legacy attrs parse and validate.""" + + def setUp(self): + super().setUp() + auto_clone_constituents.enable(_stream=io.StringIO()) + + def _make_var(self): + return MetaVar('q', _ctx()) + + def test_default_value_accepted(self): + v = self._make_var() + v.set_attr('default_value', '1.5e-12', _ctx()) + self.assertEqual(v.default_value, 1.5e-12) + + def test_default_value_kind_phys_suffix_accepted(self): + v = self._make_var() + v.set_attr('default_value', '0.0_kind_phys', _ctx()) + self.assertEqual(v.default_value, 0.0) + + def test_default_value_d_exponent_accepted(self): + v = self._make_var() + v.set_attr('default_value', '1.0d-5', _ctx()) + self.assertEqual(v.default_value, 1.0e-5) + + def test_default_value_invalid_raises(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('default_value', 'not_a_number', _ctx()) + + def test_min_value_accepted(self): + v = self._make_var() + v.set_attr('min_value', '-3.14', _ctx()) + self.assertEqual(v.min_value, -3.14) + + def test_min_value_kind_suffix_accepted(self): + v = self._make_var() + v.set_attr('min_value', '1.0e-12_kind_dyn', _ctx()) + self.assertEqual(v.min_value, 1.0e-12) + + def test_water_species_true(self): + v = self._make_var() + v.set_attr('water_species', '.true.', _ctx()) + self.assertIs(v.water_species, True) + + def test_water_species_false(self): + v = self._make_var() + v.set_attr('water_species', 'False', _ctx()) + self.assertIs(v.water_species, False) + + def test_water_species_invalid_raises(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('water_species', 'maybe', _ctx()) + + def test_mixing_ratio_type_dry(self): + v = self._make_var() + v.set_attr('mixing_ratio_type', 'dry', _ctx()) + self.assertEqual(v.mixing_ratio_type, 'dry') + + def test_mixing_ratio_type_lowercased(self): + v = self._make_var() + v.set_attr('mixing_ratio_type', 'WRT_MOIST', _ctx()) + self.assertEqual(v.mixing_ratio_type, 'wrt_moist') + + def test_mixing_ratio_type_invalid_raises(self): + v = self._make_var() + with self.assertRaises(CCPPError): + v.set_attr('mixing_ratio_type', 'bogus', _ctx()) + + +class TestParserSchemeOnly(_AutoCloneFixture): + """The four legacy attrs are scheme-only when the shim is on; the + parser must reject them on host/control/ddt tables.""" + + def setUp(self): + super().setUp() + auto_clone_constituents.enable(_stream=io.StringIO()) + + def test_default_value_rejected_on_host_table(self): + src = ( + '[ccpp-table-properties]\n' + ' name = h\n' + ' type = host\n' + '[ccpp-arg-table]\n' + ' name = h\n' + ' type = host\n' + '[ q ]\n' + ' standard_name = some_field\n' + ' units = kg kg-1\n' + ' dimensions = ()\n' + ' type = real | kind = kind_phys\n' + ' default_value = 0.0\n' + ) + with self.assertRaises(CCPPError) as cm: + _parse_lines(src.splitlines(keepends=True), 't.meta') + self.assertIn('scheme-only', str(cm.exception)) + + +######################################################################## +# Suite-cap helpers +######################################################################## + +class TestKindPhysFormatter(unittest.TestCase): + + def test_positive(self): + self.assertTrue(_fmt_kind_phys_real(0.0).endswith('_kind_phys')) + self.assertIn('e', _fmt_kind_phys_real(1.0e-12)) + + def test_negative(self): + s = _fmt_kind_phys_real(-3.14) + self.assertTrue(s.endswith('_kind_phys')) + self.assertTrue(s.startswith('-')) + + +class TestFortranCharEscape(unittest.TestCase): + + def test_no_quote_passthrough(self): + self.assertEqual(_esc_fortran_char("CLDLIQ"), "CLDLIQ") + + def test_single_quote_doubled(self): + self.assertEqual(_esc_fortran_char("don't"), "don''t") + + +class TestEmitAutoCloneInstantiate(unittest.TestCase): + """Render one synthesised %instantiate call and check the lines.""" + + def _entry(self, **kw): + defaults = dict( + std_name='cloud_liquid_dry_mixing_ratio', + long_name='cloud liquid water', + diag_name='CLDLIQ', + units='kg kg-1', + vertical_dim='vertical_layer_dimension', + advected=False, + molar_mass=0.0, + default_value=None, + min_value=None, + water_species=None, + mixing_ratio_type=None, + ) + defaults.update(kw) + return AutoCloneEntry(**defaults) + + def _emit(self, entry): + lines = [] + _emit_auto_clone_instantiate( + entry, buf='suite_dynamic_constituents', + inst_idx='1', indent=' ', + errflg_local='errflg', errmsg_local='errmsg', + lines=lines, + ) + return lines + + def test_required_kwargs_always_emitted(self): + lines = self._emit(self._entry()) + joined = '\n'.join(lines) + self.assertIn("std_name = 'cloud_liquid_dry_mixing_ratio'", joined) + self.assertIn("long_name = 'cloud liquid water'", joined) + self.assertIn("diag_name = 'CLDLIQ'", joined) + self.assertIn("units = 'kg kg-1'", joined) + self.assertIn("vertical_dim = 'vertical_layer_dimension'", joined) + self.assertIn("errcode = errflg", joined) + self.assertIn("errmsg = errmsg", joined) + # Error guard. + self.assertIn('if (errflg /= 0) return', joined) + # Counter increment. + self.assertEqual(lines[0].strip(), 'num_consts = num_consts + 1') + + def test_optional_kwargs_omitted_when_unset(self): + lines = self._emit(self._entry()) + joined = '\n'.join(lines) + for kw in ('advected', 'default_value', 'min_value', + 'water_species', 'mixing_ratio_type', 'molar_mass'): + self.assertNotIn(kw + ' ', joined, + "unset optional '{}' leaked into output".format(kw)) + + def test_advected_only_emitted_when_true(self): + lines = self._emit(self._entry(advected=True)) + joined = '\n'.join(lines) + self.assertIn('advected = .true.', joined) + + def test_real_kwargs_emit_kind_phys_literal(self): + lines = self._emit(self._entry( + default_value=0.0, min_value=1.0e-12, molar_mass=18.015, + )) + joined = '\n'.join(lines) + self.assertIn('default_value=', joined) + self.assertIn('min_value =', joined) + self.assertIn('molar_mass =', joined) + # Each real kwarg uses the kind_phys suffix. + self.assertEqual(joined.count('_kind_phys'), 3) + + def test_water_species_true_and_false(self): + on = self._emit(self._entry(water_species=True)) + off = self._emit(self._entry(water_species=False)) + self.assertIn('water_species= .true.', '\n'.join(on)) + self.assertIn('water_species= .false.', '\n'.join(off)) + + def test_mixing_ratio_type_quoted(self): + lines = self._emit(self._entry(mixing_ratio_type='wrt_moist')) + self.assertIn("mixing_ratio_type = 'wrt_moist'", '\n'.join(lines)) + + def test_quoted_long_name_escaped(self): + # Fortran character literal escape: every single quote is + # doubled. + lines = self._emit(self._entry(long_name="don't")) + self.assertIn("long_name = 'don''t'", '\n'.join(lines)) + + +######################################################################## +# Resolver helpers +######################################################################## + +class TestVerticalDimOf(unittest.TestCase): + + class _StubVar: + def __init__(self, dims): + self.dimensions = dims + + def test_extracts_layer_dim(self): + v = self._StubVar(['horizontal_dimension', 'vertical_layer_dimension']) + self.assertEqual(_vertical_dim_of(v), 'vertical_layer_dimension') + + def test_extracts_interface_dim(self): + v = self._StubVar( + ['horizontal_dimension', 'vertical_interface_dimension']) + self.assertEqual(_vertical_dim_of(v), 'vertical_interface_dimension') + + def test_no_vertical_dim_returns_default(self): + v = self._StubVar(['horizontal_dimension']) + self.assertEqual(_vertical_dim_of(v), 'vertical_layer_dimension') + + def test_explicit_range_form_supported(self): + v = self._StubVar([ + 'ccpp_constant_one:horizontal_dimension', + 'ccpp_constant_one:vertical_layer_dimension', + ]) + self.assertEqual(_vertical_dim_of(v), 'vertical_layer_dimension') + + +class TestMakeAutoCloneEntry(_AutoCloneFixture): + """``_make_auto_clone_entry`` snapshots a scheme MetaVar.""" + + def setUp(self): + super().setUp() + auto_clone_constituents.enable(_stream=io.StringIO()) + + def _scheme_var(self, **set_attrs): + ctx = _ctx() + v = MetaVar('qv', ctx) + v.set_attr('standard_name', 'water_vapor_specific_humidity', ctx) + v.set_attr('long_name', 'water vapor', ctx) + v.set_attr('units', 'kg kg-1', ctx) + v.set_attr('dimensions', + '(horizontal_dimension, vertical_layer_dimension)', ctx) + v.set_attr('type', 'real', ctx) + v.set_attr('kind', 'kind_phys', ctx) + v.set_attr('intent', 'inout', ctx) + for k, val in set_attrs.items(): + v.set_attr(k, val, ctx) + return v + + def test_diag_name_defaults_to_local_name(self): + v = self._scheme_var() + entry = _make_auto_clone_entry(v) + self.assertEqual(entry.diag_name, 'qv') # MetaVar.diagnostic_name fallback + + def test_explicit_diagnostic_name_wins(self): + v = self._scheme_var(diagnostic_name='QV') + entry = _make_auto_clone_entry(v) + self.assertEqual(entry.diag_name, 'QV') + + def test_optional_kwargs_passthrough(self): + v = self._scheme_var( + advected='.true.', + molar_mass='18.015', + default_value='1.0e-12', + min_value='0.0', + water_species='.true.', + mixing_ratio_type='wrt_moist', + ) + entry = _make_auto_clone_entry(v) + self.assertTrue(entry.advected) + self.assertAlmostEqual(entry.molar_mass, 18.015) + self.assertEqual(entry.default_value, 1.0e-12) + self.assertEqual(entry.min_value, 0.0) + self.assertIs(entry.water_species, True) + self.assertEqual(entry.mixing_ratio_type, 'wrt_moist') + + def test_vertical_dim_lifted_from_dimensions(self): + v = self._scheme_var() + entry = _make_auto_clone_entry(v) + self.assertEqual(entry.vertical_dim, 'vertical_layer_dimension') + + def test_explicit_long_name_wins(self): + # When the metadata supplies a long_name, the entry carries it + # verbatim (no synthesis). The shared _scheme_var helper + # already sets long_name='water vapor', so we re-use that + # fixture as the "explicit long_name set" case. + v = self._scheme_var() + entry = _make_auto_clone_entry(v) + self.assertEqual(entry.long_name, 'water vapor') + + def test_long_name_synthesised_when_missing(self): + # The _scheme_var helper sets long_name='water vapor'; override + # to a no-op so the helper sees an empty long_name. capgen-ng + # then synthesises from std_name: 'water_vapor_specific_humidity' + # → 'Water vapor specific humidity'. + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar('qv', ctx) + v.set_attr('standard_name', + 'water_vapor_specific_humidity', ctx) + v.set_attr('units', 'kg kg-1', ctx) + v.set_attr('dimensions', + '(horizontal_dimension, vertical_layer_dimension)', ctx) + v.set_attr('type', 'real', ctx) + v.set_attr('kind', 'kind_phys', ctx) + v.set_attr('intent', 'inout', ctx) + # No long_name attribute set. + entry = _make_auto_clone_entry(v) + self.assertEqual(entry.long_name, 'Water vapor specific humidity') + + +class TestSynthesisedLongName(unittest.TestCase): + """The ``_synthesised_long_name_from_std`` helper mirrors original + capgen's behaviour: replace each underscore with a space and + capitalise the first character.""" + + def test_typical_constituent(self): + self.assertEqual( + _synthesised_long_name_from_std('cloud_liquid_dry_mixing_ratio'), + 'Cloud liquid dry mixing ratio', + ) + + def test_single_underscore(self): + self.assertEqual( + _synthesised_long_name_from_std('specific_humidity'), + 'Specific humidity', + ) + + def test_no_underscore(self): + self.assertEqual( + _synthesised_long_name_from_std('temperature'), + 'Temperature', + ) + + def test_mixed_case_lowercased_after_first(self): + # Python's ``.capitalize()`` lowercases everything after the + # first character. That's a deliberate match for original + # capgen's behaviour and means a std_name with embedded + # uppercase (atypical) gets normalised. + self.assertEqual( + _synthesised_long_name_from_std('Foo_Bar'), + 'Foo bar', + ) + + +class TestCollectAutoCloneEntriesSkips(_AutoCloneFixture): + """``_collect_auto_clone_entries`` must skip framework-named std + names (``ccpp_constituents``, ``ccpp_constituent_tendencies``, + ``ccpp_constituent_properties``, ``number_of_ccpp_constituents``, + ``index_of_*``) — those resolve through ``source='constituent'`` + too but reference the framework-provided buffers, not individual + species, so synthesising a ``%instantiate`` for them would + register bogus duplicate constituents (e.g. an entry named + ``ccpp_constituents`` in the dynamic-constituents buffer). + """ + + def _resolved_arg(self, std_name): + from generator.suite_resolver import ResolvedArg + return ResolvedArg( + standard_name=std_name, + scheme_local_name='dummy', + intent='inout', + is_optional=False, + active='', active_local='', + source='constituent', + host_entry=None, suite_var=None, + base_expr='', subscript='', call_expr='', + used_dim_std_names=set(), + needs_unit_transform=False, + needs_kind_transform=False, + unit_forward='', unit_backward='', + kind_scheme='', kind_host='', + temp_name='', ptr_name='', + transform_case=1, + scheme_dimensions=[], + ) + + def _resolved_groups_with(self, std_names): + from generator.suite_resolver import ( + ResolvedCall, ResolvedGroup, + ) + call = ResolvedCall( + scheme_name='dummy_scheme', phase='run', + args=[self._resolved_arg(s) for s in std_names], + ) + group = ResolvedGroup(group_name='g') + group.phase_calls['run'] = [call] + return [group] + + def test_framework_array_names_skipped(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + groups = self._resolved_groups_with([ + 'ccpp_constituents', + 'ccpp_constituent_tendencies', + 'ccpp_constituent_properties', + 'number_of_ccpp_constituents', + ]) + # No matching scheme metadata exists, but the framework-name + # filter must fire BEFORE the scheme-var lookup so this never + # gets that far. Returns [] without raising. + entries = _collect_auto_clone_entries(groups, scheme_store=None) + self.assertEqual(entries, []) + + def test_index_of_names_skipped(self): + auto_clone_constituents.enable(_stream=io.StringIO()) + groups = self._resolved_groups_with([ + 'index_of_water_vapor_specific_humidity', + 'index_of_cloud_liquid_dry_mixing_ratio', + ]) + entries = _collect_auto_clone_entries(groups, scheme_store=None) + self.assertEqual(entries, []) + + +######################################################################## +# Integration: full parse + resolver pass on the auto-clone fixture +######################################################################## + +class TestFixtureParse(_AutoCloneFixture): + """The sample fixture parses cleanly with the shim on and carries + every legacy attr on the right scheme args.""" + + def setUp(self): + super().setUp() + auto_clone_constituents.enable(_stream=io.StringIO()) + + def test_fixture_parses(self): + tables = parse_metadata_file(_sf('scheme_auto_clone_consumer.meta')) + self.assertEqual(len(tables), 1) + section = tables[0].sections()[0] + by_name = {v.local_name: v for v in section.variables} + self.assertIn('qv', by_name) + self.assertIn('qc', by_name) + # qv carries the full legacy attr set. + qv = by_name['qv'] + self.assertTrue(qv.is_constituent) + self.assertTrue(qv.advected) + self.assertEqual(qv.default_value, 1.0e-12) + self.assertEqual(qv.min_value, 0.0) + self.assertIs(qv.water_species, True) + self.assertEqual(qv.mixing_ratio_type, 'wrt_moist') + # qc only sets default_value. + qc = by_name['qc'] + self.assertEqual(qc.default_value, 0.0) + self.assertIsNone(qc.min_value) + self.assertIsNone(qc.water_species) + self.assertIsNone(qc.mixing_ratio_type) + + def test_fixture_rejected_without_shim(self): + # Drop the flag; the same .meta file now fails. + auto_clone_constituents.disable() + with self.assertRaises(CCPPError) as cm: + parse_metadata_file(_sf('scheme_auto_clone_consumer.meta')) + # The first rejected attr is ``default_value`` (qv's first + # legacy attr in declaration order). Any of the four would + # be acceptable; check the generic "Unknown variable + # attribute" wording. + self.assertIn('Unknown variable attribute', str(cm.exception)) + + + +######################################################################## +# Doctest loader +######################################################################## + +def load_tests(loader, tests, ignore): + tests.addTests(doctest.DocTestSuite(auto_clone_constituents)) + return tests + + +if __name__ == '__main__': + unittest.main(verbosity=2) From 43c17613e3b03041273840a1775502c3c27485fa Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 1 Jun 2026 08:59:23 -0600 Subject: [PATCH 43/74] Update docs/* --- doc/briefing.md | 70 ++++++++++++----- doc/briefing_pm.md | 94 ++++++++++++++-------- doc/constituents.md | 17 ++-- doc/constituents_overhaul.md | 26 +++++-- doc/migration.md | 147 ++++++++++++++++++++++++++++++++--- doc/redesign_prompt.md | 106 +++++++++++++++++++++++-- 6 files changed, 383 insertions(+), 77 deletions(-) diff --git a/doc/briefing.md b/doc/briefing.md index 39f9984c..62132c8d 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -1,8 +1,8 @@ # capgen-ng — Briefing for CCPP Framework Developers & Power Users -*Prepared for the 2026-05-14 walk-through. Companion document to -`doc/migration.md` (the detailed migration guide) and -`doc/redesign_prompt.md` (the implementation spec).* +*Prepared for the 2026-05-14 walk-through; last revised 2026-06-01. +Companion document to `doc/migration.md` (the detailed migration +guide) and `doc/redesign_prompt.md` (the implementation spec).* --- @@ -143,7 +143,7 @@ Both share the same metadata-parsing library (`metadata/`). | Variable matching algorithm | Five-layer scope-chain promotion | Flat host+control dict + suite-owned discovery (inherited from prebuild — primary reason runtime is comparable to prebuild) | | External types (MPI f08 comm, ESMF clock) | Tabled (solution complexity) | First-class via `type = external::` | | `type = module` in metadata | Yes | Renamed `type = host` | -| `is_constituent` scheme args | Auto-cloned by generator | Schemes register constituents explicitly in the `register` phase via `ccpp_constituent_properties_t(:)` | +| `is_constituent` scheme args | Auto-cloned by generator | Schemes register constituents explicitly in the `register` phase via `ccpp_constituent_properties_t(:)`; original capgen's auto-clone path is available behind the opt-in `--legacy-auto-clone-constituents` shim for legacy hosts (single-instance only) | | `ConstituentVarDict` | Synthetic scope between suite + host | Removed; constituents are one of four sources (`control`/`host`/`suite`/`constituent`) on `ResolvedArg` | | `_state` runtime check | String | Integer (named parameters) | | Fortran-vs-metadata check | Inside the generator | Separate tool (`ccpp_validator.py`) | @@ -177,6 +177,23 @@ Both are rewritten on the fly by **`--legacy-mode`** for a transition period; the shim prints a banner listing every rewrite it performs and is marked for clean removal. +### 6.3b Transient migration shims — full set + +Three opt-in CLI flags exist for migrating legacy hosts. Each lives +in its own self-contained module, prints a loud banner at startup, and +every touchpoint is grep-tagged for clean removal: + +| Flag | What it does | Removal grep | +|---------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| +| `--legacy-mode` | Parse-time substitution of two deprecated CCPP standard names (see §6.3 above). Active on both `ccpp_capgen_ng.py` and `ccpp_validator.py`. | `legacy-compat` | +| `--gfs-dim-aliases` | Treats GFS-physics names `adjusted_vertical_layer_dimension_for_radiation` and `vertical_composition_dimension` as equivalent to `vertical_layer_dimension` **inside the upper-bound dim identity check only** (the variables themselves stay distinct). Resolver-only, so `ccpp_capgen_ng.py` carries the flag; `ccpp_validator.py` does not (the validator never reaches the dim canonicaliser). | `dim-aliases` | +| `--legacy-auto-clone-constituents` | Reinstates original ccpp-capgen's auto-clone-static-constituent registration path: every `is_constituent` consumer (`advected = True` / `constituent = True` / `molar_mass = …`) with no register-phase source is auto-registered into the per-suite dynamic-constituents buffer using values lifted straight from the scheme metadata. Adds four legacy `%instantiate` kwargs to the parser (`default_value`, `min_value`, `water_species`, `mixing_ratio_type`). **Single-instance only** — declaring the `instance_number` + `number_of_instances` pair while the flag is on is a hard error. Available on both `ccpp_capgen_ng.py` and `ccpp_validator.py`. | `auto-clone-constituents` | + +All three flags are listed as runways, not destinations: drop the +underlying legacy spelling from host/scheme metadata and the flag can +be retired. See `doc/auto_clone_constituents.md` for the full +auto-clone reference. + ### 6.4 Required host `type = control` table Every host MUST declare scalar integers (and one character) with @@ -339,19 +356,32 @@ don't rebuild downstream objects unless something actually moved. ## 10. Where things stand right now -- **Unit tests**: 1353 passing on `feature/capgen-ng` (1365 with - doctests; as of 2026-05-20). -- **End-to-end tests passing**: `advection`, `unit_conv`, - `nested_suite`, `variable_transform`, `instances`, - `instances_advection`, `ddt`. +- **Unit tests**: 1426 passing on `feature/capgen-ng` (1438 with + doctests; as of 2026-06-01). +- **End-to-end tests passing** (10): `advection`, + `advection_auto_clone`, `capgen_ng`, `chunked_data`, `ddthost`, + `instances`, `instances_advection`, `nested_suite`, `opt_arg`, + `var_compat`. `advection_auto_clone` is the newest — a port of + CAM-SIMA's `advection_test` exercising the auto-clone legacy + registration path under `--legacy-auto-clone-constituents`. +- **Code size**: ~17.8k LOC of Python under `capgen-ng/` (includes + docstrings, inline comments, and the three transient shim modules) + + ~18k LOC of unit/doctest under `unit-tests/`. Still procedural, + still flat data classes. +- **Three transient migration shims now live** (see §6.3b): + `--legacy-mode` (2026-05-13), `--gfs-dim-aliases` (2026-05-21), + and `--legacy-auto-clone-constituents` (2026-05-21). Each is + isolated in its own module + grep-tag so removal is a single + cleanup pass once the underlying legacy spelling is gone from + host/scheme metadata. - **CCPP-SCM**: actively driving development — every build / runtime - failure surfaced this week landed as a fix in capgen-ng (rather than - being patched around in the host). Most of the `phys_ps` group now - builds end-to-end via `--legacy-mode`. On 2026-05-20 the - per-arg-attribute validator caught **67 real metadata/Fortran - disagreements** in the SCM physics tree (12 missing `kind = kind_phys` - + 42 intent mismatches + a mix of optional-flag and bare-`real` - cases); all fixed. + failure surfaced this month landed as a fix in capgen-ng (rather + than being patched around in the host). Most of the `phys_ps` group + now builds end-to-end via `--legacy-mode` + `--gfs-dim-aliases`. + On 2026-05-20 the per-arg-attribute validator caught **67 real + metadata/Fortran disagreements** in the SCM physics tree (12 missing + `kind = kind_phys` + 42 intent mismatches + a mix of optional-flag + and bare-`real` cases); all fixed. - **Validator** now checks per-argument `intent`, `type`, `kind`, and dimension rank in addition to the original name/count check. Asymmetric `optional` rule, DDT + `external::` @@ -368,7 +398,7 @@ don't rebuild downstream objects unless something actually moved. (`if (.not. (active)) errflg = 1; return`) before the call. Replaces an earlier static rule that forced scheme metadata to lie about optionality. See `doc/migration.md` §1.3.1. -- **`--no-host-introspection`** (new, 2026-05-14): stubs the bodies of +- **`--no-host-introspection`** (2026-05-14): stubs the bodies of the five suite-introspection routines in `_ccpp_cap.F90`, shrinking the file from ~33k lines to ~800 for the 10-suite SCM build (the introspection case-blocks were making even `-O1` @@ -383,7 +413,11 @@ don't rebuild downstream objects unless something actually moved. ground first. An anticipated complication is the "fast physics" called directly from the FV3 dynamical core as a separate group. - **CAM-SIMA**: not yet reconnected; pending the constituents - overhaul decision and CAM-SIMA developer availability. + overhaul decision and CAM-SIMA developer availability. Note that + `--legacy-auto-clone-constituents` is the no-decision-needed bridge + that lets capgen-ng accept CAM-SIMA atmospheric_physics metadata + as-is — ~16 of the ~20 schemes that touch constituents rely on the + auto-clone path today. --- diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md index bd59f75b..86a74b59 100644 --- a/doc/briefing_pm.md +++ b/doc/briefing_pm.md @@ -7,7 +7,7 @@ program managers; it summarises the case for `capgen-ng` in terms of product risk, schedule, and cross-organization impact rather than implementation detail.* -*Last revised: 2026-05-18.* +*Last revised: 2026-06-01.* --- @@ -60,8 +60,9 @@ limits that make it impractical for UFS, NEPTUNE, or multi-instance hosts, and the implementation is concentrated enough that few people can extend it safely (if at all - primary developer gone). -**`capgen-ng`** (new, 2026-05). Procedural Python (a few thousand -lines; flat data classes); reads the same metadata format; passes +**`capgen-ng`** (new, 2026-05). Procedural Python (~17.8k lines +including inline comments and the three transient shim modules; flat +data classes); reads the same metadata format; passes arguments like prebuild; supports the features capgen pioneered (constituents, suite-owned variables, introspection); supports multi-instance, an integer state machine, six explicit scheme @@ -161,7 +162,16 @@ read, harder to port between hosts, and harder to debug when registrations collide. `capgen-ng` keeps only the first two (explicit) paths. Auto-clone -is deliberately gone — see `doc/constituents_overhaul.md` §2.3. +is deliberately gone from the default behaviour — see +`doc/constituents_overhaul.md` §2.3. For legacy hosts that already +ship metadata in the original-capgen shape (production CAM-SIMA's +atmospheric_physics tree is the immediate consumer; ~16 of the ~20 +schemes that touch constituents rely on auto-clone today), an opt-in +shim `--legacy-auto-clone-constituents` reinstates the original path +behind a single CLI flag with a loud startup banner; see +`doc/auto_clone_constituents.md`. The shim is single-instance only +and is marked for removal once consumers migrate to explicit +registration. ### 3.4 Host-specific values baked into scheme metadata @@ -201,16 +211,19 @@ capgen is roughly an order of magnitude larger than prebuild, with a deeply layered class hierarchy. This is not a moral failing — it reflects the feature set — but the practical consequence is that the maintenance burden falls on a small subset of the framework -team. capgen-ng is comparable to prebuild in size (a few thousand -lines of mostly procedural Python with small data classes), so the -"who can fix this" pool is closer to "anyone with framework -context". capgen-ng comes with over a thousand docstring tests -and unit tests, as well as a comprehensive end-to-end test suite -that covers all of prebuild's and capgen's existing end to end -tests. capgen-ng adds additional end-to-end tests for new -features such as the multi-instance constituents test. Including -these tests and the rich inline comments makes capgen-ng comparable -in size to capgen. +team. capgen-ng is comparable to prebuild in *shape* (procedural +Python with small data classes — no deep class hierarchy), and the +generator itself sits at ~17.8k lines (capgen is several times +larger). The "who can fix this" pool is closer to "anyone with +framework context". capgen-ng comes with ~1.4k docstring + unit +tests (~18k lines of test code), plus an end-to-end test suite of +10 fixtures that covers all of prebuild's and capgen's existing +end-to-end tests and adds new ones for multi-instance + constituents +(`instances_advection`) and the auto-clone-constituents shim +(`advection_auto_clone`). Including these tests and the rich inline +comments puts capgen-ng's full tree on the same order of magnitude as +capgen — almost all of which is test coverage and human-readable +prose, not load-bearing logic. --- @@ -229,7 +242,7 @@ even to CAM-SIMA-shape problems: | Generator code style | Deep class hierarchy | Flat data classes + procedural resolver | | Error reporting | Variable amount of context | "Loud, specific, actionable" enforced — every parse-time error names file, line, variable, attribute, value, and reason | | Constituent registration | Three sources (one invisible) | Two sources, both explicit | -| `is_constituent` auto-clone | Yes (host-specific values baked into scheme metadata) | Removed | +| `is_constituent` auto-clone | Yes (host-specific values baked into scheme metadata) | Removed by default; reinstated for legacy hosts behind opt-in `--legacy-auto-clone-constituents` shim (single-instance only) | | `_finalize` vs `_final` phase name | `_finalize` | `_final` (renamed to keep symmetry with init/timestep_init/timestep_final) | --- @@ -244,37 +257,56 @@ Features that exist only in capgen-ng (some exist in prebuild): | **Registered scalar-index dimensions** | When metadata says a variable is dimensioned by `number_of_threads` or `number_of_instances`, capgen-ng injects the right per-call subscript automatically; the host's OpenMP-thread-private DDT layout works unchanged | | **Subcycle loop-counter automation** | Schemes inside a `` element can access `ccpp_loop_counter` / `ccpp_loop_extent` directly; the generator emits the Fortran `do` loop and binds the locals | | **`--legacy-mode` migration shim** | One CLI flag enables silent rewrite of two known-good deprecated standard names (`horizontal_loop_extent` → `horizontal_dimension`, `number_of_openmp_threads` → `number_of_threads`) with a loud warning — buys time for host metadata to migrate | +| **`--gfs-dim-aliases` migration shim** (2026-05-21) | One CLI flag treats GFS-physics names (`adjusted_vertical_layer_dimension_for_radiation`, `vertical_composition_dimension`) as equivalent to `vertical_layer_dimension` in the dim-identity check only — variables remain distinct everywhere else. Resolver-only; clean grep-revert. Required for CCPP-SCM 17p8 to build under capgen-ng. | +| **`--legacy-auto-clone-constituents` migration shim** (2026-05-21) | One CLI flag reinstates original ccpp-capgen's auto-clone-static-constituent registration path for the ~16 production-CAM-SIMA schemes that depend on it. Single-instance only (predates multi-instance); fails fast if a multi-instance host is supplied. This is the no-decision-needed bridge that lets capgen-ng accept CAM-SIMA's atmospheric_physics metadata before any constituent-overhaul work lands. | | **`--no-host-introspection` flag** | The five runtime introspection routines (`ccpp_physics_suite_list`, etc.) emit large `select case` blocks at SCM scale; this flag stubs the bodies, dropping the generated static API from ~33,000 lines to ~800 for the SCM build (the introspection routines were making `-O1` compilation effectively hang) | | **Consistent handling of external types** (MPI f08 communicator, ESMF clock) | Tabled in capgen because of the complexity of the solution | --- -## 6. Where things stand right now (2026-05-20) - -- **Unit tests**: 1353 passing (1365 with doctests). No known failures. -- **End-to-end tests**: 10 passing — `chunked_data`, `opt_arg`, - `nested_suite`, `ddthost`, `instances`, `capgen_ng`, - `var_compat`, `advection`, and the new `instances_advection` - (multi-instance + constituents). +## 6. Where things stand right now (2026-06-01) + +- **Unit tests**: 1426 passing (1438 with doctests). No known failures. +- **End-to-end tests**: 10 passing — `advection`, + `advection_auto_clone` (new 2026-05-21; CAM-SIMA advection_test + port exercising the auto-clone shim), `capgen_ng`, `chunked_data`, + `ddthost`, `instances`, `instances_advection` + (multi-instance + constituents), `nested_suite`, `opt_arg`, + `var_compat`. +- **Code size**: ~17.8k lines of Python under `capgen-ng/` including + inline comments and the three transient shim modules; ~18k lines of + unit/doctest under `unit-tests/`. Still procedural; still flat + data classes; still well below capgen. - **CCPP-SCM**: actively driving development. Each build / runtime - issue surfaced this week landed as a fix in capgen-ng rather than + issue surfaced this month landed as a fix in capgen-ng rather than a host-side workaround. All available suites in CCPP-SCM now - build and run end-to-end via `--legacy-mode`. + build and run end-to-end via `--legacy-mode` + `--gfs-dim-aliases`. +- **Three transient migration shims in place** (see §5). Each is + isolated in its own module with a single grep tag, so removal once + hosts migrate is a single cleanup pass. +- **Auto-clone shim landed 2026-05-21**. Reinstates original capgen's + auto-clone path behind `--legacy-auto-clone-constituents`. This is + the no-decision-needed bridge for CAM-SIMA — the ~16 schemes that + declare `advected = True` in `_run` arg-tables and rely on the + framework to register the constituent will now work under capgen-ng + without metadata edits. - **Multi-instance + constituents fix landed 2026-05-18**. The new combined end-to-end test surfaced a latent shared-buffer mutation bug; the fix moves the per-suite dynamic-constituents buffer per-instance. No coordination with CAM-SIMA / UFS / NEPTUNE required (host-facing API unchanged). -- **NEPTUNE**: cleanup and acceptance testing in progress this week. +- **NEPTUNE**: cleanup and acceptance testing in progress. Regular/lower atmosphere physics builds and runs, and produces results within the tolerance (i.e. similar to compiler changes). High altitude atmosphere testing is next. - **UFS Weather Model**: not yet attempted; SCM is the proving ground first. Expecting updates due to the "fast physics" called directly from the FV3 dynamical core as separate group. -- **CAM-SIMA**: not yet re-connected; the constituent overhaul - decision (see §7) and the availability of CAM-SIMA developers - are the gating items. +- **CAM-SIMA**: not yet re-connected; the availability of CAM-SIMA + developers is now the primary gating item. The auto-clone shim + removes the metadata-edit blocker; the constituent overhaul + decision (see §7) is no longer on the critical path for getting + CAM-SIMA built. --- @@ -312,9 +344,9 @@ proposals are implementable on top of it. | Risk | Status | Mitigation | |---|---|---| | capgen-ng diverges from capgen feature set | LOW | Cross-checked by `doc/redesign_analysis.md`; the feature comparison table in §4 / §5 is exhaustive | -| Host metadata break for UFS / NEPTUNE / CAM-SIMA | MEDIUM | `--legacy-mode` shim covers the known incompatible standard-name pair; remaining required changes (e.g., `_finalize` → `_final`) are mechanical and listed in `doc/migration.md` §3 | -| Constituent overhaul stalls | MEDIUM | Proposal A unblocks the immediate bug; capgen-ng works with the current framework today; the overhaul is a separate decision track | -| Bus-factor on capgen-ng itself | MEDIUM | Procedural code style + flat data classes + 1319-test safety net; significantly lower than capgen's bus factor | +| Host metadata break for UFS / NEPTUNE / CAM-SIMA | LOW | Three transient shims (`--legacy-mode`, `--gfs-dim-aliases`, `--legacy-auto-clone-constituents`) together cover the known-incompatible standard-name pair, the GFS radiation/composition vertical-dim spellings, and original capgen's auto-clone registration path. Remaining required changes (e.g., `_finalize` → `_final`) are mechanical and listed in `doc/migration.md` §3 | +| Constituent overhaul stalls | LOW | Proposal A unblocks the immediate bug; capgen-ng works with the current framework today; `--legacy-auto-clone-constituents` lets CAM-SIMA's atmospheric_physics build without an overhaul decision; the overhaul is a separate decision track | +| Bus-factor on capgen-ng itself | MEDIUM | Procedural code style + flat data classes + 1426-test safety net; significantly lower than capgen's bus factor | | Two host call-shape conventions (prebuild-style vs capgen-style) coexist forever | LOW | capgen-ng emits one shape; downstream host conversions are tracked in `doc/migration.md` | | Regression discovered during NEPTUNE / UFS testing | EXPECTED | SCM proving ground catches most; remaining issues become capgen-ng tickets, not host-side patches | | ccpp-prebuild end-of-life requires a sunset plan | OPEN | Not yet scoped; both generators currently coexist in the framework repo | diff --git a/doc/constituents.md b/doc/constituents.md index 65f906eb..0dc6e46f 100644 --- a/doc/constituents.md +++ b/doc/constituents.md @@ -852,15 +852,20 @@ message naming the offending token. | Module-level pointers | `_constituents_array` etc. as functions returning pointers | Same idea, now per-instance via `instance_number` arg | | Scheme std-name set | `_model_const_stdnames` parameter array | `ccpp_model_const_stdnames` parameter array (no host prefix) | | Host-facing API surface | `_ccpp_register_constituents`, `_ccpp_initialize_constituents`, `_ccpp_number_constituents`, `_ccpp_is_scheme_constituent`, `_ccpp_gather_constituents`, `_ccpp_update_constituents`, `_ccpp_deallocate_dynamic_constituents`, `_constituents_array`, `_advected_constituents_array`, `_model_const_properties`, `_const_get_index` | Same surface, no `_` prefix | -| Dynamic constituent buffer dimensionality | 1D, per host | 1D, per generator run, **shared across instances** | -| Static suite constituents | Auto-cloned by `ConstituentVarDict.find_variable` and registered via `_constituents_copy_const` accessors | Tracked at code-gen time via `is_constituent` flag; included in the constituent table only if a register-phase scheme produces them (rule 1). Schemes that *consume* a base constituent (rule 2) don't trigger registration — the constituent must be registered by SOMEONE (host or another scheme's register) for the access to work at runtime. | +| Dynamic constituent buffer dimensionality | 1D, per host | 1D **per instance** (wrapper-DDT array indexed by `instance_number`; was shared across instances pre-2026-05-18, until the combined multi-instance + constituents e2e test surfaced a latent set_const_index conflict) | +| Static suite constituents | Auto-cloned by `ConstituentVarDict.find_variable` and registered via `_constituents_copy_const` accessors | Default behaviour: tracked at code-gen time via `is_constituent` flag; included in the constituent table only if a register-phase scheme produces them (rule 1). Schemes that *consume* a base constituent (rule 2) don't trigger registration — the constituent must be registered by SOMEONE (host or another scheme's register) for the access to work at runtime. **Opt-in shim** `--legacy-auto-clone-constituents` (2026-05-21, transient) reinstates original capgen's auto-clone path for legacy hosts; single-instance only. See `doc/auto_clone_constituents.md`. | ### Migration notes for cam-sima hosts -- **Scheme metadata**: no changes needed. Cam-sima follows rules 2 and - 3 already (audited 2026-05-11). The 4 schemes that register - constituents via `ccpp_constituent_properties_t` (rule 1) work - unchanged. +- **Scheme metadata**: no changes needed for the 4 schemes that + register constituents via `ccpp_constituent_properties_t` (rule 1) — + those work unchanged. For the ~16 schemes that rely on original + capgen's auto-clone path (`advected = True` on a `_run` arg with no + matching register-phase source), pass + `--legacy-auto-clone-constituents` to `ccpp_capgen_ng.py` and + `ccpp_validator.py` — capgen-ng then auto-registers those + constituents into the per-suite dynamic-constituents buffer the same + way original capgen did. See `doc/auto_clone_constituents.md`. - **Host metadata**: drop any explicit declaration of `ccpp_model_constituents_object` if you carried one over from a previous capgen-ng experiment — the generator owns it now. diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index 4ec40b4b..a709364c 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -195,10 +195,23 @@ The resolver classifies each scheme arg into exactly one source. A drained into `ccpp_model_constituents_obj(inst)` by `ccpp_register_constituents`. -The auto-clone-from-metadata path is deliberately **gone**. If a scheme -declares `advected=true` on an arg but no source registers that -standard name, capgen-ng now emits a runtime check during -`ccpp_initialize_constituents` that errors with the missing name. +The auto-clone-from-metadata path is **gone from capgen-ng's default +behaviour**. If a scheme declares `advected=true` on an arg but no +source registers that standard name, capgen-ng emits a runtime check +during `ccpp_initialize_constituents` that errors with the missing +name. + +**Legacy escape hatch** (added 2026-05-21): the opt-in CLI flag +`--legacy-auto-clone-constituents` reinstates the original +auto-clone path for hosts whose scheme metadata predates explicit +registration (production CAM-SIMA's atmospheric_physics tree is the +immediate consumer). This is a transient migration shim — see +`doc/auto_clone_constituents.md` for the full reference and +removal procedure. It is single-instance only and explicitly +flagged so future capgen-ng work is *not* expected to keep it +indefinitely. The reform proposals in §6–§8 below are unchanged by +the shim's existence: capgen-ng's chosen architecture is still +explicit registration. ### 2.4 Per-instance state @@ -600,8 +613,9 @@ shim. Remove the rewrite once known consumers are migrated. - **Cost**: ~50 lines across the two generator emitters, plus updates to six pinned unit tests. No CAM-SIMA / NEPTUNE / SCM coordination needed (host-facing API unchanged). -- **Status**: framework tests pass; full unit-test suite (1319 tests) - is green; all 10 end-to-end tests pass. +- **Status**: framework tests pass; full unit-test suite + (1319 tests at fix landing, 1426 as of 2026-06-01) is green; all 10 + end-to-end tests pass. - **Position relative to Proposals A/B/C**: orthogonal — none of the three proposed touching the buffer. Independently adopted. diff --git a/doc/migration.md b/doc/migration.md index 6aa35548..c9333e43 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -6,7 +6,7 @@ Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to **capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and `doc/redesign_analysis.md` (analysis of the old systems). -*Last revised: 2026-05-20 (end-of-day).* Current unit-test suite: 1353 passing (1365 with doctests). +*Last revised: 2026-06-01.* Current unit-test suite: 1426 passing (1438 with doctests). **Repository layout** (post-2026-05-13 cleanup): tooling lives under `capgen-ng/` (top-level of this repo). Unit tests live at the top @@ -16,13 +16,15 @@ the unit suite from the repo root with `python -m pytest unit-tests/`. ## Table of contents 1. [Metadata format changes](#1-metadata-format-changes) - 1. [1.8 `horizontal_loop_extent` → `horizontal_dimension`](#18-horizontal_loop_extent--horizontal_dimension) + 1. [1.8 `horizontal_loop_extent` → `horizontal_dimension`](#18-deprecated-standard-names-rewritten-by---legacy-mode) + 2. [1.10 GFS-physics vertical-dim aliases (`--gfs-dim-aliases`)](#110-gfs-physics-vertical-dim-aliases---gfs-dim-aliases) 2. [Suite definition file (SDF) changes](#2-suite-definition-file-sdf-changes) 3. [Host Fortran requirements](#3-host-fortran-requirements) 4. [Generator CLI and build integration](#4-generator-cli-and-build-integration) 5. [Generated cap layout — what's new and what changed](#5-generated-cap-layout--whats-new-and-what-changed) 6. [Framework changes (constituents)](#6-framework-changes-constituents) 1. [6.3 Host metadata wins over auto-provisioning](#63-host-metadata-wins-over-auto-provisioning-2026-05-12) + 2. [6.4 Legacy auto-clone registration (`--legacy-auto-clone-constituents`)](#64-legacy-auto-clone-registration---legacy-auto-clone-constituents) 7. [Validator (`ccpp_validator.py`)](#7-validator) 8. [Known gaps and deferred items](#8-known-gaps-and-deferred-items) @@ -294,6 +296,40 @@ the historic blank-line convention, but is *not* treated as an inline comment marker (`;` can plausibly appear inside a `long_name`). +### 1.10 GFS-physics vertical-dim aliases (`--gfs-dim-aliases`) + +GFS-physics scheme metadata uses two spellings for what is physically +the vertical-layer axis: + +- `adjusted_vertical_layer_dimension_for_radiation` (radiation schemes) +- `vertical_composition_dimension` (chemistry schemes) + +Both are the **same axis** as `vertical_layer_dimension` from the +host's point of view, but legacy hosts (CCPP-SCM 17p8, GFS) carry the +three names as **distinct host variables** (each addressable as its +own scalar dim std name) — so a parse-time substitution like +`--legacy-mode` would erase the variable behind the renamed token and +break host metadata. + +`--gfs-dim-aliases` collapses the three names **only inside the +resolver's per-position dimension-identity check** (upper bound only; +lower bounds never alias). The variables themselves stay distinct +everywhere else — `[ adjusted_vertical_layer_dimension_for_radiation ]` +remains its own host `type=control` entry; the access path in +generated code is unchanged; only the resolver's +"these dims describe the same axis" comparison treats the three names +as equivalent. + +Single touchpoint in the generator +(`generator/suite_resolver.py::_canonical_dim`); the validator does +not carry the flag (it never reaches the resolver's canonicaliser). +Self-contained module `metadata/dim_aliases.py`; every touchpoint +tagged `# dim-aliases:` for clean removal. + +Like `--legacy-mode`, this is a transient migration shim with a loud +startup banner — drop the GFS spellings from your scheme metadata in +favour of `vertical_layer_dimension` and the flag becomes unnecessary. + --- ## 2. Suite definition file (SDF) changes @@ -563,6 +599,9 @@ python ccpp_capgen_ng.py \ --output-root /ccpp \ [--kind-type =[:]] \ [--legacy-mode] \ + [--gfs-dim-aliases] \ + [--legacy-auto-clone-constituents] \ + [--no-host-introspection] \ [--verbose] [--verbose] ``` @@ -571,10 +610,15 @@ omitted, `` must be an ISO_FORTRAN_ENV constant (REAL32/REAL64/...) and the module defaults to `iso_fortran_env`. `kind_phys` is auto-defaulted to `iso_fortran_env:REAL64` when not supplied. -`--legacy-mode` (transient migration shim, will be removed): silently -rewrites a small set of deprecated CCPP standard names to their -capgen-ng equivalents at parse time — see §1.8 for the full table -(`horizontal_loop_extent` → `horizontal_dimension`, +#### Transient migration shims + +Three opt-in flags exist for migrating legacy hosts. Each is +self-contained and grep-tagged for clean removal: + +**`--legacy-mode`** (transient migration shim, will be removed): +silently rewrites a small set of deprecated CCPP standard names to +their capgen-ng equivalents at parse time — see §1.8 for the full +table (`horizontal_loop_extent` → `horizontal_dimension`, `number_of_openmp_threads` → `number_of_threads`). The rewrite fires for both standard-name attributes AND dimension tokens. Prints a loud warning banner at startup, enumerating every pair the shim is @@ -582,8 +626,36 @@ rewriting, so the substitution is never invisible. Available on both `ccpp_capgen_ng.py` and `ccpp_validator.py` (keep the flag consistent between the two when both are invoked from CMake). All translation logic is isolated in `metadata/legacy_compat.py` and tagged with -`# legacy-compat:` comments at every touchpoint, so the shim can be -cleanly removed when migration is complete. +`# legacy-compat:` comments at every touchpoint. + +**`--gfs-dim-aliases`** (transient migration shim, see §1.10): +treats GFS-physics names +`adjusted_vertical_layer_dimension_for_radiation` and +`vertical_composition_dimension` as equivalent to +`vertical_layer_dimension` **inside the resolver's per-position +dim-identity check only** (upper bound only). The host variables +themselves stay distinct. Generator-only (the validator never +reaches the dim canonicaliser). Module `metadata/dim_aliases.py`; +touchpoints tagged `# dim-aliases:`. + +**`--legacy-auto-clone-constituents`** (transient migration shim, see +`doc/auto_clone_constituents.md` for the full reference): +reinstates original ccpp-capgen's auto-clone-static-constituent +registration path. Every `is_constituent` consumer scheme arg +(`advected = True`, `constituent = True`, or `molar_mass = …`) with +no register-phase source is auto-registered into the per-suite +dynamic-constituents buffer using values lifted straight from the +scheme metadata (with sensible defaults: `long_name` synthesised from +the standard name when missing, `diag_name` falls back to local_name, +`vertical_dim` lifted from the arg's dim list). Adds four legacy +`%instantiate` kwargs to the parser (`default_value`, `min_value`, +`water_species`, `mixing_ratio_type`). Available on both +`ccpp_capgen_ng.py` and `ccpp_validator.py` (the validator must +accept the four extra attrs). **Single-instance only** — declaring +the `instance_number` + `number_of_instances` pair while the flag is +on is a hard error before any suite is parsed. Module +`metadata/auto_clone_constituents.py`; touchpoints tagged +`# auto-clone-constituents:`. ### 4.2 `ccpp_datafile.py` query CLI @@ -759,6 +831,10 @@ state array is unallocated — there, "not allocated" really does mean Backward-compatible. Original capgen's auto-clone path in `scripts/constituents.py` has been updated to call the setter. +capgen-ng's `--legacy-auto-clone-constituents` shim (§6.4) +synthesises `%instantiate(...)` directly on slots of the per-suite +dynamic-constituents buffer, so the properties objects are owned by +the buffer from creation — no ownership transfer call needed. ### 6.2 capgen-ng constituent API @@ -801,6 +877,58 @@ Active design review for the next constituents iteration: `doc/constituents_overhaul.md` (Class A vs Class B property classification, three reform proposals). +### 6.4 Legacy auto-clone registration (`--legacy-auto-clone-constituents`) + +For hosts that ship metadata in original ccpp-capgen's shape — most +notably CAM-SIMA's atmospheric_physics tree, where ~16 of the ~20 +constituent-touching schemes declare `advected = True` (or +`constituent = True`, or `molar_mass = …`) in `_run` arg tables and +rely on the framework to register the constituent — pass +`--legacy-auto-clone-constituents` to both `ccpp_capgen_ng.py` and +`ccpp_validator.py`. + +What changes: + +- The parser accepts four extra scheme-arg attributes + (`default_value`, `min_value`, `water_species`, + `mixing_ratio_type`). Fortran-style literal suffixes + (`0.0_kind_phys`, `1.0d-5`, `-3.14_8`) are accepted on the real + fields, since legacy metadata writes the values in source form. +- For every unique standard name that appears as an `is_constituent` + consumer with no register-phase source, the suite cap emits a + synthesised `%instantiate(...)` call into the per-suite + dynamic-constituents buffer. The scheme author writes no Fortran + registration code. +- `long_name` is auto-synthesised from the standard name when missing + (`cloud_liquid_dry_mixing_ratio` → `'Cloud liquid dry mixing + ratio'`); `diag_name` falls back to local_name; `vertical_dim` is + lifted from the arg's `dimensions = (...)` entry. +- Schemes that pass the whole constituents buffer (e.g. + `apply_constituent_tendencies_run` with `ccpp_constituents` / + `ccpp_constituent_tendencies` / `index_of_*` args) are excluded + from auto-clone — those resolve through the framework + whole-buffer path, not as individual registrations. + +What capgen-ng's other rules still require (the shim does **not** +relax them): + +- `intent = inout` on base constituents (`advected = True` on a + non-`tendency_of_*` std_name). `intent = out` is reserved for + tendency args. +- Metadata arg tables must match the Fortran subroutine signature. + Declaring a constituent in `_init`'s arg table when + `_init` doesn't take it as a Fortran dummy is rejected by + the validator. + +Single-instance only: declaring `instance_number` + +`number_of_instances` while the flag is on aborts before any suite is +parsed. Legacy hosts predate multi-instance support, so this matches +the use case. + +Full reference: `doc/auto_clone_constituents.md`. E2e fixture: +`end-to-end-tests/advection_auto_clone/` (a port of CAM-SIMA's +`advection_test`). + --- ## 7. Validator @@ -886,8 +1014,9 @@ complete). See `project_validator_host_check_deferred.md` (memory). | Generated Fortran ↔ Codee formatter idempotency | Deferred; emitted `.F90` must round-trip cleanly through the project's Codee Fortran formatter. | | `fortran_to_metadata` developer utility | Deferred; bootstraps a `.meta` skeleton from an existing `.F90` subroutine. | | `--legacy-mode` shim removal | Transient; remove `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and every `# legacy-compat:` touchpoint when scheme metadata has migrated. | +| `--gfs-dim-aliases` shim removal | Transient; remove `metadata/dim_aliases.py`, `unit-tests/test_dim_aliases.py`, and every `# dim-aliases:` touchpoint when GFS metadata stops spelling `vertical_layer_dimension` as `adjusted_vertical_layer_dimension_for_radiation` / `vertical_composition_dimension`. | +| `--legacy-auto-clone-constituents` shim removal | Transient; remove `metadata/auto_clone_constituents.py`, `unit-tests/test_auto_clone_constituents.py`, sample files under `unit-tests/sample_files/scheme_auto_clone_consumer.meta` + `sample_suite_files/suite_auto_clone.xml`, and every `# auto-clone-constituents:` touchpoint when consumers have moved to explicit `host_constituents(:)` declaration or register-phase scheme registration. | | `ccpp_datafile.py` query CLI rework | Deferred (2026-05-13); collapse `--host-files` / `--suite-files` / `--utility-files` into `--capgen-files`, then repurpose `--host-files` as a filtered list of **input** host metadata files (parallel to `--scheme-files`). Most hosts pack all host data into a handful of shared files, so the filtering pay-off is small — the draw is API symmetry. | -| Original capgen auto-clone path | Intentionally dropped in favor of explicit registration; kept in memory as "Option B" fallback. | --- diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index 529168e7..c23c539b 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -1156,7 +1156,7 @@ The following patterns from prebuild or capgen are explicitly **not** carried fo | `type = module` (capgen) | Renamed to `type = host` | | `finalize` phase name | Renamed to `final` | | Array size checks in caps | Not generated by default; rely on compiler bounds checking | -| Auto-clone of `is_constituent` scheme args into framework `%instantiate` calls | Replaced by explicit registration (host_constituents arg + register-phase `ccpp_constituent_properties_t`) | +| Auto-clone of `is_constituent` scheme args into framework `%instantiate` calls | Replaced by explicit registration (host_constituents arg + register-phase `ccpp_constituent_properties_t`); the auto-clone path is also available behind the opt-in `--legacy-auto-clone-constituents` shim for legacy hosts (single-instance only — see `doc/auto_clone_constituents.md`) | | `ConstituentVarDict` synthetic scope between suite and host | Removed; constituents are a `source='constituent'` classification on `ResolvedArg` | --- @@ -1283,14 +1283,90 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` enumerates every pair automatically; no hard-coded text per pairing. +### Landed in the 2026-05-14 → 2026-05-20 window + +- **`--no-host-introspection` flag** (2026-05-14) — stubs the bodies + of the five suite-introspection routines in `_ccpp_cap.F90`, + dropping the file from ~33k lines to ~800 for the 10-suite SCM + build (the case-blocks were making even `-O1` compilation + effectively hang). Signatures stay so existing host callers still + link; stubbed bodies return `errflg = 1` with a clear `errmsg`. +- **Per-instance dynamic-constituents buffer** (2026-05-18) — the + per-suite buffer that holds register-phase-allocated constituents + was lifted from "shared across instances" to a per-instance wrapper + DDT array. Surfaced by the new combined multi-instance + + constituents end-to-end test (`instances_advection`). Fixes a + latent set_const_index conflict and a class-B setter-mutation + problem. +- **Final-path silent idempotency** (2026-05-15) — both + `ccpp_physics_final` and `ccpp_final` return `errflg = 0` on + repeats; `ccpp_physics_final` additionally silent-skips after + `ccpp_final` teardown. +- **Validator per-arg attribute checks** (2026-05-20) — `intent`, + `type`, `kind`, and `rank` checked per argument; asymmetric + `optional` rule; DDT + `external::` normalisation; + character `len=*` wildcard. Caught 67 real metadata/Fortran + disagreements in the SCM physics tree on landing day. +- **Host/scheme metadata cross-checks** (late 2026-05-20) — resolver + enforces type identity, rank, and per-position dim entries between + host metadata (or first-writer suite-owned var) and every consuming + scheme arg. Bare `X` ≡ `1:X` ≡ `ccpp_constant_one:X` collapse; + every other lower bound stays distinct; numeric kind stays lenient + (transform path). +- **Host `active` + scheme arg runtime guard** (late 2026-05-20) — + replaces the earlier static rule that required scheme metadata to + declare `optional = True` for any host-active variable. Optional + scheme arg → pointer-association (PRESENT()-aware); non-optional + scheme arg → group-cap runtime guard before any transform. +- **`ccpp_static_api.F90` → `_ccpp_cap.F90`** — generated public + entry-point file is now per-host (filename and module name driven + by `--host-name`). Multiple host integrations can coexist in one + build. + +### Landed 2026-05-21 + +- **`--gfs-dim-aliases` shim** — transient CLI flag that treats GFS + radiation/composition vertical-dim names + (`adjusted_vertical_layer_dimension_for_radiation` and + `vertical_composition_dimension`) as equivalent to + `vertical_layer_dimension` **inside the resolver's + per-position dim-identity check only** (upper bound only). Host + variables stay distinct everywhere else. Single touchpoint at + `generator/suite_resolver.py::_canonical_dim`; module + `metadata/dim_aliases.py`; touchpoints tagged `# dim-aliases:` for + clean removal. Generator-only (the validator never reaches the + canonicaliser). Required for CCPP-SCM 17p8 to build under + capgen-ng. +- **`--legacy-auto-clone-constituents` shim** — transient CLI flag + that reinstates original ccpp-capgen's auto-clone-static-constituent + registration path. Every `is_constituent` consumer scheme arg + (`advected = True` / `constituent = True` / `molar_mass = …`) with + no register-phase source is auto-registered into the per-suite + dynamic-constituents buffer using values lifted straight from the + scheme metadata. Adds four legacy `%instantiate` kwargs to the + parser (`default_value`, `min_value`, `water_species`, + `mixing_ratio_type`). Synthesises `long_name` from std_name when + missing; falls back `diag_name` to local_name; lifts `vertical_dim` + from the arg's dim list. Available on both `ccpp_capgen_ng.py` and + `ccpp_validator.py`. **Single-instance only** — aborts before + parsing if the host declares `instance_number` + + `number_of_instances`. Module + `metadata/auto_clone_constituents.py`; touchpoints tagged + `# auto-clone-constituents:`. New e2e fixture + `end-to-end-tests/advection_auto_clone/` is a port of CAM-SIMA's + `advection_test`. Full reference: `doc/auto_clone_constituents.md`. + ### Test status -- **Unit tests**: 1353 passing (1365 with doctests). Run via `python unit-tests/run_tests.py [--doctest]`. -- **End-to-end tests**: `advection`, `unit_conv`, `nested_suite`, - `variable_transform`, `instances`, `ddt` covered. SCM running - against ccpp-physics is the active driver right now — most of the - late-evening landings were surfaced by SCM build/runtime failures. - Tree is off-limits for in-session edits — user-driven. +- **Unit tests**: 1426 passing (1438 with doctests; as of 2026-06-01). + Run via `python unit-tests/run_tests.py [--doctest]`. +- **End-to-end tests** (10 passing): `advection`, + `advection_auto_clone`, `capgen_ng`, `chunked_data`, `ddthost`, + `instances`, `instances_advection`, `nested_suite`, `opt_arg`, + `var_compat`. SCM running against ccpp-physics continues to be + the active driver — most of the landings since 2026-05-13 were + surfaced by SCM build/runtime failures. Tree is off-limits for + in-session edits — user-driven. ### Still deferred @@ -1314,6 +1390,22 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and every `# legacy-compat:` touchpoint when scheme metadata has migrated. +- **`--gfs-dim-aliases` shim removal** (added 2026-05-21) — + transient; remove `metadata/dim_aliases.py`, + `unit-tests/test_dim_aliases.py`, and every `# dim-aliases:` + touchpoint when GFS metadata stops spelling + `vertical_layer_dimension` as + `adjusted_vertical_layer_dimension_for_radiation` / + `vertical_composition_dimension`. +- **`--legacy-auto-clone-constituents` shim removal** (added + 2026-05-21) — transient; remove + `metadata/auto_clone_constituents.py`, + `unit-tests/test_auto_clone_constituents.py`, the sample fixtures + (`unit-tests/sample_files/scheme_auto_clone_consumer.meta`, + `unit-tests/sample_suite_files/suite_auto_clone.xml`), and every + `# auto-clone-constituents:` touchpoint when consumers have moved + to explicit `host_constituents(:)` declaration or register-phase + scheme registration. - **Nested subcycle `ccpp_loop_counter` semantics**: a scheme inside a nested subcycle requesting `ccpp_loop_counter` would get the OUTERMOST counter, not the innermost. None of the cam-sima schemes From 8b91f6f87103db60ab8e164a29fd355f9d64bdc0 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 1 Jun 2026 09:36:48 -0600 Subject: [PATCH 44/74] Clean up docs/* --- doc/briefing_20260519T0905.md | 381 --- doc/briefing_pm.md | 26 +- doc/briefing_pm_20260519T0905.md | 360 --- doc/constituents_20260513T0733.md | 1029 ------- doc/constituents_20260519T0905.md | 1029 ------- doc/constituents_overhaul_20260513T0733.md | 836 ------ doc/constituents_overhaul_20260519T0905.md | 947 ------ doc/migration_20260513T0733.md | 545 ---- doc/migration_20260519T0905.md | 729 ----- doc/redesign_analysis_20260513T0733.md | 2639 ----------------- doc/redesign_prompt_20260513T0733.md | 1230 -------- doc/redesign_prompt_original_20260505T2044.md | 814 ----- 12 files changed, 13 insertions(+), 10552 deletions(-) delete mode 100644 doc/briefing_20260519T0905.md delete mode 100644 doc/briefing_pm_20260519T0905.md delete mode 100644 doc/constituents_20260513T0733.md delete mode 100644 doc/constituents_20260519T0905.md delete mode 100644 doc/constituents_overhaul_20260513T0733.md delete mode 100644 doc/constituents_overhaul_20260519T0905.md delete mode 100644 doc/migration_20260513T0733.md delete mode 100644 doc/migration_20260519T0905.md delete mode 100644 doc/redesign_analysis_20260513T0733.md delete mode 100644 doc/redesign_prompt_20260513T0733.md delete mode 100644 doc/redesign_prompt_original_20260505T2044.md diff --git a/doc/briefing_20260519T0905.md b/doc/briefing_20260519T0905.md deleted file mode 100644 index 69f60336..00000000 --- a/doc/briefing_20260519T0905.md +++ /dev/null @@ -1,381 +0,0 @@ -# capgen-ng — Briefing for CCPP Framework Developers & Power Users - -*Prepared for the 2026-05-14 walk-through. Companion document to -`doc/migration.md` (the detailed migration guide) and -`doc/redesign_prompt.md` (the implementation spec).* - ---- - -## 1. Why a new generator? - -The CCPP Framework runs two code generators today: - -- **`ccpp-prebuild`** — simple, procedural Python; fast; DDT-argument - passing; in production use by NOAA UFS Weather Model, Navy NEPTUNE, - and CCPP-SCM. Reliable but feature-light. No framework-owned - variables. -- **`ccpp-capgen`** — complex, deeply object-oriented Python; flat-field - argument passing; in use by NCAR CAM-SIMA. Many advanced features - designed but never implemented; at UFS/NEPTUNE scale (1200+ variables) - flat-field passing prevents the use of strict error-checking flags - (`-check all`, `-fcheck=all`) required for operational implementation, - and even when it does compile it produces unmaintainably large source - files; nobody on the team fully understands it. - -**`capgen-ng`** starts fresh, drawing lessons from both. Guiding -principle: **simplicity of prebuild, feature set of capgen**. - -What we wanted to fix: - -1. **Flat fields → DDT arguments at all scales.** No "flat-field - group cap" failure mode. -2. **No scope-chain variable promotion.** Variables flow through - metadata, not through a runtime synthetic dictionary stacking. -3. **Code anyone can read and extend.** No 10-deep class hierarchy. -4. **One generator, one CLI, one query tool** for both prebuild-style - and capgen-style hosts. - ---- - -## 2. What capgen-ng is (in one paragraph) - -capgen-ng reads metadata for the **host model**, the **physics -schemes**, and the **suite definition files** (SDFs), produces a -small set of Fortran cap modules that bridge them, and writes a -`datatable.xml` describing the result for CMake / Make to consume. -At runtime the host calls a small set of public entry points -(`ccpp_register`, `ccpp_init`, `ccpp_physics_init`, -`ccpp_physics_run`, `ccpp_physics_*_init`/`_final`, `ccpp_final`); the -generated caps dispatch by `suite_name` (and optionally `group_name`) -to the right scheme. - ---- - -## 3. Core concepts - -### 3.1 Five metadata table types - -| `type = ` | Owner | How it reaches the cap | -|-------------|------------------|----------------------------| -| `scheme` | Physics scheme | Intent args on scheme subs | -| `host` | Host model | Module USE (direct / DDT) | -| `control` | Framework runtime layer | `ccpp_physics_*` args | -| `ddt` | Type definition | Structural — fields only | -| `suite` | Generated suite cap | Module USE | - -### 3.2 Three layers of generated cap - -- **Static API** (`ccpp_static_api.F90`) — public entry points; one per - build. Dispatches by `suite_name` → suite cap. -- **Suite cap** (`ccpp__cap.F90`) — per-suite state machine, plus - dispatch by `group_name` → group cap. Suite-owned interstitial data - lives in a sibling `ccpp__data.F90`. -- **Group cap** (`ccpp___cap.F90`) — scheme call sites - with full argument lists, unit/kind/vertical-flip transforms, - optional-arg pointer wrappers, subcycle `do` loops. - -### 3.3 Two-level integer state machine - -Replaces both the boolean `initialized(:)` array from prebuild and -the string-based `ccpp_suite_state` from CAM-SIMA capgen. Per -instance: - -- **Suite-level**: `UNREGISTERED → REGISTERED → FRAMEWORK_INITIALIZED`. -- **Group-level**: `UNINITIALIZED → INITIALIZED → IN_TIMESTEP`. - -### 3.4 Six scheme phases - -`register`, `init`, `timestep_init`, `run`, `timestep_final`, `final`. -`register` is new — schemes that contribute to the constituent table -do so here. `final` replaces the older `finalize` (breaking change, -intentional). - -### 3.5 Variable resolution - -For each scheme arg: - -1. Found in host+control metadata → use the access path. If units / - kind / vertical orientation differ, generate a transform. -2. Not found, first use is `intent(out)` → **suite-owned** variable - (interstitial); add to `ccpp__data.F90`. -3. Not found, first use is `intent(in/inout)` → **error**. -4. Found in suite data (a prior scheme provided it) → use suite data - access path. - -### 3.6 Two tools, one parser - -- `ccpp_capgen_ng.py` — the code generator. Trusts metadata; no - Fortran parsing. -- `ccpp_validator.py` — the standalone Fortran-vs-metadata checker. - The ONE place capgen-ng parses Fortran. Run by developers / - CMake before generation. - -Both share the same metadata-parsing library (`metadata/`). - ---- - -## 4. How capgen-ng differs from `ccpp-prebuild` - -| Topic | prebuild | capgen-ng | -|-----------------------------|-----------------------------------|---------------------------------------------------| -| Host metadata mechanism | Hard-coded Python dict (`TYPEDEFS_NEW_METADATA`) | Regular `type = ddt` + `type = host` tables | -| Framework-owned variables | Not supported | First-class (suite-owned interstitial via Case 2) | -| Constituents | Hand-rolled, host-specific glue | Standardized opt-in mechanism with auto-provision | -| `register` phase | Doesn't exist | First phase; schemes declare dynamic constituents | -| Multi-instance API | Implicit, ad-hoc | Paired-opt-in (`instance_number` / `number_of_instances`) | -| Subcycle loop counter | Host plumbs it manually | Registered std names `ccpp_loop_counter` / `ccpp_loop_extent` resolve to the do-loop locals automatically inside `` | -| Suite introspection | Limited | Five runtime queries (`ccpp_physics_suite_list`, `_part_list`, `_schemes`, `_variables`, `_host_data`) | - ---- - -## 5. How capgen-ng differs from `ccpp-capgen` - -| Topic | capgen | capgen-ng | -|-----------------------------|-----------------------------------|---------------------------------------------------| -| Group-cap arguments | Flat fields (1200+ at UFS scale) | DDT arguments (as in prebuild) | -| Variable matching algorithm | Five-layer scope-chain promotion | Flat host+control dict + suite-owned discovery (inherited from prebuild — primary reason runtime is comparable to prebuild) | -| External types (MPI f08 comm, ESMF clock) | Tabled (solution complexity) | First-class via `type = external::` | -| `type = module` in metadata | Yes | Renamed `type = host` | -| `is_constituent` scheme args | Auto-cloned by generator | Schemes register constituents explicitly in the `register` phase via `ccpp_constituent_properties_t(:)` | -| `ConstituentVarDict` | Synthetic scope between suite + host | Removed; constituents are one of four sources (`control`/`host`/`suite`/`constituent`) on `ResolvedArg` | -| `_state` runtime check | String | Integer (named parameters) | -| Fortran-vs-metadata check | Inside the generator | Separate tool (`ccpp_validator.py`) | -| Code complexity | Deep OO hierarchy | Flat data classes + procedural resolver | - ---- - -## 6. Breaking metadata changes hosts must make - -Comprehensive list — see `doc/migration.md` for full detail. - -### 6.1 Table types - -- `type = module` → **`type = host`**. - -### 6.2 Phase names - -- `_finalize` → **`_final`** in both metadata and - Fortran source. - -### 6.3 Standard names - -- `horizontal_loop_extent` → **`horizontal_dimension`** uniformly in - scheme metadata. (The chunk-vs-full-domain distinction is driven - by what the host passes for `horizontal_loop_begin` / - `horizontal_loop_end`.) -- `number_of_openmp_threads` → **`number_of_threads`** (matches the - `thread_number` control variable convention). - -Both are rewritten on the fly by **`--legacy-mode`** for a transition -period; the shim prints a banner listing every rewrite it performs -and is marked for clean removal. - -### 6.4 Required host `type = control` table - -Every host MUST declare scalar integers (and one character) with -these CCPP standard names: - -- `suite_name`, `horizontal_loop_begin`, `horizontal_loop_end`, - `thread_number`, `number_of_threads`, `number_of_physics_threads`, - `ccpp_error_code`, `ccpp_error_message`. - -Optional (paired): `instance_number` (control) + -`number_of_instances` (host). - -### 6.5 DDT-instance variables with scalar-index dims - -Container DDT-instance variables (`physics%Interstitial`, -`physics%Coupling`, ...) dimensioned by a count standard name -(`number_of_threads`, `number_of_instances`) get their scalar index -inserted **automatically** by capgen-ng. The host metadata declares -the dim; the generator emits -`physics%Interstitial(thread_number)%alpha(...)` at every call site. - -The host's Fortran can keep its existing OpenMP-thread-private DDT -layout — no glue code needed on the host side. - -### 6.6 Leaf variables MUST NOT carry registered scalar-index dims - -Rule 2 of the registered-scalar-index-dimension contract: scalar -variables (real / integer / character / DDT-typed leaves the scheme -binds to) cannot declare `number_of_threads` or `number_of_instances` -as a dimension. Wrap them in a container DDT instead. This is -enforced at parse time with an explicit remediation message; existing -CCPP-physics, UFS-WM, and CAM-SIMA host metadata is already -compliant. - -### 6.7 No more `cdata` / `ccpp_t` struct passing - -The framework-owned bag-of-state struct is replaced by explicit -control-variable arguments to the public entry points. - ---- - -## 7. What capgen-ng does NOT support (yet) - -### 7.1 Deferred — to be resolved in upcoming work - -- **Constituents overhaul.** Three reform proposals on the table - (`doc/constituents_overhaul.md`); decision pending an upcoming - meeting. Pieces involved: framework setter additions - (`set_advected`, `set_diagnostic_name`, `set_default_value`), - `is_match` relaxation, Class A vs Class B property classification. -- **Validator host-metadata check.** `ccpp_validator.py` currently - validates scheme metadata only; host-metadata-vs-Fortran is on - hold until the e2e test suite settles. -- **Codegen-time scheme-registration cross-check.** Today's - registration check is at runtime - (`ccpp_initialize_constituents`). Stronger options: new metadata - attribute `registers_std_names = a, b, c` on register-phase - tables; cross-check at codegen. -- **Nested-subcycle `ccpp_loop_counter` semantics.** When a scheme - inside a deeply nested subcycle asks for `ccpp_loop_counter`, it - currently resolves to the **outermost** loop's counter. None of - the in-tree physics catalogs uses the inner-counter case. -- **`ccpp_datafile.py --host-files` repurpose.** The current - `--host-files` returns the generated host-API file; should be a - filtered list of *input* host metadata files (parallel to the new - `--scheme-files`). Deferred. -- **`ccpp_host_constituents.F90` suppression** when no suite touches - constituents (file is correct-but-empty under host-wins; should - not be emitted at all). -- **Python linter / formatter pass.** Pick `ruff`, apply across - `capgen-ng/`. - -### 7.2 Intentionally NOT supported - -- **`_finalize` phase spelling.** Use `_final`. No legacy-mode - shim — rename in metadata + Fortran. -- **`type = module`.** Use `type = host`. -- **Flat-field scheme call arguments** (capgen's failure mode). -- **`character(len=*)` as a DDT component** (Fortran disallows it; - we error at parse time with a remediation pointing at - `character(len=:)` deferred-length). -- **Multiple registration sources for the same constituent** with - silent dedup. Today's behavior is to error on conflict; the - proposed reform sets a clear precedence rule (host-set Class B - properties win) — pending the constituents-overhaul decision. -- **`ConstituentVarDict`** synthetic scope between suite and host. - Gone for good. - ---- - -## 8. Validation and error reporting - -A deliberate design choice across capgen-ng: **errors are loud, -specific, and actionable**. Examples surfaced during the SCM -shake-down: - -- Empty `units =` line → error names file, line, variable, - attribute, raw value, AND inner reason. -- Scheme metadata file passed via `--scheme-files` but missing from - the SDF → silently ignored (and dropped from `` so - CMake doesn't compile orphan code). -- Scheme listed in the SDF but its metadata not supplied → single - CCPPError listing every missing scheme + pointer to - `--scheme-files`. Replaces silent empty-cap emission. -- DDT-instance variable with a non-registered scalar-index dim AND - flattenable fields → error shows the broken access pattern - capgen-ng WOULD have emitted and quotes the Fortran compiler - error verbatim ("Component to the right of a part reference with - nonzero rank must not have the POINTER attribute"). -- Generated `case default` on `select case(suite_name)` / - `select case(group_name)` → unknown suite or group at runtime - produces a clear errflg + errmsg, not silent fall-through. - ---- - -## 9. Build-system integration (capsule view) - -```cmake -# In your CMakeLists.txt -set(SCHEME_METADATA_FILES …list of .meta paths…) -set(HOST_METADATA_FILES …list of host .meta paths…) -set(SUITE_FILES …list of suite XML paths…) - -# Validate before generation (developer step, optional in CI) -ccpp_validator(SOURCE_FILES ${SCHEME_FORTRAN_FILES} - METADATA_FILES ${SCHEME_METADATA_FILES}) - -# Run the code generator -ccpp_capgen(HOSTFILES ${HOST_METADATA_FILES} - SCHEMEFILES ${SCHEME_METADATA_FILES} - SUITES ${SUITE_FILES} - HOST_NAME ${HOST} - OUTPUT_ROOT ${OUTPUT_ROOT}) - -# Pull the manifest from the datatable -ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" - REPORT_NAME "--scheme-files") -set(SCHEME_FORTRAN_FILES ${CCPP_FILES}) - -ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" - REPORT_NAME "--dependencies") -set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) - -ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" - REPORT_NAME "--capgen-files") -set(CAPGEN_FILES ${CCPP_FILES}) - -add_library(scm-ccpp STATIC - ${CAPGEN_DEPENDENCIES} - ${SCHEME_FORTRAN_FILES} - ${HOST_FORTRAN_FILES} - ${CAPGEN_FILES}) -``` - -Regenerating on every CMake configure is cheap — `write_if_changed` -preserves mtimes when content hasn't changed, so `make` / `ninja` -don't rebuild downstream objects unless something actually moved. - ---- - -## 10. Where things stand right now - -- **Unit tests**: 1316 passing on `feature/capgen-ng` (as of 2026-05-19). -- **End-to-end tests passing**: `advection`, `unit_conv`, - `nested_suite`, `variable_transform`, `instances`, - `instances_advection`, `ddt`. -- **CCPP-SCM**: actively driving development — every build / runtime - failure surfaced this week landed as a fix in capgen-ng (rather than - being patched around in the host). Most of the `phys_ps` group now - builds end-to-end via `--legacy-mode`. -- **`--no-host-introspection`** (new, 2026-05-14): stubs the bodies of - the five suite-introspection routines in `ccpp_static_api.F90`, - shrinking the file from ~33k lines to ~800 for the 10-suite SCM - build (the introspection case-blocks were making even `-O1` - compilation effectively hang). Signatures stay so existing host - callers still link; stubbed bodies return `errflg = 1` with a clear - `errmsg`. -- **NEPTUNE**: cleanup and acceptance testing in progress. - Regular/lower-atmosphere physics builds and runs and produces - results within tolerance (deviations similar to compiler changes). - High-altitude physics testing is next. -- **UFS Weather Model**: not yet attempted; SCM is the proving - ground first. An anticipated complication is the "fast physics" - called directly from the FV3 dynamical core as a separate group. -- **CAM-SIMA**: not yet reconnected; pending the constituents - overhaul decision and CAM-SIMA developer availability. - ---- - -## 11. Walk-through outline (suggested order for the meeting) - -1. Live `ccpp_capgen_ng.py --help` (CLI shape). -2. Show one scheme's `.meta` + its generated group-cap fragment. -3. Run the generator twice — note the `Unchanged: …` messages on the - second pass (write-if-changed in action). -4. Run `ccpp_datafile.py --scheme-files datatable.xml` to show the - filtered manifest. -5. Demonstrate a deliberately-broken metadata (`units =` empty, or - missing scheme, or invalid `case default` group) to show the - error UX. -6. Walk through the registered scalar-index dimension table and the - two rules. -7. Open the floor — focus areas for the audience: - - **Host metadata maintainers**: anything in §6 that surprises - you for your model? - - **Scheme metadata maintainers**: anything in §6.2 / §6.3 that - can't be migrated cleanly? - - **Framework devs**: §7.1 — which deferred items block your - downstream work? diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md index 86a74b59..06101d20 100644 --- a/doc/briefing_pm.md +++ b/doc/briefing_pm.md @@ -40,7 +40,7 @@ stand. ## 1. The three generators in one paragraph each -**`ccpp-prebuild`** (NOAA, in production for UFS, NEPTUNE, SCM). +**`ccpp-prebuild`** (NOAA/NAVY/DTC, in production for UFS/NEPTUN/SCM). Procedural Python; reads metadata; emits Fortran caps; passes host-defined derived-type (DDT) arguments to scheme call sites. In production for several years; bug rate is low; the team understands @@ -97,7 +97,7 @@ Three pressures converged in 2025/26: `ConstituentVarDict` scope-chain or the auto-clone path requires reading several modules together. Realistically, only one or two people on the framework team can change capgen without breaking - something downstream. One of them now lives overseas and rejects + something downstream. One of them now lives overseas and opposes simplification attempts from the others. This is an unacceptable **bus-factor risk** that the redesign retires. @@ -117,11 +117,11 @@ by the SCM / multi-instance test work this month. CAM-SIMA's group caps pass **every individual variable as a separate argument** to the scheme dispatch routine. At CAM-SIMA's roughly two-hundred-variable scale this works. At UFS scale (~1200 -variables per group), the generated Fortran exceeds compiler limits +variables per group), the generated Fortran exceeds compiler limits, prevents the use of strict error-checking flags (`-check all`, `-fcheck=all`) required for operational implementation, and even when it does compile produces unmaintainably large source files. -**This is the technical reason capgen cannot drive UFS today**, +**This is one technical reason capgen cannot drive UFS today**, independent of any other concern. `capgen-ng` reverts to prebuild's DDT-argument convention. Host @@ -213,8 +213,8 @@ reflects the feature set — but the practical consequence is that the maintenance burden falls on a small subset of the framework team. capgen-ng is comparable to prebuild in *shape* (procedural Python with small data classes — no deep class hierarchy), and the -generator itself sits at ~17.8k lines (capgen is several times -larger). The "who can fix this" pool is closer to "anyone with +generator itself sits at ~17.8k lines. +The "who can fix this" pool is closer to "anyone with framework context". capgen-ng comes with ~1.4k docstring + unit tests (~18k lines of test code), plus an end-to-end test suite of 10 fixtures that covers all of prebuild's and capgen's existing @@ -222,7 +222,7 @@ end-to-end tests and adds new ones for multi-instance + constituents (`instances_advection`) and the auto-clone-constituents shim (`advection_auto_clone`). Including these tests and the rich inline comments puts capgen-ng's full tree on the same order of magnitude as -capgen — almost all of which is test coverage and human-readable +capgen — about half of which is test coverage and human-readable prose, not load-bearing logic. --- @@ -257,7 +257,7 @@ Features that exist only in capgen-ng (some exist in prebuild): | **Registered scalar-index dimensions** | When metadata says a variable is dimensioned by `number_of_threads` or `number_of_instances`, capgen-ng injects the right per-call subscript automatically; the host's OpenMP-thread-private DDT layout works unchanged | | **Subcycle loop-counter automation** | Schemes inside a `` element can access `ccpp_loop_counter` / `ccpp_loop_extent` directly; the generator emits the Fortran `do` loop and binds the locals | | **`--legacy-mode` migration shim** | One CLI flag enables silent rewrite of two known-good deprecated standard names (`horizontal_loop_extent` → `horizontal_dimension`, `number_of_openmp_threads` → `number_of_threads`) with a loud warning — buys time for host metadata to migrate | -| **`--gfs-dim-aliases` migration shim** (2026-05-21) | One CLI flag treats GFS-physics names (`adjusted_vertical_layer_dimension_for_radiation`, `vertical_composition_dimension`) as equivalent to `vertical_layer_dimension` in the dim-identity check only — variables remain distinct everywhere else. Resolver-only; clean grep-revert. Required for CCPP-SCM 17p8 to build under capgen-ng. | +| **`--gfs-dim-aliases` migration shim** (2026-05-21) | One CLI flag treats GFS-physics names (`adjusted_vertical_layer_dimension_for_radiation`, `vertical_composition_dimension`) as equivalent to `vertical_layer_dimension` in the dim-identity check only — variables remain distinct everywhere else. Resolver-only; clean grep-revert. Required for CCPP-SCM v17p8 to build under capgen-ng. | | **`--legacy-auto-clone-constituents` migration shim** (2026-05-21) | One CLI flag reinstates original ccpp-capgen's auto-clone-static-constituent registration path for the ~16 production-CAM-SIMA schemes that depend on it. Single-instance only (predates multi-instance); fails fast if a multi-instance host is supplied. This is the no-decision-needed bridge that lets capgen-ng accept CAM-SIMA's atmospheric_physics metadata before any constituent-overhaul work lands. | | **`--no-host-introspection` flag** | The five runtime introspection routines (`ccpp_physics_suite_list`, etc.) emit large `select case` blocks at SCM scale; this flag stubs the bodies, dropping the generated static API from ~33,000 lines to ~800 for the SCM build (the introspection routines were making `-O1` compilation effectively hang) | | **Consistent handling of external types** (MPI f08 communicator, ESMF clock) | Tabled in capgen because of the complexity of the solution | @@ -295,10 +295,10 @@ Features that exist only in capgen-ng (some exist in prebuild): bug; the fix moves the per-suite dynamic-constituents buffer per-instance. No coordination with CAM-SIMA / UFS / NEPTUNE required (host-facing API unchanged). -- **NEPTUNE**: cleanup and acceptance testing in progress. - Regular/lower atmosphere physics builds and runs, and produces - results within the tolerance (i.e. similar to compiler changes). - High altitude atmosphere testing is next. +- **NEPTUNE**: Final cleanup and acceptance testing in progress. + All regression tests (~300) pass with the three mandatory + compilers (Intel LLVM, GCC, LLVM native) for regular physics, + mid-altitude, and high-altitude physics (feature-complete). - **UFS Weather Model**: not yet attempted; SCM is the proving ground first. Expecting updates due to the "fast physics" called directly from the FV3 dynamical core as separate group. @@ -368,7 +368,7 @@ Three points worth raising explicitly: discarded.** capgen-ng is genuinely the successor, not a parallel project. The contributions made on the capgen side are what made the capgen-ng feature set possible. A significant - portion of capgen's code, in particular metadata parsine, + portion of capgen's code, in particular metadata parsing, Fortran-metadata validation, and constituents, were imported into capgen-ng. 3. **The team owning capgen-ng can be larger than the team owning diff --git a/doc/briefing_pm_20260519T0905.md b/doc/briefing_pm_20260519T0905.md deleted file mode 100644 index ccbbcd2f..00000000 --- a/doc/briefing_pm_20260519T0905.md +++ /dev/null @@ -1,360 +0,0 @@ -# capgen-ng — Briefing for Project Management - -*Companion to `doc/briefing.md` (the developer walk-through) and -`doc/redesign_analysis.md` (the deep-dive technical comparison of -prebuild and capgen). This document targets project leadership and -program managers; it summarises the case for `capgen-ng` in terms of -product risk, schedule, and cross-organization impact rather than -implementation detail.* - -*Last revised: 2026-05-18.* - ---- - -## TL;DR - -The CCPP Framework today ships **two** code generators that solve the -same problem differently: - -- **`ccpp-prebuild`** powers NOAA UFS, Navy NEPTUNE, and CCPP-SCM. - Simple and reliable, but feature-light — does not support features - CAM-SIMA needs (constituents, framework-owned variables, - introspection). -- **`ccpp-capgen`** powers NCAR CAM-SIMA. Feature-rich, but built on - technical choices that **do not scale** to UFS or NEPTUNE and that - **do not support multi-instance hosts** at all. - -Neither generator can be the basis for a single shared toolchain. -**`capgen-ng`** is a third generator, started in early May 2026, -designed to do everything both other generators do, in code small -enough for a few people to own, with the architectural choices that -make it work at UFS/NEPTUNE scale and beyond. The redesign is -running on the SCM as proving ground; UFS / NEPTUNE / CAM-SIMA -re-integration is sequenced behind that. - -This document explains, in plain language, **why we did not extend -capgen instead**, what risks the redesign retires, and where things -stand. - ---- - -## 1. The three generators in one paragraph each - -**`ccpp-prebuild`** (NOAA, in production for UFS, NEPTUNE, SCM). -Procedural Python; reads metadata; emits Fortran caps; passes -host-defined derived-type (DDT) arguments to scheme call sites. In -production for several years; bug rate is low; the team understands -it (those who worked with it). What it doesn't do: framework-owned -variables, the constituent mechanism CAM-SIMA needs, and runtime -introspection. Treated as the **baseline for simplicity and -reliability**. - -**`ccpp-capgen`** (NCAR, in production for CAM-SIMA). Heavy -object-oriented Python (deep class hierarchy, ~tens of thousands of -lines); reads metadata; emits Fortran caps that pass **flat scalar -fields** to scheme call sites instead of DDTs; supports the -constituent mechanism, suite-owned variables, introspection, and a -few other features prebuild lacks. **It is the only existing -generator with those features.** But — see §3 — it has structural -limits that make it impractical for UFS, NEPTUNE, or multi-instance -hosts, and the implementation is concentrated enough that few people -can extend it safely (if at all - primary developer gone). - -**`capgen-ng`** (new, 2026-05). Procedural Python (a few thousand -lines; flat data classes); reads the same metadata format; passes -arguments like prebuild; supports the features capgen pioneered -(constituents, suite-owned variables, introspection); supports -multi-instance, an integer state machine, six explicit scheme -phases, vertical-flip / unit / kind transforms, registered -scalar-index dimensions for threading and ensembles, write-if-changed -build integration, and a separate Fortran-vs-metadata validator -tool. Designed so the same generator works for prebuild-style hosts -(UFS / NEPTUNE / SCM) and capgen-style hosts (CAM-SIMA). - ---- - -## 2. Why this matters now - -Three pressures converged in 2025/26: - -1. **Framework unification heavily delayed.** A fully-functional - capgen that supports UFS / NEPTUNE / SCM and replaces prebuild - was promised for years, and never delivered. Pressure from - project management and sponsors is building. -2. **UFS / NEPTUNE want the capgen feature set.** Constituents - in particular are increasingly central to atmospheric physics - (chemistry, aerosols, deep atmosphere), and re-implementing the - prebuild-side glue per host is duplicated effort. Extending - capgen to UFS-scale runs into the flat-field problem (§3.1) — - not a small refactor, a fundamental data-shape change. The - performance of capgen generating multi-suite caps is up to - 20 times slower than that of prebuild for the CCPP SCM. This - is caused by fundamental design choices (five layers of - classes inheriting from each other) that are integral to capgen. -3. **The team owning capgen has limited bandwidth to extend it.** - The class hierarchy is intricate; understanding the - `ConstituentVarDict` scope-chain or the auto-clone path requires - reading several modules together. Realistically, only one or two - people on the framework team can change capgen without breaking - something downstream. One of them now lives overseas and rejects - simplification attempts from the others. This is an unacceptable - **bus-factor risk** that the redesign retires. - ---- - -## 3. What capgen does that does not extend to UFS / NEPTUNE / multi-instance - -This section is for the project lead who came from the capgen side: -none of these are critiques of capgen as a *product*. They are -specific architectural choices that worked for CAM-SIMA's -single-instance design and don't generalize. Each is sourced from -the technical analysis in `doc/redesign_analysis.md` and validated -by the SCM / multi-instance test work this month. - -### 3.1 Flat-field argument passing fails at UFS / NEPTUNE scale - -CAM-SIMA's group caps pass **every individual variable as a separate -argument** to the scheme dispatch routine. At CAM-SIMA's roughly -two-hundred-variable scale this works. At UFS scale (~1200 -variables per group), the generated Fortran exceeds compiler limits -prevents the use of strict error-checking flags (`-check all`, -`-fcheck=all`) required for operational implementation, and even -when it does compile produces unmaintainably large source files. -**This is the technical reason capgen cannot drive UFS today**, -independent of any other concern. - -`capgen-ng` reverts to prebuild's DDT-argument convention. Host -authors pass their physics DDTs by reference (one or a few arguments -per scheme call); component access happens **at the scheme call level**. -This works at every scale we've measured. - -### 3.2 Single-instance constituents are baked into the generated code - -CAM-SIMA runs one host per executable, so capgen generates a single -module-level `ccpp_model_constituents_obj`. The constituent -mechanism — the central feature capgen-ng inherited from capgen — -references that global directly. Re-targeting capgen to -multi-instance is not a configuration toggle; it requires -re-emitting the constituent module per-instance throughout the -generator, plus refactoring the framework setters. - -`capgen-ng` was multi-instance **from day one**: every constituent -entry point takes an `instance_number` argument; the property -storage, the state machine, the dynamic-constituent buffers are all -per-instance. As of 2026-05-18, the per-suite dynamic-constituents -buffer was also moved per-instance after the new combined -multi-instance + constituents end-to-end test surfaced a latent bug -that capgen would never have hit (because capgen never supported -multi-instance). **The redesign is finding bugs the legacy -toolchain hid.** - -### 3.3 Constituent registration has three competing paths in capgen - -capgen accepts constituent declarations from (a) host-supplied -arrays, (b) scheme `register`-phase Fortran subroutines, and (c) an -**auto-clone path** that scans scheme metadata for the -`is_constituent` attribute and silently generates a registration in -the host cap. The auto-clone path is invisible from the scheme -Fortran — to know whether a scheme registers a constituent you have -to know the generator semantics. This makes scheme code harder to -read, harder to port between hosts, and harder to debug when -registrations collide. - -`capgen-ng` keeps only the first two (explicit) paths. Auto-clone -is deliberately gone — see `doc/constituents_overhaul.md` §2.3. - -### 3.4 Host-specific values baked into scheme metadata - -capgen requires `diagnostic_name` (host's diagnostic-output label, -e.g. `CLDLIQ` for CAM-SIMA but something else for UFS) at -constituent instantiation time. Schemes therefore embed -host-specific strings into their own metadata. Porting a scheme -between hosts requires either editing the scheme or maintaining a -fork. - -`capgen-ng` is moving `diagnostic_name` (and a handful of other -host-configuration properties) to a host-side override mechanism; -schemes carry physics-portable defaults only. The reform is -documented in `doc/constituents_overhaul.md`; the decision is on the -agenda for one of the next framework-team meetings. - -### 3.5 Synthetic variable-resolution scopes are hard to extend - -capgen introduces a five-layer deep synthetic dictionary -(`ConstituentVarDict`) between the suite and host scopes during -variable matching. The mechanism works for capgen's use cases -but is a code path most contributors don't read. Extending the -resolver to handle multi-instance dimensions, scalar-index -substitution, or constituent host-wins semantics required -undoing parts of the synthetic scope. - -`capgen-ng`'s resolver is flat: each scheme arg is classified into -exactly one source (control / host / suite / constituent), recorded -on a small data class (`ResolvedArg`), and used directly by the -emitter. No synthetic dictionary. **This design inherits from -`prebuild` and is the primary reason `capgen-ng` is comparable -in performance to `prebuild`. - -### 3.6 Code volume and team coverage - -capgen is roughly an order of magnitude larger than prebuild, with a -deeply layered class hierarchy. This is not a moral failing — it -reflects the feature set — but the practical consequence is that -the maintenance burden falls on a small subset of the framework -team. capgen-ng is comparable to prebuild in size (a few thousand -lines of mostly procedural Python with small data classes), so the -"who can fix this" pool is closer to "anyone with framework -context". capgen-ng comes with over a thousand docstring tests -and unit tests, as well as a comprehensive end-to-end test suite -that covers all of prebuild's and capgen's existing end to end -tests. capgen-ng adds additional end-to-end tests for new -features such as the multi-instance constituents test. Including -these tests and the rich inline comments makes capgen-ng comparable -in size to capgen. - ---- - -## 4. What `capgen-ng` does better than capgen — at any scale - -For audiences who already accept the multi-instance and UFS-scale -arguments, the day-to-day quality-of-life improvements that apply -even to CAM-SIMA-shape problems: - -| Topic | capgen | capgen-ng | -|---|---|---| -| Scheme call argument shape | Flat fields | DDT references | -| Variable resolution | Scope-chain promotion via synthetic dict | Flat 4-source classification on `ResolvedArg` | -| Suite state runtime check | String comparison | Integer-named-parameter state machine | -| Fortran-vs-metadata validation | Embedded in generator | Standalone tool (`ccpp_validator.py`) — run by developers or CMake before generation | -| Generator code style | Deep class hierarchy | Flat data classes + procedural resolver | -| Error reporting | Variable amount of context | "Loud, specific, actionable" enforced — every parse-time error names file, line, variable, attribute, value, and reason | -| Constituent registration | Three sources (one invisible) | Two sources, both explicit | -| `is_constituent` auto-clone | Yes (host-specific values baked into scheme metadata) | Removed | -| `_finalize` vs `_final` phase name | `_finalize` | `_final` (renamed to keep symmetry with init/timestep_init/timestep_final) | - ---- - -## 5. Additional features of `capgen-ng` compared to `capgen` - -Features that exist only in capgen-ng (some exist in prebuild): - -| Capability | Why it matters | -|---|---| -| **Multi-instance host support** (per-instance state machine, per-instance constituent objects, per-instance dynamic-constituents buffers as of 2026-05-18) | Required by NEPTUNE (prebuild has basic solution) | -| **Registered scalar-index dimensions** | When metadata says a variable is dimensioned by `number_of_threads` or `number_of_instances`, capgen-ng injects the right per-call subscript automatically; the host's OpenMP-thread-private DDT layout works unchanged | -| **Subcycle loop-counter automation** | Schemes inside a `` element can access `ccpp_loop_counter` / `ccpp_loop_extent` directly; the generator emits the Fortran `do` loop and binds the locals | -| **`--legacy-mode` migration shim** | One CLI flag enables silent rewrite of two known-good deprecated standard names (`horizontal_loop_extent` → `horizontal_dimension`, `number_of_openmp_threads` → `number_of_threads`) with a loud warning — buys time for host metadata to migrate | -| **`--no-host-introspection` flag** | The five runtime introspection routines (`ccpp_physics_suite_list`, etc.) emit large `select case` blocks at SCM scale; this flag stubs the bodies, dropping the generated static API from ~33,000 lines to ~800 for the SCM build (the introspection routines were making `-O1` compilation effectively hang) | -| **Consistent handling of external types** (MPI f08 communicator, ESMF clock) | Tabled in capgen because of the complexity of the solution | - ---- - -## 6. Where things stand right now (2026-05-18) - -- **Unit tests**: 1319 passing. No known failures. -- **End-to-end tests**: 10 passing — `chunked_data`, `opt_arg`, - `nested_suite`, `ddthost`, `instances`, `capgen_ng`, - `var_compat`, `advection`, and the new `instances_advection` - (multi-instance + constituents). -- **CCPP-SCM**: actively driving development. Each build / runtime - issue surfaced this week landed as a fix in capgen-ng rather than - a host-side workaround. All available suites in CCPP-SCM now - build and run end-to-end via `--legacy-mode`. -- **Multi-instance + constituents fix landed 2026-05-18**. The new - combined end-to-end test surfaced a latent shared-buffer mutation - bug; the fix moves the per-suite dynamic-constituents buffer - per-instance. No coordination with CAM-SIMA / UFS / NEPTUNE - required (host-facing API unchanged). -- **NEPTUNE**: cleanup and acceptance testing in progress this week. - Regular/lower atmosphere physics builds and runs, and produces - results within the tolerance (i.e. similar to compiler changes). - High altitude atmosphere testing is next. -- **UFS Weather Model**: not yet attempted; SCM is the proving - ground first. Expecting updates due to the "fast physics" - called directly from the FV3 dynamical core as separate group. -- **CAM-SIMA**: not yet re-connected; the constituent overhaul - decision (see §7) and the availability of CAM-SIMA developers - are the gating items. - ---- - -## 7. What is intentionally NOT decided yet - -The redesign is opinionated about the architectural choices (DDT -arguments, per-instance everything, integer state machine, two-tool -split). It is **not** opinionated about the framework-level -constituent reform. - -`doc/constituents_overhaul.md` lays out three reform proposals on -the table: - -- **Proposal A** (mostly landed): bug-fix on the deallocate path + - add missing host setters for properties the host wants to - override. Conservative. -- **Proposal B** (recommended for the next 4–6 weeks): relax the - identity-equality check, formally classify properties as - "scheme-intrinsic" (immutable) vs "host-configuration" (mutable - after registration). Physics schemes using constituents become - genuinely portable across hosts. -- **Proposal C** (tabled): drop scheme-side constituent - registration entirely; only the host registers. Cleaner but - requires coordinated PRs across the framework, both generators, - the CAM-SIMA atmospheric_physics tree, and CAM-SIMA itself. - -These are open questions for the framework-team meeting, not -capgen-ng decisions. capgen-ng is structured so all three -proposals are implementable on top of it. - ---- - -## 8. Risk register (project-management view) - -| Risk | Status | Mitigation | -|---|---|---| -| capgen-ng diverges from capgen feature set | LOW | Cross-checked by `doc/redesign_analysis.md`; the feature comparison table in §4 / §5 is exhaustive | -| Host metadata break for UFS / NEPTUNE / CAM-SIMA | MEDIUM | `--legacy-mode` shim covers the known incompatible standard-name pair; remaining required changes (e.g., `_finalize` → `_final`) are mechanical and listed in `doc/migration.md` §3 | -| Constituent overhaul stalls | MEDIUM | Proposal A unblocks the immediate bug; capgen-ng works with the current framework today; the overhaul is a separate decision track | -| Bus-factor on capgen-ng itself | MEDIUM | Procedural code style + flat data classes + 1319-test safety net; significantly lower than capgen's bus factor | -| Two host call-shape conventions (prebuild-style vs capgen-style) coexist forever | LOW | capgen-ng emits one shape; downstream host conversions are tracked in `doc/migration.md` | -| Regression discovered during NEPTUNE / UFS testing | EXPECTED | SCM proving ground catches most; remaining issues become capgen-ng tickets, not host-side patches | -| ccpp-prebuild end-of-life requires a sunset plan | OPEN | Not yet scoped; both generators currently coexist in the framework repo | - ---- - -## 9. The pragmatic case (for the meeting) - -Three points worth raising explicitly: - -1. **Extending capgen to UFS/NEPTUNE scale is not a configuration - change — it is a refactor of the same magnitude as a redesign.** - The flat-field convention is load-bearing throughout capgen's - variable-matching, resolution, and emission code. Once that - change is made, the resulting generator looks substantially - like capgen-ng anyway. -2. **The features capgen pioneered (constituents, suite-owned - variables, introspection) are kept and improved — not - discarded.** capgen-ng is genuinely the successor, not a - parallel project. The contributions made on the capgen side are - what made the capgen-ng feature set possible. A significant - portion of capgen's code, in particular metadata parsine, - Fortran-metadata validation, and constituents, were imported - into capgen-ng. -3. **The team owning capgen-ng can be larger than the team owning - capgen.** This is the most important practical point for - long-term program health. A framework that three organizations - can maintain is more resilient than a framework that one - organization (or one individual in that organization) can maintain. - ---- - -## 10. References - -- `doc/briefing.md` — developer walk-through; same outline, more - technical detail. -- `doc/redesign_analysis.md` — deep-dive technical comparison of - prebuild and capgen with named-product examples. -- `doc/migration.md` — host-author migration guide. -- `doc/constituents_overhaul.md` — the constituent-reform discussion - document. -- `end-to-end-tests/` — the working examples (`instances_advection` - is the newest, exercises everything end-to-end). diff --git a/doc/constituents_20260513T0733.md b/doc/constituents_20260513T0733.md deleted file mode 100644 index c7581801..00000000 --- a/doc/constituents_20260513T0733.md +++ /dev/null @@ -1,1029 +0,0 @@ -# CCPP capgen-ng — Constituents Reference - -*Last revised: 2026-05-13.* - -This document is the authoritative reference for **constituent variables** in -capgen-ng — what they are, how scheme authors declare them in metadata, what -the host model has to do to plumb them through, what the generator emits, and -how the per-instance lifecycle works. - -> If you are migrating a host or scheme from the original capgen, jump to -> [§9 Differences from original capgen](#9-differences-from-original-capgen) -> first. - ---- - -## Table of Contents - -1. [What is a constituent?](#1-what-is-a-constituent) -2. [The four rules (scheme-author conventions)](#2-the-four-rules-scheme-author-conventions) -3. [Required host metadata + Fortran](#3-required-host-metadata--fortran) -4. [Host-side lifecycle (call sequence)](#4-host-side-lifecycle-call-sequence) -5. [Public API reference](#5-public-api-reference) -6. [Generated code structure](#6-generated-code-structure) -7. [Multi-instance design](#7-multi-instance-design) -8. [Limitations and gotchas](#8-limitations-and-gotchas) -9. [Differences from original capgen](#9-differences-from-original-capgen) -10. [Worked example](#10-worked-example) - ---- - -## 1. What is a constituent? - -A **constituent** is a model variable owned by the host's dynamical core (or -its constituent infrastructure) that is read and updated by physics schemes — -typically a tracer / mass mixing ratio (water vapor, cloud liquid, ozone, -chemistry species) — together with its **tendency**, the rate of change that -physics writes back so the dycore can advect/integrate it forward. - -In capgen-ng, the constituent layer has three concerns: - -1. **Registration** — declaring at model startup which constituents exist - (their standard name, units, vertical layout, advection flag, …). -2. **Storage** — the framework owns one `ccpp_model_constituents_t` DDT per - host instance (see [§7](#7-multi-instance-design)) which holds the - constituent values (`%vars_layer`), tendencies (`%vars_layer_tend`), and - metadata (`%const_metadata`). -3. **Access** — schemes reference constituents by standard name in their - metadata; the resolver translates those references to - `ccpp_model_constituents_obj(inst)%vars_layer(slice, index_of_)` - subscripts at code-gen time. - -All constituent state lives in **one generated module**: -`ccpp_host_constituents.F90` (one per generator run, emitted only when at -least one suite touches constituent state). Public symbols from this module -are also re-exported by `ccpp_static_api`, so most host code only needs - -```fortran -use ccpp_static_api, only: ccpp_register_constituents, ccpp_initialize_constituents, & - ccpp_constituents_array, ccpp_const_get_index, ... -``` - ---- - -## 2. The four rules (scheme-author conventions) - -These four rules govern every scheme-arg metadata pattern related to -constituents. They derive from a 2026-05-11 audit of all 12 cam-sima scheme -metadata files that touch constituent attributes. - -### Rule 1 — Register a new constituent (register phase) - -A scheme that creates a new constituent declares it in the **register** -phase via an `intent=out, allocatable` array of -`ccpp_constituent_properties_t`: - -``` -[ccpp-arg-table] - name = my_scheme_register - type = scheme -[ dyn_const ] - standard_name = dynamic_constituents_for_my_scheme - long_name = per-scheme constituent array - units = none - dimensions = (:) - type = ccpp_constituent_properties_t - allocatable = True - intent = out -[ errmsg ] - ... -[ errflg ] - ... -``` - -The scheme's Fortran register routine `allocate`s this array, populates -each entry via `%instantiate(std_name=..., long_name=..., units=..., -vertical_dim=..., advected=..., ...)` and returns it. The framework -captures every register-phase scheme's array, packs them into a per-suite -buffer (`_dynamic_constituents`), and merges them into each -host-instance's constituent object during `ccpp_register_constituents`. - -This is the **only path** for declaring a new constituent. - -### Rule 2 — Consume a base constituent (any physics phase) - -A scheme that reads (or reads + writes) an existing base constituent -declares the variable with `is_constituent` set (any of `advected`, -`constituent`, or `molar_mass` non-default) and `intent=in` or `intent=inout`: - -``` -[ cldliq ] - standard_name = cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water - units = kg kg-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys - intent = in ! or inout - advected = true -``` - -The resolver translates this scheme arg to -`ccpp_model_constituents_obj()%vars_layer(, index_of_)` -in the generated group cap. No host metadata declaration is needed for -the variable. - -### Rule 3 — Produce a tendency (any physics phase) - -A scheme that writes a constituent tendency declares the variable with -`is_constituent` set, `intent=out`, and a standard name that **starts -with `tendency_of_`**: - -``` -[ tend_cldliq ] - standard_name = tendency_of_cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water - units = kg kg-1 s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys - intent = out - constituent = true -``` - -The resolver translates this scheme arg to -`ccpp_model_constituents_obj()%vars_layer_tend(, index_of_)` -where `` is the std_name with the `tendency_of_` prefix stripped. -The tendency variable is implicitly tied to the base constituent of the -same name. - -### Rule 4 — Mismatched combinations are hard errors - -Two combinations are explicitly rejected by the resolver at code-gen time: - -| Mismatch | Error | -|---|---| -| `is_constituent=True` + `intent=out` + std_name does NOT start with `tendency_of_` | *"Physics phases may only produce constituent tendencies; new base constituents must be declared via a `ccpp_constituent_properties_t` argument in a register-phase scheme."* | -| `is_constituent=True` + `intent in (in, inout)` + std_name starts with `tendency_of_` | *"Constituent tendency arg must be declared with intent=out; physics phases only produce tendencies, never consume them."* | - -### Direct framework-array access - -A scheme may also access the framework's bulk arrays directly by -declaring an arg with one of these standard names: - -| Standard name | Maps to | -|---|---| -| `ccpp_constituents` | `ccpp_model_constituents_obj()%vars_layer` (3D) | -| `ccpp_constituent_tendencies` | `ccpp_model_constituents_obj()%vars_layer_tend` (3D) | -| `ccpp_constituent_properties` | `ccpp_model_constituents_obj()%const_metadata` (1D of `ccpp_constituent_prop_ptr_t`) | -| `number_of_ccpp_constituents` | `ccpp_model_constituents_obj()%num_layer_vars` (scalar integer) | -| `index_of_` | module-level `integer :: index_of_` (no per-instance access — the index is identical for every instance) | - -The trailing dimension `number_of_ccpp_constituents` in a 3D scheme arg -is emitted as `:` (whole-axis slice). - ---- - -## 3. Required host metadata + Fortran - -### Host metadata (`type=host` table) - -The host **must** declare: - -``` -[ ] - standard_name = number_of_instances - units = count - dimensions = () - type = integer -``` - -… **only when the host actually wants multi-instance support**. When -absent, every per-instance allocation falls back to size `1` and the -host effectively runs single-instance. - -The host **does not** need to declare: - -- `ccpp_model_constituents_object` — the constituent object is owned - by the generator (in `ccpp_host_constituents`); the host doesn't - declare it in metadata. -- `ccpp_constituents`, `ccpp_constituent_tendencies`, - `ccpp_constituent_properties`, `number_of_ccpp_constituents`, - `index_of_` — all auto-provided by the generator. - -#### Host metadata wins over auto-provisioning - -If the host **does** declare any of the framework-named standard -names above as a regular host variable, the resolver uses the host's -declaration instead of auto-provisioning. This matters for legacy -hosts (GFS / SCM) that own their own tracer indices: - -```meta -[ ntcw ] - standard_name = index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array - units = index - type = integer - protected = True - dimensions = () -``` - -A scheme arg requesting the same standard name resolves to the host's -short local name (`ntcw`), not a parallel module-level integer in -`ccpp_host_constituents` named after the full standard name (which -would blow the Fortran 63-character identifier limit). Auto-provisioning -only fires for framework-named standard names the host has **not** -claimed. - -### Host control-table requirements - -The host's `type=control` table must declare: - -``` -[ ] - standard_name = instance_number - units = 1 - dimensions = () - type = integer -``` - -… so the framework signature knows the index for per-instance state. -Same caveat as `number_of_instances` — required only when multi-instance -is wanted. - -### Host Fortran code - -The host's Fortran code only needs to: - -1. Maintain its own `integer :: ` for `number_of_instances` - in a module that's USE'd by the generator. (Same module that owns - the metadata.) -2. Build its **host constituents** array (water vapor, ozone, etc. — - the constituents that the host model owns directly, separately from - any scheme-registered ones). Pass this to - `ccpp_register_constituents`. - -The host does **not** need to allocate or own a -`type(ccpp_model_constituents_t)` variable. - ---- - -## 4. Host-side lifecycle (call sequence) - -``` - ┌─ host startup ─┐ - │ - ▼ - ┌──────────────────────────────────────┐ - │ for each instance: │ - │ ccpp_register(suite_name, │ - │ errmsg, errflg, │ - │ instance_number) │ ─── per-instance ───┐ - └──────────────────────────────────────┘ │ - │ │ - ▼ │ - ┌──────────────────────────────────────┐ │ - │ allocate host_constituents(:) │ │ - │ host_constituents(1)%instantiate( │ ─── once ─────────┘ - │ std_name='water_vapor_specific_humidity', ...) │ - │ ... │ │ - └──────────────────────────────────────┘ │ - │ │ - ▼ │ - ┌──────────────────────────────────────┐ │ - │ for each instance: │ │ - │ ccpp_register_constituents( │ │ - │ host_constituents, │ │ - │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ - └──────────────────────────────────────┘ │ - │ │ - ▼ │ - ┌──────────────────────────────────────┐ │ - │ for each instance: │ │ - │ ccpp_initialize_constituents( │ │ - │ ncols, num_layers, │ │ - │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ - └──────────────────────────────────────┘ │ - │ │ - ▼ │ - ┌──────────────────────────────────────┐ │ - │ for each instance: │ │ - │ ccpp_init(suite_name, │ │ - │ errmsg, errflg, │ │ - │ instance_number) │ ─── per-instance ──┤ - └──────────────────────────────────────┘ │ - │ │ - ▼ (model time-stepping) │ - ┌──────────────────────────────────────┐ │ - │ ccpp_physics_*(...) │ ─── per-instance ──┤ - └──────────────────────────────────────┘ │ - │ │ - ▼ (host shutdown) │ - ┌──────────────────────────────────────┐ │ - │ for each instance: │ │ - │ ccpp_deallocate_dynamic_constituents( │ - │ instance_number) │ ─── per-instance ──┤ - │ ccpp_final(suite_name, │ │ - │ errmsg, errflg, │ │ - │ instance_number) │ │ - └──────────────────────────────────────┘ │ - │ - ┌────────────────────────────────────────┘ - │ ◀── last-to-leave dealloc fires - │ automatically inside the per-instance - │ calls when the final instance finishes. - ▼ -``` - -### Important ordering rules - -- `ccpp_register_constituents` **must** be called *after* `ccpp_register` - (per instance). The latter populates the per-suite dynamic-constituent - buffers via `_register`; the former merges them into the - per-instance constituent object. -- `ccpp_initialize_constituents` **must** be called *after* - `ccpp_register_constituents` (per instance). It calls `%lock_data` - on the per-instance object — which can only happen once - `%lock_table` has fired (which `ccpp_register_constituents` does). -- Physics phases (`ccpp_init`, `ccpp_physics_run`, etc.) require - the constituent state to be locked + bound (i.e., - `ccpp_initialize_constituents` already called). -- `ccpp_deallocate_dynamic_constituents` is per-instance with - last-to-leave teardown. Once the last instance calls it, the shared - per-suite buffers and the constituent object array are deallocated - automatically. - -### Built-in constituents vs scheme-registered constituents - -`ccpp_register_constituents` takes one explicit argument: an array of -`ccpp_constituent_properties_t` describing the **host's own constituents** -(typically water vapor and any other tracers the dycore carries -intrinsically). The framework then merges those entries with every -suite's per-suite dynamic-constituent buffer (populated during -`ccpp_register` from each register-phase scheme's output). - -Pass an empty (zero-size) array if the host has no built-in constituents -of its own. - ---- - -## 5. Public API reference - -All routines below live in `ccpp_host_constituents` and are also -re-exported from `ccpp_static_api` for convenience. The dummy-argument -name `instance_number` is the **standard name**; the actual emitted -dummy uses the host's local name for it (typically also -`instance_number` or `inst_num`). - -### `ccpp_register_constituents(host_constituents, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `host_constituents` | `type(ccpp_constituent_properties_t), target, intent(in) :: (:)` | Host-owned constituent declarations (water vapor, etc.). May be zero-size. | -| `instance_number` | `integer, intent(in)` | Per-instance index. | -| `errflg` | `integer, intent(out)` | Error flag (0 = success). | -| `errmsg` | `character(len=*), intent(out)` | Error message. | - -**Effect**: -- On the first call across instances, allocates - `ccpp_model_constituents_obj(number_of_instances)`. -- Calls `obj(instance_number)%initialize_table(num_consts)` where - `num_consts = size(host_constituents) + sum(size(_dynamic_constituents))`. -- Iterates `host_constituents` first, then every suite's - `_dynamic_constituents` buffer, calling - `obj(instance_number)%new_field(const_prop, errcode=errflg, errmsg=errmsg)` - for each entry. -- Calls `obj(instance_number)%lock_table(...)`. - -**Preconditions**: every `_register` call (across all suites) for -this instance has already happened (so the per-suite buffers are -populated). - -### `ccpp_initialize_constituents(ncols, num_layers, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `ncols` | `integer, intent(in)` | Horizontal extent for this instance's chunk. | -| `num_layers` | `integer, intent(in)` | Vertical layer count. | -| `instance_number` | `integer, intent(in)` | | -| `errflg` / `errmsg` | `intent(out)` | | - -**Effect**: -- Calls `obj(instance_number)%lock_data(ncols, num_layers, ...)` — - allocates `obj(inst)%vars_layer` and `%vars_layer_tend` arrays. -- Registers a singleton pointer with - `ccpp_scheme_utils.ccpp_initialize_constituent_ptr(obj(inst))` so - cam-sima schemes that call `ccpp_constituent_index` see the - constituent table. **First instance wins** — see - [§8 Limitations](#8-limitations-and-gotchas). -- Queries `obj(instance_number)%const_index(index_of_, '', ...)` for - every constituent `` known at code-gen time; populates the - module-level integer `index_of_`. These integers are identical - across instances; the last call to set them wins (benign — the - constituent table is the same per instance). - -**Preconditions**: `ccpp_register_constituents` has been called for this -instance. - -### `ccpp_is_scheme_constituent(var_name, constituent_exists, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `var_name` | `character(len=*), intent(in)` | Standard name to query. | -| `constituent_exists` | `logical, intent(out)` | True iff *var_name* matches one of the constituent std names known to capgen at code-gen time. | -| `errflg` / `errmsg` | `intent(out)` | | - -**No `instance_number`** — the data lookup is against the module-level -`character(len=N), parameter :: ccpp_model_const_stdnames(K)` array -(compile-time constant, identical across instances). - -### `ccpp_number_constituents(num_flds, advected, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `num_flds` | `integer, intent(out)` | Constituent count returned. | -| `advected` | `logical, optional, intent(in)` | If `.true.`, count advected only. | -| `instance_number` | `integer, intent(in)` | | -| `errflg` / `errmsg` | `intent(out)` | | - -Wraps `obj(instance_number)%num_constituents(num_flds, advected=advected, ...)`. - -> Even though every `obj(i)` returns the same count (registration is -> identical across instances), `instance_number` is part of the -> signature so the caller can guarantee they're querying an -> already-locked instance. Useful for hosts that lifecycle one -> instance at a time. - -### `ccpp_gather_constituents(const_array, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `const_array` | `real(kind=kind_phys), intent(out) :: (:,:,:)` | Destination buffer for constituent values. | -| `instance_number` | `integer, intent(in)` | | -| `errflg` / `errmsg` | `intent(out)` | | - -Wraps `obj(instance_number)%copy_in(const_array, ...)`. Use this to -pull the per-instance constituent values into a host-side array. - -### `ccpp_update_constituents(const_array, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `const_array` | `real(kind=kind_phys), intent(in) :: (:,:,:)` | Source buffer with updated constituent values. | -| `instance_number` | `integer, intent(in)` | | -| `errflg` / `errmsg` | `intent(out)` | | - -Wraps `obj(instance_number)%copy_out(const_array, ...)`. Use this to -push host-side updates back into the per-instance constituent object. - -### `ccpp_const_get_index(stdname, const_index, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `stdname` | `character(len=*), intent(in)` | Constituent standard name to look up. | -| `const_index` | `integer, intent(out)` | Returned index into the constituent array (or `int_unassigned` on miss). | -| `instance_number` | `integer, intent(in)` | | -| `errflg` / `errmsg` | `intent(out)` | | - -Wraps `obj(instance_number)%const_index(standard_name=stdname, -index=const_index, ...)`. For constituents whose std names are known -at code-gen time, prefer using the module-level `index_of_` integer -directly (no call needed; it's bound during -`ccpp_initialize_constituents`). - -### `ccpp_constituents_array(instance_number) result(const_ptr)` - -Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → -`obj(instance_number)%field_data_ptr()`. - -### `ccpp_advected_constituents_array(instance_number) result(const_ptr)` - -Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → -`obj(instance_number)%advected_constituents_ptr()`. Subset of the -full constituent array containing only those flagged `advected=.true.`. - -### `ccpp_model_const_properties(instance_number) result(const_ptr)` - -Returns `type(ccpp_constituent_prop_ptr_t), pointer :: const_ptr(:)` → -`obj(instance_number)%constituent_props_ptr()`. - -### `ccpp_deallocate_dynamic_constituents(instance_number)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `instance_number` | `integer, intent(in)` | | - -**Per-instance reset + last-to-leave teardown**: -1. `obj(instance_number)%reset()` — unlocks the table for this instance. -2. Iterates every `obj(i)` and checks `%const_props_locked()`. If any - instance is still locked, the routine returns. -3. If **every** instance has been reset (none still locked), the routine - tears down the shared state: - - Deallocates every `_dynamic_constituents` buffer. - - Deallocates `ccpp_model_constituents_obj(:)`. - - Resets every `index_of_` integer to 0. - -The host should call this for every instance that successfully called -`ccpp_register_constituents`. - ---- - -## 6. Generated code structure - -When any suite touches constituent state, capgen-ng emits one extra -module per generator run: **`ccpp_host_constituents.F90`**. - -### Module declarations - -```fortran -module ccpp_host_constituents - use ccpp_kinds, only: kind_phys - use ccpp_constituent_prop_mod, only: & - ccpp_model_constituents_t, & - ccpp_constituent_properties_t, & - ccpp_constituent_prop_ptr_t - - implicit none - private - - ! ----- public state ---------------------------------------------------- - public :: ccpp_model_constituents_obj - public :: index_of_ ! one per known constituent std name - public :: index_of_ - public :: ccpp_model_const_stdnames ! parameter array - - ! ----- public routines (also re-exported from ccpp_static_api) -------- - public :: ccpp_register_constituents - public :: ccpp_initialize_constituents - public :: ccpp_is_scheme_constituent - public :: ccpp_number_constituents - public :: ccpp_gather_constituents - public :: ccpp_update_constituents - public :: ccpp_const_get_index - public :: ccpp_constituents_array - public :: ccpp_advected_constituents_array - public :: ccpp_model_const_properties - public :: ccpp_deallocate_dynamic_constituents - public :: _dynamic_constituents ! one per suite with register-phase producers - public :: _dynamic_constituents - - ! ----- module-level state --------------------------------------------- - type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) - type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) - type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) - integer :: index_of_ = 0 - integer :: index_of_ = 0 - character(len=N), parameter :: ccpp_model_const_stdnames(K) = (/ & - ' ', & - ' ' /) - -contains - ! ... routines as documented in §5 ... -end module ccpp_host_constituents -``` - -### Suite-cap responsibilities - -`ccpp__cap.F90` does NOT own constituent state. Its -`_register` routine packs each register-phase scheme's -constituent array into the suite's `_dynamic_constituents` -buffer (USE'd from `ccpp_host_constituents`): - -```fortran -if (.not. allocated(_dynamic_constituents)) then - ! First-instance-only two-pass count + populate. - num_consts = 0 - call _register(scheme_consts=scheme_consts, ...) - num_consts = num_consts + size(scheme_consts, 1) - deallocate(scheme_consts) - ... - allocate(_dynamic_constituents(num_consts)) - num_consts = 0 - call _register(scheme_consts=scheme_consts, ...) - do i = 1, size(scheme_consts, 1) - _dynamic_constituents(num_consts + i) = scheme_consts(i) - end do - num_consts = num_consts + size(scheme_consts, 1) - deallocate(scheme_consts) - ... -end if -``` - -The buffer is **shared across instances** (registration is identical -per instance); only the first instance to call `_register` -populates it. The host-wide merge happens in -`ccpp_register_constituents`. - -### Group-cap call sites - -`ccpp___cap.F90` USE's the constituent symbols it needs -from `ccpp_host_constituents`: - -```fortran -use ccpp_host_constituents, only: ccpp_model_constituents_obj, & - index_of_cloud_liquid_water_mixing_ratio -``` - -… and emits scheme call sites with the per-instance access expression: - -```fortran -call cld_liq_run( & - ... - cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer(lb:ub, 1:nlev, & - index_of_cloud_liquid_water_mixing_ratio), & - tend_cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer_tend(lb:ub, 1:nlev, & - index_of_cloud_liquid_water_mixing_ratio), & - ...) -``` - -The `instance_number` dummy is auto-injected into the group-cap -subroutine signatures by `_extra_dim_ctrl_entries` because the -resolver adds `instance_number` to every constituent arg's -`used_dim_std_names`. - -### Framework F90 dependencies - -`ccpp_host_constituents.F90` and the suite caps depend on these -framework files (listed under `` in `datatable.xml`): - -| File | Why | -|---|---| -| `ccpp_constituent_prop_mod.F90` | Provides `ccpp_model_constituents_t`, `ccpp_constituent_properties_t`, `ccpp_constituent_prop_ptr_t`. | -| `ccpp_hashable.F90` | Transitive dep of `ccpp_constituent_prop_mod`. | -| `ccpp_hash_table.F90` | Transitive dep. | -| `ccpp_scheme_utils.F90` | Provides `ccpp_initialize_constituent_ptr` + `ccpp_constituent_index` (used by cam-sima rrtmgp / mmm schemes). | - -The host's CMake should query `ccpp_datafile.py --utility-files` to -get the absolute paths to these files at the right output location. - ---- - -## 7. Multi-instance design - -In capgen-ng, **per-instance state** means: each "instance" (typically -an OpenMP team / chunk-domain partition) has its own copy of the -state arrays, indexed by `instance_number ∈ [1, number_of_instances]`. - -### What's per-instance - -| State | Storage | -|---|---| -| Constituent values + tendencies | `ccpp_model_constituents_obj(:)` — one DDT per instance | -| Suite-level state machine | `ccpp_suite_state(:)` — declared in each suite cap | -| Suite-owned data | `ccpp_suite_data(:)` — declared in each suite-data module | -| Group-level state machine | `ccpp_group_state(:)` — declared in each group cap | - -### What's shared across instances - -| State | Reason | -|---|---| -| `_dynamic_constituents(:)` per-suite buffers | Registration is identical per instance — populated by the first instance to call `_register` and reused by the rest | -| `index_of_` integers | The constituent table is identical per instance, so the indices are too | -| `ccpp_model_const_stdnames` parameter array | Compile-time constant | - -### Sizing - -`number_of_instances` is the single source of truth. The host declares -it in metadata + Fortran; the generator USE's it from the host module -wherever per-instance allocation happens. See the prior memo -[*Where the total number of instances comes from*](#) for the call -chain (and matching values across all four state arrays: -`ccpp_suite_state`, `ccpp_suite_data`, `ccpp_group_state`, -`ccpp_model_constituents_obj`). - -If the host doesn't declare `number_of_instances`, every per-instance -allocation falls back to `1` and the framework runs single-instance. - -### Two host-side lifecycle patterns - -Both work; pick whichever fits your model. - -**Pattern A: all instances registered first** -``` -do isuite = 1, num_suites - do iinst = 1, num_instances - call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) - end do -end do -do iinst = 1, num_instances - call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) - call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) -end do -do isuite = 1, num_suites - do iinst = 1, num_instances - call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) - end do -end do -! ... time-stepping ... -do iinst = 1, num_instances - call ccpp_deallocate_dynamic_constituents(iinst) - ... -end do -``` - -**Pattern B: serial per instance** -``` -do iinst = 1, num_instances - do isuite = 1, num_suites - call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) - end do - call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) - call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) - do isuite = 1, num_suites - call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) - end do - ! ... per-instance time-stepping ... - call ccpp_deallocate_dynamic_constituents(iinst) -end do -``` - -### Last-to-leave teardown - -`ccpp_deallocate_dynamic_constituents(inst)`: -1. Per-instance `obj(inst)%reset()`. -2. Iterates every `obj(i)`; if any has `%const_props_locked() == .true.`, - returns early. -3. Otherwise (every instance reset): deallocates the shared per-suite - buffers, deallocates `ccpp_model_constituents_obj(:)`, and zeros every - `index_of_` integer. - -This works for both lifecycle patterns above. - ---- - -## 8. Limitations and gotchas - -> **Note (2026-05-12).** Several items in this section are under active -> discussion for an upcoming framework + generator overhaul. See -> `doc/constituents_overhaul.md` for the full architectural review and -> three reform proposals. - -### Framework property ownership (2026-05-12) - -The framework's `ccpp_constituent_properties_t` now carries a private -`framework_owns_me` flag (default `.false.`) with -`is_framework_owned()` getter and `set_framework_owned(value)` setter. -`ccpt_deallocate` only deallocates the underlying prop when the flag -is `.true.`; otherwise it just nullifies its pointer. - -Under capgen-ng's explicit-registration model, all -`ccpp_constituent_properties_t` objects are **target-owned by the -caller** (the host's `host_constituents(:)` array, or the per-suite -`_dynamic_constituents(:)` buffer). We never set the flag, so -the framework correctly skips deallocation. Hosts that hand-allocate -property objects on the heap and want the framework to free them must -call `set_framework_owned(.true.)` before passing to `%new_field`. - -### Missing setters (framework gap) - -The framework lacks setters for `advected`, `diagnostic_name`, -`default_value` (and `mixing_ratio_type`). This means once a -constituent is `%instantiate`d, those properties cannot be changed. -If your host needs to override a scheme-supplied `diagnostic_name` or -`advected` value, you currently cannot — open item in the constituents -overhaul proposal. - -### `ccpp_scheme_utils` singleton - -`ccpp_scheme_utils.ccpp_initialize_constituent_ptr` accepts only one -singleton pointer. It's a framework-level convenience used by cam-sima -schemes that call `ccpp_constituent_index` from `ccpp_scheme_utils`. - -`ccpp_initialize_constituents` calls -`ccpp_initialize_constituent_ptr(obj(inst))` on each instance, but -**only the first call across instances actually sets the pointer** -(the routine is internally guarded by an `initialized` flag). -Subsequent calls are silent no-ops. - -For multi-instance hosts, schemes that use -`ccpp_scheme_utils.ccpp_constituent_index` will see only the first -instance's object — a known limitation inherited from the framework -module's design. Schemes that use the per-instance accessors -(`obj(inst)%const_index(...)` via `ccpp_const_get_index`) are -unaffected. - -### Constituent metadata is identical across instances - -The constituent table (which constituents exist, their properties, the -`index_of_` mapping) is **identical** for every instance. Every -instance's `obj(i)` has the same hash table, populated identically by -its own `ccpp_register_constituents` call. - -This means: - -- `ccpp_number_constituents` returns the same value regardless of - `instance_number`. -- `ccpp_const_get_index` returns the same index regardless of - `instance_number`. -- The `index_of_` integers are populated identically by every - instance's `ccpp_initialize_constituents` (last-write-wins is fine - since every write is the same value). - -`instance_number` is still in the signatures of these routines — see -[§5](#5-public-api-reference) for the rationale. - -### Forbidden patterns recap - -These are rejected at code-gen time (Rule 4 of [§2](#2-the-four-rules-scheme-author-conventions)): - -- `is_constituent + intent=out + non-tendency std_name` — physics phases - may only produce tendencies, not new base constituents. -- `is_constituent + intent=in/inout + tendency_of_*` — tendencies are - write-only. - -### Subscript indices in sliced local_names must be standard names - -If a host metadata variable is declared with a sliced local name -like `q(:,:,index_of_water_vapor_specific_humidity)`, every subscript -token (other than `:` and integer literals) must be a known standard -name. Otherwise the resolver raises a `CCPPError` with a clear -message naming the offending token. - -### Open work items - -- **Unconditional `ccpp_host_constituents.F90` emission.** The - generator currently emits `ccpp_host_constituents.F90` for every - build, even when no scheme or host actually uses the constituent - system (no `ccpp_constituent_properties_t(:)` register-phase arg, - no `is_constituent`-flagged scheme arg, no framework-named - `index_of_` / `ccpp_constituents` / etc. claimed by capgen-ng). - When the host owns its own indices (SCM/GFS) and no scheme exercises - the constituent path, the generated file is dead code that should be - suppressed. Tracked as a deferred item; the `host_dict` precedence - rule above already keeps the file *correct* (empty) in that case. - ---- - -## 9. Differences from original capgen - -| Aspect | Original capgen | capgen-ng | -|---|---|---| -| Constituent object location | Generated `_ccpp_cap.F90` module | `ccpp_host_constituents.F90` (one per generator run) | -| Per-instance | No (single instance) | Yes (`obj(:)` allocatable, sized to `number_of_instances`) | -| Routine name prefix | `_ccpp_register_constituents`, etc. | `ccpp_register_constituents`, etc. (no host prefix; one set per generator run) | -| Routine signatures | No `instance_number` arg | `instance_number` in every per-instance routine | -| Host metadata for constituent obj | None (auto-created by generator) | None (still auto-created by generator) | -| Module-level pointers | `_constituents_array` etc. as functions returning pointers | Same idea, now per-instance via `instance_number` arg | -| Scheme std-name set | `_model_const_stdnames` parameter array | `ccpp_model_const_stdnames` parameter array (no host prefix) | -| Host-facing API surface | `_ccpp_register_constituents`, `_ccpp_initialize_constituents`, `_ccpp_number_constituents`, `_ccpp_is_scheme_constituent`, `_ccpp_gather_constituents`, `_ccpp_update_constituents`, `_ccpp_deallocate_dynamic_constituents`, `_constituents_array`, `_advected_constituents_array`, `_model_const_properties`, `_const_get_index` | Same surface, no `_` prefix | -| Dynamic constituent buffer dimensionality | 1D, per host | 1D, per generator run, **shared across instances** | -| Static suite constituents | Auto-cloned by `ConstituentVarDict.find_variable` and registered via `_constituents_copy_const` accessors | Tracked at code-gen time via `is_constituent` flag; included in the constituent table only if a register-phase scheme produces them (rule 1). Schemes that *consume* a base constituent (rule 2) don't trigger registration — the constituent must be registered by SOMEONE (host or another scheme's register) for the access to work at runtime. | - -### Migration notes for cam-sima hosts - -- **Scheme metadata**: no changes needed. Cam-sima follows rules 2 and - 3 already (audited 2026-05-11). The 4 schemes that register - constituents via `ccpp_constituent_properties_t` (rule 1) work - unchanged. -- **Host metadata**: drop any explicit declaration of - `ccpp_model_constituents_object` if you carried one over from a - previous capgen-ng experiment — the generator owns it now. -- **Host Fortran**: change all `_ccpp_*_constituents` calls to - the unprefixed names (`ccpp_register_constituents` etc.) and add - `instance_number` to every call site. - ---- - -## 10. Worked example - -A minimal cam-sima-style suite with one scheme that consumes a base -constituent and produces its tendency. - -### Scheme metadata (`consume_constituent.meta`) - -``` -[ccpp-table-properties] - name = consume_constituent - type = scheme - -[ccpp-arg-table] - name = consume_constituent_run - type = scheme -[ cldliq ] - standard_name = cloud_liquid_water_mixing_ratio - units = kg kg-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys - intent = in - advected = .true. -[ tend_cldliq ] - standard_name = tendency_of_cloud_liquid_water_mixing_ratio - units = kg kg-1 s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys - intent = out - constituent = .true. -[ errmsg ] - ... -[ errflg ] - ... -``` - -### Host metadata (`my_host.meta`) - -``` -[ccpp-table-properties] - name = my_host - type = host - -[ccpp-arg-table] - name = my_host - type = host -[ ncols ] - standard_name = horizontal_dimension - units = count - dimensions = () - type = integer -[ nlev ] - standard_name = vertical_layer_dimension - units = count - dimensions = () - type = integer -[ ninstances ] - standard_name = number_of_instances - units = count - dimensions = () - type = integer -``` - -(Plus a `type=control` table declaring `instance_number`, -`horizontal_loop_begin`, `horizontal_loop_end`, `ccpp_error_message`, -`ccpp_error_code`, etc.) - -### Suite XML (`my_suite.xml`) - -```xml - - - - consume_constituent - - -``` - -### Generated `ccpp_host_constituents.F90` (excerpt) - -```fortran -module ccpp_host_constituents - use ccpp_kinds, only: kind_phys - use ccpp_constituent_prop_mod, only: & - ccpp_model_constituents_t, ccpp_constituent_properties_t, & - ccpp_constituent_prop_ptr_t - - implicit none - private - - public :: ccpp_model_constituents_obj - public :: index_of_cloud_liquid_water_mixing_ratio - public :: ccpp_register_constituents, ccpp_initialize_constituents - public :: ccpp_is_scheme_constituent, ccpp_number_constituents - public :: ccpp_gather_constituents, ccpp_update_constituents - public :: ccpp_const_get_index, ccpp_constituents_array - public :: ccpp_advected_constituents_array, ccpp_model_const_properties - public :: ccpp_deallocate_dynamic_constituents - public :: ccpp_model_const_stdnames - - type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) - integer :: index_of_cloud_liquid_water_mixing_ratio = 0 - character(len=31), parameter :: ccpp_model_const_stdnames(1) = (/ & - 'cloud_liquid_water_mixing_ratio' /) - -contains - ! ... full subroutine bodies as in §5 ... -end module ccpp_host_constituents -``` - -### Host code skeleton (single-instance illustration) - -```fortran -subroutine my_host_run() - use ccpp_static_api, only: ccpp_register, ccpp_register_constituents, & - ccpp_initialize_constituents, ccpp_init, & - ccpp_physics_run, ccpp_final, & - ccpp_deallocate_dynamic_constituents - use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t - - type(ccpp_constituent_properties_t), allocatable :: host_consts(:) - integer :: errflg - character(len=512) :: errmsg - integer, parameter :: inst = 1 - - ! 1. Run register phase: populates per-suite dynamic-constituent buffers. - call ccpp_register('my_suite', errmsg, errflg, inst) - - ! 2. Build host's own constituent declarations (water vapor, etc.). - allocate(host_consts(1)) - call host_consts(1)%instantiate( & - std_name='water_vapor_specific_humidity', long_name='water vapor', & - units='kg kg-1', vertical_dim='vertical_layer_dimension', & - advected=.true., errcode=errflg, errmsg=errmsg) - - ! 3. Merge host + suite-side constituents into obj(inst). - call ccpp_register_constituents(host_consts, inst, errflg, errmsg) - - ! 4. Allocate vars_layer + bind cached indices. - call ccpp_initialize_constituents(ncols, nlev, inst, errflg, errmsg) - - ! 5. Framework init phase. - call ccpp_init('my_suite', errmsg, errflg, inst) - - ! 6. Time-stepping (omitted). - call ccpp_physics_run('my_suite', 'phys', col_start, col_end, & - thread_num, nthreads, nphys_threads, & - errflg, errmsg, inst) - - ! 7. Shutdown. - call ccpp_final('my_suite', errmsg, errflg, inst) - call ccpp_deallocate_dynamic_constituents(inst) - deallocate(host_consts) -end subroutine my_host_run -``` - -For multi-instance, wrap each per-instance call in -`do iinst = 1, ninstances ... end do` per the patterns in -[§7](#7-multi-instance-design). diff --git a/doc/constituents_20260519T0905.md b/doc/constituents_20260519T0905.md deleted file mode 100644 index c7581801..00000000 --- a/doc/constituents_20260519T0905.md +++ /dev/null @@ -1,1029 +0,0 @@ -# CCPP capgen-ng — Constituents Reference - -*Last revised: 2026-05-13.* - -This document is the authoritative reference for **constituent variables** in -capgen-ng — what they are, how scheme authors declare them in metadata, what -the host model has to do to plumb them through, what the generator emits, and -how the per-instance lifecycle works. - -> If you are migrating a host or scheme from the original capgen, jump to -> [§9 Differences from original capgen](#9-differences-from-original-capgen) -> first. - ---- - -## Table of Contents - -1. [What is a constituent?](#1-what-is-a-constituent) -2. [The four rules (scheme-author conventions)](#2-the-four-rules-scheme-author-conventions) -3. [Required host metadata + Fortran](#3-required-host-metadata--fortran) -4. [Host-side lifecycle (call sequence)](#4-host-side-lifecycle-call-sequence) -5. [Public API reference](#5-public-api-reference) -6. [Generated code structure](#6-generated-code-structure) -7. [Multi-instance design](#7-multi-instance-design) -8. [Limitations and gotchas](#8-limitations-and-gotchas) -9. [Differences from original capgen](#9-differences-from-original-capgen) -10. [Worked example](#10-worked-example) - ---- - -## 1. What is a constituent? - -A **constituent** is a model variable owned by the host's dynamical core (or -its constituent infrastructure) that is read and updated by physics schemes — -typically a tracer / mass mixing ratio (water vapor, cloud liquid, ozone, -chemistry species) — together with its **tendency**, the rate of change that -physics writes back so the dycore can advect/integrate it forward. - -In capgen-ng, the constituent layer has three concerns: - -1. **Registration** — declaring at model startup which constituents exist - (their standard name, units, vertical layout, advection flag, …). -2. **Storage** — the framework owns one `ccpp_model_constituents_t` DDT per - host instance (see [§7](#7-multi-instance-design)) which holds the - constituent values (`%vars_layer`), tendencies (`%vars_layer_tend`), and - metadata (`%const_metadata`). -3. **Access** — schemes reference constituents by standard name in their - metadata; the resolver translates those references to - `ccpp_model_constituents_obj(inst)%vars_layer(slice, index_of_)` - subscripts at code-gen time. - -All constituent state lives in **one generated module**: -`ccpp_host_constituents.F90` (one per generator run, emitted only when at -least one suite touches constituent state). Public symbols from this module -are also re-exported by `ccpp_static_api`, so most host code only needs - -```fortran -use ccpp_static_api, only: ccpp_register_constituents, ccpp_initialize_constituents, & - ccpp_constituents_array, ccpp_const_get_index, ... -``` - ---- - -## 2. The four rules (scheme-author conventions) - -These four rules govern every scheme-arg metadata pattern related to -constituents. They derive from a 2026-05-11 audit of all 12 cam-sima scheme -metadata files that touch constituent attributes. - -### Rule 1 — Register a new constituent (register phase) - -A scheme that creates a new constituent declares it in the **register** -phase via an `intent=out, allocatable` array of -`ccpp_constituent_properties_t`: - -``` -[ccpp-arg-table] - name = my_scheme_register - type = scheme -[ dyn_const ] - standard_name = dynamic_constituents_for_my_scheme - long_name = per-scheme constituent array - units = none - dimensions = (:) - type = ccpp_constituent_properties_t - allocatable = True - intent = out -[ errmsg ] - ... -[ errflg ] - ... -``` - -The scheme's Fortran register routine `allocate`s this array, populates -each entry via `%instantiate(std_name=..., long_name=..., units=..., -vertical_dim=..., advected=..., ...)` and returns it. The framework -captures every register-phase scheme's array, packs them into a per-suite -buffer (`_dynamic_constituents`), and merges them into each -host-instance's constituent object during `ccpp_register_constituents`. - -This is the **only path** for declaring a new constituent. - -### Rule 2 — Consume a base constituent (any physics phase) - -A scheme that reads (or reads + writes) an existing base constituent -declares the variable with `is_constituent` set (any of `advected`, -`constituent`, or `molar_mass` non-default) and `intent=in` or `intent=inout`: - -``` -[ cldliq ] - standard_name = cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water - units = kg kg-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys - intent = in ! or inout - advected = true -``` - -The resolver translates this scheme arg to -`ccpp_model_constituents_obj()%vars_layer(, index_of_)` -in the generated group cap. No host metadata declaration is needed for -the variable. - -### Rule 3 — Produce a tendency (any physics phase) - -A scheme that writes a constituent tendency declares the variable with -`is_constituent` set, `intent=out`, and a standard name that **starts -with `tendency_of_`**: - -``` -[ tend_cldliq ] - standard_name = tendency_of_cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water - units = kg kg-1 s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys - intent = out - constituent = true -``` - -The resolver translates this scheme arg to -`ccpp_model_constituents_obj()%vars_layer_tend(, index_of_)` -where `` is the std_name with the `tendency_of_` prefix stripped. -The tendency variable is implicitly tied to the base constituent of the -same name. - -### Rule 4 — Mismatched combinations are hard errors - -Two combinations are explicitly rejected by the resolver at code-gen time: - -| Mismatch | Error | -|---|---| -| `is_constituent=True` + `intent=out` + std_name does NOT start with `tendency_of_` | *"Physics phases may only produce constituent tendencies; new base constituents must be declared via a `ccpp_constituent_properties_t` argument in a register-phase scheme."* | -| `is_constituent=True` + `intent in (in, inout)` + std_name starts with `tendency_of_` | *"Constituent tendency arg must be declared with intent=out; physics phases only produce tendencies, never consume them."* | - -### Direct framework-array access - -A scheme may also access the framework's bulk arrays directly by -declaring an arg with one of these standard names: - -| Standard name | Maps to | -|---|---| -| `ccpp_constituents` | `ccpp_model_constituents_obj()%vars_layer` (3D) | -| `ccpp_constituent_tendencies` | `ccpp_model_constituents_obj()%vars_layer_tend` (3D) | -| `ccpp_constituent_properties` | `ccpp_model_constituents_obj()%const_metadata` (1D of `ccpp_constituent_prop_ptr_t`) | -| `number_of_ccpp_constituents` | `ccpp_model_constituents_obj()%num_layer_vars` (scalar integer) | -| `index_of_` | module-level `integer :: index_of_` (no per-instance access — the index is identical for every instance) | - -The trailing dimension `number_of_ccpp_constituents` in a 3D scheme arg -is emitted as `:` (whole-axis slice). - ---- - -## 3. Required host metadata + Fortran - -### Host metadata (`type=host` table) - -The host **must** declare: - -``` -[ ] - standard_name = number_of_instances - units = count - dimensions = () - type = integer -``` - -… **only when the host actually wants multi-instance support**. When -absent, every per-instance allocation falls back to size `1` and the -host effectively runs single-instance. - -The host **does not** need to declare: - -- `ccpp_model_constituents_object` — the constituent object is owned - by the generator (in `ccpp_host_constituents`); the host doesn't - declare it in metadata. -- `ccpp_constituents`, `ccpp_constituent_tendencies`, - `ccpp_constituent_properties`, `number_of_ccpp_constituents`, - `index_of_` — all auto-provided by the generator. - -#### Host metadata wins over auto-provisioning - -If the host **does** declare any of the framework-named standard -names above as a regular host variable, the resolver uses the host's -declaration instead of auto-provisioning. This matters for legacy -hosts (GFS / SCM) that own their own tracer indices: - -```meta -[ ntcw ] - standard_name = index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array - units = index - type = integer - protected = True - dimensions = () -``` - -A scheme arg requesting the same standard name resolves to the host's -short local name (`ntcw`), not a parallel module-level integer in -`ccpp_host_constituents` named after the full standard name (which -would blow the Fortran 63-character identifier limit). Auto-provisioning -only fires for framework-named standard names the host has **not** -claimed. - -### Host control-table requirements - -The host's `type=control` table must declare: - -``` -[ ] - standard_name = instance_number - units = 1 - dimensions = () - type = integer -``` - -… so the framework signature knows the index for per-instance state. -Same caveat as `number_of_instances` — required only when multi-instance -is wanted. - -### Host Fortran code - -The host's Fortran code only needs to: - -1. Maintain its own `integer :: ` for `number_of_instances` - in a module that's USE'd by the generator. (Same module that owns - the metadata.) -2. Build its **host constituents** array (water vapor, ozone, etc. — - the constituents that the host model owns directly, separately from - any scheme-registered ones). Pass this to - `ccpp_register_constituents`. - -The host does **not** need to allocate or own a -`type(ccpp_model_constituents_t)` variable. - ---- - -## 4. Host-side lifecycle (call sequence) - -``` - ┌─ host startup ─┐ - │ - ▼ - ┌──────────────────────────────────────┐ - │ for each instance: │ - │ ccpp_register(suite_name, │ - │ errmsg, errflg, │ - │ instance_number) │ ─── per-instance ───┐ - └──────────────────────────────────────┘ │ - │ │ - ▼ │ - ┌──────────────────────────────────────┐ │ - │ allocate host_constituents(:) │ │ - │ host_constituents(1)%instantiate( │ ─── once ─────────┘ - │ std_name='water_vapor_specific_humidity', ...) │ - │ ... │ │ - └──────────────────────────────────────┘ │ - │ │ - ▼ │ - ┌──────────────────────────────────────┐ │ - │ for each instance: │ │ - │ ccpp_register_constituents( │ │ - │ host_constituents, │ │ - │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ - └──────────────────────────────────────┘ │ - │ │ - ▼ │ - ┌──────────────────────────────────────┐ │ - │ for each instance: │ │ - │ ccpp_initialize_constituents( │ │ - │ ncols, num_layers, │ │ - │ instance_number, errflg, errmsg) │ ─── per-instance ──┤ - └──────────────────────────────────────┘ │ - │ │ - ▼ │ - ┌──────────────────────────────────────┐ │ - │ for each instance: │ │ - │ ccpp_init(suite_name, │ │ - │ errmsg, errflg, │ │ - │ instance_number) │ ─── per-instance ──┤ - └──────────────────────────────────────┘ │ - │ │ - ▼ (model time-stepping) │ - ┌──────────────────────────────────────┐ │ - │ ccpp_physics_*(...) │ ─── per-instance ──┤ - └──────────────────────────────────────┘ │ - │ │ - ▼ (host shutdown) │ - ┌──────────────────────────────────────┐ │ - │ for each instance: │ │ - │ ccpp_deallocate_dynamic_constituents( │ - │ instance_number) │ ─── per-instance ──┤ - │ ccpp_final(suite_name, │ │ - │ errmsg, errflg, │ │ - │ instance_number) │ │ - └──────────────────────────────────────┘ │ - │ - ┌────────────────────────────────────────┘ - │ ◀── last-to-leave dealloc fires - │ automatically inside the per-instance - │ calls when the final instance finishes. - ▼ -``` - -### Important ordering rules - -- `ccpp_register_constituents` **must** be called *after* `ccpp_register` - (per instance). The latter populates the per-suite dynamic-constituent - buffers via `_register`; the former merges them into the - per-instance constituent object. -- `ccpp_initialize_constituents` **must** be called *after* - `ccpp_register_constituents` (per instance). It calls `%lock_data` - on the per-instance object — which can only happen once - `%lock_table` has fired (which `ccpp_register_constituents` does). -- Physics phases (`ccpp_init`, `ccpp_physics_run`, etc.) require - the constituent state to be locked + bound (i.e., - `ccpp_initialize_constituents` already called). -- `ccpp_deallocate_dynamic_constituents` is per-instance with - last-to-leave teardown. Once the last instance calls it, the shared - per-suite buffers and the constituent object array are deallocated - automatically. - -### Built-in constituents vs scheme-registered constituents - -`ccpp_register_constituents` takes one explicit argument: an array of -`ccpp_constituent_properties_t` describing the **host's own constituents** -(typically water vapor and any other tracers the dycore carries -intrinsically). The framework then merges those entries with every -suite's per-suite dynamic-constituent buffer (populated during -`ccpp_register` from each register-phase scheme's output). - -Pass an empty (zero-size) array if the host has no built-in constituents -of its own. - ---- - -## 5. Public API reference - -All routines below live in `ccpp_host_constituents` and are also -re-exported from `ccpp_static_api` for convenience. The dummy-argument -name `instance_number` is the **standard name**; the actual emitted -dummy uses the host's local name for it (typically also -`instance_number` or `inst_num`). - -### `ccpp_register_constituents(host_constituents, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `host_constituents` | `type(ccpp_constituent_properties_t), target, intent(in) :: (:)` | Host-owned constituent declarations (water vapor, etc.). May be zero-size. | -| `instance_number` | `integer, intent(in)` | Per-instance index. | -| `errflg` | `integer, intent(out)` | Error flag (0 = success). | -| `errmsg` | `character(len=*), intent(out)` | Error message. | - -**Effect**: -- On the first call across instances, allocates - `ccpp_model_constituents_obj(number_of_instances)`. -- Calls `obj(instance_number)%initialize_table(num_consts)` where - `num_consts = size(host_constituents) + sum(size(_dynamic_constituents))`. -- Iterates `host_constituents` first, then every suite's - `_dynamic_constituents` buffer, calling - `obj(instance_number)%new_field(const_prop, errcode=errflg, errmsg=errmsg)` - for each entry. -- Calls `obj(instance_number)%lock_table(...)`. - -**Preconditions**: every `_register` call (across all suites) for -this instance has already happened (so the per-suite buffers are -populated). - -### `ccpp_initialize_constituents(ncols, num_layers, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `ncols` | `integer, intent(in)` | Horizontal extent for this instance's chunk. | -| `num_layers` | `integer, intent(in)` | Vertical layer count. | -| `instance_number` | `integer, intent(in)` | | -| `errflg` / `errmsg` | `intent(out)` | | - -**Effect**: -- Calls `obj(instance_number)%lock_data(ncols, num_layers, ...)` — - allocates `obj(inst)%vars_layer` and `%vars_layer_tend` arrays. -- Registers a singleton pointer with - `ccpp_scheme_utils.ccpp_initialize_constituent_ptr(obj(inst))` so - cam-sima schemes that call `ccpp_constituent_index` see the - constituent table. **First instance wins** — see - [§8 Limitations](#8-limitations-and-gotchas). -- Queries `obj(instance_number)%const_index(index_of_, '', ...)` for - every constituent `` known at code-gen time; populates the - module-level integer `index_of_`. These integers are identical - across instances; the last call to set them wins (benign — the - constituent table is the same per instance). - -**Preconditions**: `ccpp_register_constituents` has been called for this -instance. - -### `ccpp_is_scheme_constituent(var_name, constituent_exists, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `var_name` | `character(len=*), intent(in)` | Standard name to query. | -| `constituent_exists` | `logical, intent(out)` | True iff *var_name* matches one of the constituent std names known to capgen at code-gen time. | -| `errflg` / `errmsg` | `intent(out)` | | - -**No `instance_number`** — the data lookup is against the module-level -`character(len=N), parameter :: ccpp_model_const_stdnames(K)` array -(compile-time constant, identical across instances). - -### `ccpp_number_constituents(num_flds, advected, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `num_flds` | `integer, intent(out)` | Constituent count returned. | -| `advected` | `logical, optional, intent(in)` | If `.true.`, count advected only. | -| `instance_number` | `integer, intent(in)` | | -| `errflg` / `errmsg` | `intent(out)` | | - -Wraps `obj(instance_number)%num_constituents(num_flds, advected=advected, ...)`. - -> Even though every `obj(i)` returns the same count (registration is -> identical across instances), `instance_number` is part of the -> signature so the caller can guarantee they're querying an -> already-locked instance. Useful for hosts that lifecycle one -> instance at a time. - -### `ccpp_gather_constituents(const_array, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `const_array` | `real(kind=kind_phys), intent(out) :: (:,:,:)` | Destination buffer for constituent values. | -| `instance_number` | `integer, intent(in)` | | -| `errflg` / `errmsg` | `intent(out)` | | - -Wraps `obj(instance_number)%copy_in(const_array, ...)`. Use this to -pull the per-instance constituent values into a host-side array. - -### `ccpp_update_constituents(const_array, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `const_array` | `real(kind=kind_phys), intent(in) :: (:,:,:)` | Source buffer with updated constituent values. | -| `instance_number` | `integer, intent(in)` | | -| `errflg` / `errmsg` | `intent(out)` | | - -Wraps `obj(instance_number)%copy_out(const_array, ...)`. Use this to -push host-side updates back into the per-instance constituent object. - -### `ccpp_const_get_index(stdname, const_index, instance_number, errflg, errmsg)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `stdname` | `character(len=*), intent(in)` | Constituent standard name to look up. | -| `const_index` | `integer, intent(out)` | Returned index into the constituent array (or `int_unassigned` on miss). | -| `instance_number` | `integer, intent(in)` | | -| `errflg` / `errmsg` | `intent(out)` | | - -Wraps `obj(instance_number)%const_index(standard_name=stdname, -index=const_index, ...)`. For constituents whose std names are known -at code-gen time, prefer using the module-level `index_of_` integer -directly (no call needed; it's bound during -`ccpp_initialize_constituents`). - -### `ccpp_constituents_array(instance_number) result(const_ptr)` - -Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → -`obj(instance_number)%field_data_ptr()`. - -### `ccpp_advected_constituents_array(instance_number) result(const_ptr)` - -Returns `real(kind=kind_phys), pointer :: const_ptr(:,:,:)` → -`obj(instance_number)%advected_constituents_ptr()`. Subset of the -full constituent array containing only those flagged `advected=.true.`. - -### `ccpp_model_const_properties(instance_number) result(const_ptr)` - -Returns `type(ccpp_constituent_prop_ptr_t), pointer :: const_ptr(:)` → -`obj(instance_number)%constituent_props_ptr()`. - -### `ccpp_deallocate_dynamic_constituents(instance_number)` - -| Arg | Direction / Type | Purpose | -|---|---|---| -| `instance_number` | `integer, intent(in)` | | - -**Per-instance reset + last-to-leave teardown**: -1. `obj(instance_number)%reset()` — unlocks the table for this instance. -2. Iterates every `obj(i)` and checks `%const_props_locked()`. If any - instance is still locked, the routine returns. -3. If **every** instance has been reset (none still locked), the routine - tears down the shared state: - - Deallocates every `_dynamic_constituents` buffer. - - Deallocates `ccpp_model_constituents_obj(:)`. - - Resets every `index_of_` integer to 0. - -The host should call this for every instance that successfully called -`ccpp_register_constituents`. - ---- - -## 6. Generated code structure - -When any suite touches constituent state, capgen-ng emits one extra -module per generator run: **`ccpp_host_constituents.F90`**. - -### Module declarations - -```fortran -module ccpp_host_constituents - use ccpp_kinds, only: kind_phys - use ccpp_constituent_prop_mod, only: & - ccpp_model_constituents_t, & - ccpp_constituent_properties_t, & - ccpp_constituent_prop_ptr_t - - implicit none - private - - ! ----- public state ---------------------------------------------------- - public :: ccpp_model_constituents_obj - public :: index_of_ ! one per known constituent std name - public :: index_of_ - public :: ccpp_model_const_stdnames ! parameter array - - ! ----- public routines (also re-exported from ccpp_static_api) -------- - public :: ccpp_register_constituents - public :: ccpp_initialize_constituents - public :: ccpp_is_scheme_constituent - public :: ccpp_number_constituents - public :: ccpp_gather_constituents - public :: ccpp_update_constituents - public :: ccpp_const_get_index - public :: ccpp_constituents_array - public :: ccpp_advected_constituents_array - public :: ccpp_model_const_properties - public :: ccpp_deallocate_dynamic_constituents - public :: _dynamic_constituents ! one per suite with register-phase producers - public :: _dynamic_constituents - - ! ----- module-level state --------------------------------------------- - type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) - type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) - type(ccpp_constituent_properties_t), allocatable, target :: _dynamic_constituents(:) - integer :: index_of_ = 0 - integer :: index_of_ = 0 - character(len=N), parameter :: ccpp_model_const_stdnames(K) = (/ & - ' ', & - ' ' /) - -contains - ! ... routines as documented in §5 ... -end module ccpp_host_constituents -``` - -### Suite-cap responsibilities - -`ccpp__cap.F90` does NOT own constituent state. Its -`_register` routine packs each register-phase scheme's -constituent array into the suite's `_dynamic_constituents` -buffer (USE'd from `ccpp_host_constituents`): - -```fortran -if (.not. allocated(_dynamic_constituents)) then - ! First-instance-only two-pass count + populate. - num_consts = 0 - call _register(scheme_consts=scheme_consts, ...) - num_consts = num_consts + size(scheme_consts, 1) - deallocate(scheme_consts) - ... - allocate(_dynamic_constituents(num_consts)) - num_consts = 0 - call _register(scheme_consts=scheme_consts, ...) - do i = 1, size(scheme_consts, 1) - _dynamic_constituents(num_consts + i) = scheme_consts(i) - end do - num_consts = num_consts + size(scheme_consts, 1) - deallocate(scheme_consts) - ... -end if -``` - -The buffer is **shared across instances** (registration is identical -per instance); only the first instance to call `_register` -populates it. The host-wide merge happens in -`ccpp_register_constituents`. - -### Group-cap call sites - -`ccpp___cap.F90` USE's the constituent symbols it needs -from `ccpp_host_constituents`: - -```fortran -use ccpp_host_constituents, only: ccpp_model_constituents_obj, & - index_of_cloud_liquid_water_mixing_ratio -``` - -… and emits scheme call sites with the per-instance access expression: - -```fortran -call cld_liq_run( & - ... - cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer(lb:ub, 1:nlev, & - index_of_cloud_liquid_water_mixing_ratio), & - tend_cldliq=ccpp_model_constituents_obj(inst_num)%vars_layer_tend(lb:ub, 1:nlev, & - index_of_cloud_liquid_water_mixing_ratio), & - ...) -``` - -The `instance_number` dummy is auto-injected into the group-cap -subroutine signatures by `_extra_dim_ctrl_entries` because the -resolver adds `instance_number` to every constituent arg's -`used_dim_std_names`. - -### Framework F90 dependencies - -`ccpp_host_constituents.F90` and the suite caps depend on these -framework files (listed under `` in `datatable.xml`): - -| File | Why | -|---|---| -| `ccpp_constituent_prop_mod.F90` | Provides `ccpp_model_constituents_t`, `ccpp_constituent_properties_t`, `ccpp_constituent_prop_ptr_t`. | -| `ccpp_hashable.F90` | Transitive dep of `ccpp_constituent_prop_mod`. | -| `ccpp_hash_table.F90` | Transitive dep. | -| `ccpp_scheme_utils.F90` | Provides `ccpp_initialize_constituent_ptr` + `ccpp_constituent_index` (used by cam-sima rrtmgp / mmm schemes). | - -The host's CMake should query `ccpp_datafile.py --utility-files` to -get the absolute paths to these files at the right output location. - ---- - -## 7. Multi-instance design - -In capgen-ng, **per-instance state** means: each "instance" (typically -an OpenMP team / chunk-domain partition) has its own copy of the -state arrays, indexed by `instance_number ∈ [1, number_of_instances]`. - -### What's per-instance - -| State | Storage | -|---|---| -| Constituent values + tendencies | `ccpp_model_constituents_obj(:)` — one DDT per instance | -| Suite-level state machine | `ccpp_suite_state(:)` — declared in each suite cap | -| Suite-owned data | `ccpp_suite_data(:)` — declared in each suite-data module | -| Group-level state machine | `ccpp_group_state(:)` — declared in each group cap | - -### What's shared across instances - -| State | Reason | -|---|---| -| `_dynamic_constituents(:)` per-suite buffers | Registration is identical per instance — populated by the first instance to call `_register` and reused by the rest | -| `index_of_` integers | The constituent table is identical per instance, so the indices are too | -| `ccpp_model_const_stdnames` parameter array | Compile-time constant | - -### Sizing - -`number_of_instances` is the single source of truth. The host declares -it in metadata + Fortran; the generator USE's it from the host module -wherever per-instance allocation happens. See the prior memo -[*Where the total number of instances comes from*](#) for the call -chain (and matching values across all four state arrays: -`ccpp_suite_state`, `ccpp_suite_data`, `ccpp_group_state`, -`ccpp_model_constituents_obj`). - -If the host doesn't declare `number_of_instances`, every per-instance -allocation falls back to `1` and the framework runs single-instance. - -### Two host-side lifecycle patterns - -Both work; pick whichever fits your model. - -**Pattern A: all instances registered first** -``` -do isuite = 1, num_suites - do iinst = 1, num_instances - call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) - end do -end do -do iinst = 1, num_instances - call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) - call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) -end do -do isuite = 1, num_suites - do iinst = 1, num_instances - call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) - end do -end do -! ... time-stepping ... -do iinst = 1, num_instances - call ccpp_deallocate_dynamic_constituents(iinst) - ... -end do -``` - -**Pattern B: serial per instance** -``` -do iinst = 1, num_instances - do isuite = 1, num_suites - call ccpp_register(suite_names(isuite), errmsg, errflg, iinst) - end do - call ccpp_register_constituents(host_constituents, iinst, errflg, errmsg) - call ccpp_initialize_constituents(ncols, num_layers, iinst, errflg, errmsg) - do isuite = 1, num_suites - call ccpp_init(suite_names(isuite), errmsg, errflg, iinst) - end do - ! ... per-instance time-stepping ... - call ccpp_deallocate_dynamic_constituents(iinst) -end do -``` - -### Last-to-leave teardown - -`ccpp_deallocate_dynamic_constituents(inst)`: -1. Per-instance `obj(inst)%reset()`. -2. Iterates every `obj(i)`; if any has `%const_props_locked() == .true.`, - returns early. -3. Otherwise (every instance reset): deallocates the shared per-suite - buffers, deallocates `ccpp_model_constituents_obj(:)`, and zeros every - `index_of_` integer. - -This works for both lifecycle patterns above. - ---- - -## 8. Limitations and gotchas - -> **Note (2026-05-12).** Several items in this section are under active -> discussion for an upcoming framework + generator overhaul. See -> `doc/constituents_overhaul.md` for the full architectural review and -> three reform proposals. - -### Framework property ownership (2026-05-12) - -The framework's `ccpp_constituent_properties_t` now carries a private -`framework_owns_me` flag (default `.false.`) with -`is_framework_owned()` getter and `set_framework_owned(value)` setter. -`ccpt_deallocate` only deallocates the underlying prop when the flag -is `.true.`; otherwise it just nullifies its pointer. - -Under capgen-ng's explicit-registration model, all -`ccpp_constituent_properties_t` objects are **target-owned by the -caller** (the host's `host_constituents(:)` array, or the per-suite -`_dynamic_constituents(:)` buffer). We never set the flag, so -the framework correctly skips deallocation. Hosts that hand-allocate -property objects on the heap and want the framework to free them must -call `set_framework_owned(.true.)` before passing to `%new_field`. - -### Missing setters (framework gap) - -The framework lacks setters for `advected`, `diagnostic_name`, -`default_value` (and `mixing_ratio_type`). This means once a -constituent is `%instantiate`d, those properties cannot be changed. -If your host needs to override a scheme-supplied `diagnostic_name` or -`advected` value, you currently cannot — open item in the constituents -overhaul proposal. - -### `ccpp_scheme_utils` singleton - -`ccpp_scheme_utils.ccpp_initialize_constituent_ptr` accepts only one -singleton pointer. It's a framework-level convenience used by cam-sima -schemes that call `ccpp_constituent_index` from `ccpp_scheme_utils`. - -`ccpp_initialize_constituents` calls -`ccpp_initialize_constituent_ptr(obj(inst))` on each instance, but -**only the first call across instances actually sets the pointer** -(the routine is internally guarded by an `initialized` flag). -Subsequent calls are silent no-ops. - -For multi-instance hosts, schemes that use -`ccpp_scheme_utils.ccpp_constituent_index` will see only the first -instance's object — a known limitation inherited from the framework -module's design. Schemes that use the per-instance accessors -(`obj(inst)%const_index(...)` via `ccpp_const_get_index`) are -unaffected. - -### Constituent metadata is identical across instances - -The constituent table (which constituents exist, their properties, the -`index_of_` mapping) is **identical** for every instance. Every -instance's `obj(i)` has the same hash table, populated identically by -its own `ccpp_register_constituents` call. - -This means: - -- `ccpp_number_constituents` returns the same value regardless of - `instance_number`. -- `ccpp_const_get_index` returns the same index regardless of - `instance_number`. -- The `index_of_` integers are populated identically by every - instance's `ccpp_initialize_constituents` (last-write-wins is fine - since every write is the same value). - -`instance_number` is still in the signatures of these routines — see -[§5](#5-public-api-reference) for the rationale. - -### Forbidden patterns recap - -These are rejected at code-gen time (Rule 4 of [§2](#2-the-four-rules-scheme-author-conventions)): - -- `is_constituent + intent=out + non-tendency std_name` — physics phases - may only produce tendencies, not new base constituents. -- `is_constituent + intent=in/inout + tendency_of_*` — tendencies are - write-only. - -### Subscript indices in sliced local_names must be standard names - -If a host metadata variable is declared with a sliced local name -like `q(:,:,index_of_water_vapor_specific_humidity)`, every subscript -token (other than `:` and integer literals) must be a known standard -name. Otherwise the resolver raises a `CCPPError` with a clear -message naming the offending token. - -### Open work items - -- **Unconditional `ccpp_host_constituents.F90` emission.** The - generator currently emits `ccpp_host_constituents.F90` for every - build, even when no scheme or host actually uses the constituent - system (no `ccpp_constituent_properties_t(:)` register-phase arg, - no `is_constituent`-flagged scheme arg, no framework-named - `index_of_` / `ccpp_constituents` / etc. claimed by capgen-ng). - When the host owns its own indices (SCM/GFS) and no scheme exercises - the constituent path, the generated file is dead code that should be - suppressed. Tracked as a deferred item; the `host_dict` precedence - rule above already keeps the file *correct* (empty) in that case. - ---- - -## 9. Differences from original capgen - -| Aspect | Original capgen | capgen-ng | -|---|---|---| -| Constituent object location | Generated `_ccpp_cap.F90` module | `ccpp_host_constituents.F90` (one per generator run) | -| Per-instance | No (single instance) | Yes (`obj(:)` allocatable, sized to `number_of_instances`) | -| Routine name prefix | `_ccpp_register_constituents`, etc. | `ccpp_register_constituents`, etc. (no host prefix; one set per generator run) | -| Routine signatures | No `instance_number` arg | `instance_number` in every per-instance routine | -| Host metadata for constituent obj | None (auto-created by generator) | None (still auto-created by generator) | -| Module-level pointers | `_constituents_array` etc. as functions returning pointers | Same idea, now per-instance via `instance_number` arg | -| Scheme std-name set | `_model_const_stdnames` parameter array | `ccpp_model_const_stdnames` parameter array (no host prefix) | -| Host-facing API surface | `_ccpp_register_constituents`, `_ccpp_initialize_constituents`, `_ccpp_number_constituents`, `_ccpp_is_scheme_constituent`, `_ccpp_gather_constituents`, `_ccpp_update_constituents`, `_ccpp_deallocate_dynamic_constituents`, `_constituents_array`, `_advected_constituents_array`, `_model_const_properties`, `_const_get_index` | Same surface, no `_` prefix | -| Dynamic constituent buffer dimensionality | 1D, per host | 1D, per generator run, **shared across instances** | -| Static suite constituents | Auto-cloned by `ConstituentVarDict.find_variable` and registered via `_constituents_copy_const` accessors | Tracked at code-gen time via `is_constituent` flag; included in the constituent table only if a register-phase scheme produces them (rule 1). Schemes that *consume* a base constituent (rule 2) don't trigger registration — the constituent must be registered by SOMEONE (host or another scheme's register) for the access to work at runtime. | - -### Migration notes for cam-sima hosts - -- **Scheme metadata**: no changes needed. Cam-sima follows rules 2 and - 3 already (audited 2026-05-11). The 4 schemes that register - constituents via `ccpp_constituent_properties_t` (rule 1) work - unchanged. -- **Host metadata**: drop any explicit declaration of - `ccpp_model_constituents_object` if you carried one over from a - previous capgen-ng experiment — the generator owns it now. -- **Host Fortran**: change all `_ccpp_*_constituents` calls to - the unprefixed names (`ccpp_register_constituents` etc.) and add - `instance_number` to every call site. - ---- - -## 10. Worked example - -A minimal cam-sima-style suite with one scheme that consumes a base -constituent and produces its tendency. - -### Scheme metadata (`consume_constituent.meta`) - -``` -[ccpp-table-properties] - name = consume_constituent - type = scheme - -[ccpp-arg-table] - name = consume_constituent_run - type = scheme -[ cldliq ] - standard_name = cloud_liquid_water_mixing_ratio - units = kg kg-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys - intent = in - advected = .true. -[ tend_cldliq ] - standard_name = tendency_of_cloud_liquid_water_mixing_ratio - units = kg kg-1 s-1 - dimensions = (horizontal_dimension, vertical_layer_dimension) - type = real | kind = kind_phys - intent = out - constituent = .true. -[ errmsg ] - ... -[ errflg ] - ... -``` - -### Host metadata (`my_host.meta`) - -``` -[ccpp-table-properties] - name = my_host - type = host - -[ccpp-arg-table] - name = my_host - type = host -[ ncols ] - standard_name = horizontal_dimension - units = count - dimensions = () - type = integer -[ nlev ] - standard_name = vertical_layer_dimension - units = count - dimensions = () - type = integer -[ ninstances ] - standard_name = number_of_instances - units = count - dimensions = () - type = integer -``` - -(Plus a `type=control` table declaring `instance_number`, -`horizontal_loop_begin`, `horizontal_loop_end`, `ccpp_error_message`, -`ccpp_error_code`, etc.) - -### Suite XML (`my_suite.xml`) - -```xml - - - - consume_constituent - - -``` - -### Generated `ccpp_host_constituents.F90` (excerpt) - -```fortran -module ccpp_host_constituents - use ccpp_kinds, only: kind_phys - use ccpp_constituent_prop_mod, only: & - ccpp_model_constituents_t, ccpp_constituent_properties_t, & - ccpp_constituent_prop_ptr_t - - implicit none - private - - public :: ccpp_model_constituents_obj - public :: index_of_cloud_liquid_water_mixing_ratio - public :: ccpp_register_constituents, ccpp_initialize_constituents - public :: ccpp_is_scheme_constituent, ccpp_number_constituents - public :: ccpp_gather_constituents, ccpp_update_constituents - public :: ccpp_const_get_index, ccpp_constituents_array - public :: ccpp_advected_constituents_array, ccpp_model_const_properties - public :: ccpp_deallocate_dynamic_constituents - public :: ccpp_model_const_stdnames - - type(ccpp_model_constituents_t), target, allocatable :: ccpp_model_constituents_obj(:) - integer :: index_of_cloud_liquid_water_mixing_ratio = 0 - character(len=31), parameter :: ccpp_model_const_stdnames(1) = (/ & - 'cloud_liquid_water_mixing_ratio' /) - -contains - ! ... full subroutine bodies as in §5 ... -end module ccpp_host_constituents -``` - -### Host code skeleton (single-instance illustration) - -```fortran -subroutine my_host_run() - use ccpp_static_api, only: ccpp_register, ccpp_register_constituents, & - ccpp_initialize_constituents, ccpp_init, & - ccpp_physics_run, ccpp_final, & - ccpp_deallocate_dynamic_constituents - use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t - - type(ccpp_constituent_properties_t), allocatable :: host_consts(:) - integer :: errflg - character(len=512) :: errmsg - integer, parameter :: inst = 1 - - ! 1. Run register phase: populates per-suite dynamic-constituent buffers. - call ccpp_register('my_suite', errmsg, errflg, inst) - - ! 2. Build host's own constituent declarations (water vapor, etc.). - allocate(host_consts(1)) - call host_consts(1)%instantiate( & - std_name='water_vapor_specific_humidity', long_name='water vapor', & - units='kg kg-1', vertical_dim='vertical_layer_dimension', & - advected=.true., errcode=errflg, errmsg=errmsg) - - ! 3. Merge host + suite-side constituents into obj(inst). - call ccpp_register_constituents(host_consts, inst, errflg, errmsg) - - ! 4. Allocate vars_layer + bind cached indices. - call ccpp_initialize_constituents(ncols, nlev, inst, errflg, errmsg) - - ! 5. Framework init phase. - call ccpp_init('my_suite', errmsg, errflg, inst) - - ! 6. Time-stepping (omitted). - call ccpp_physics_run('my_suite', 'phys', col_start, col_end, & - thread_num, nthreads, nphys_threads, & - errflg, errmsg, inst) - - ! 7. Shutdown. - call ccpp_final('my_suite', errmsg, errflg, inst) - call ccpp_deallocate_dynamic_constituents(inst) - deallocate(host_consts) -end subroutine my_host_run -``` - -For multi-instance, wrap each per-instance call in -`do iinst = 1, ninstances ... end do` per the patterns in -[§7](#7-multi-instance-design). diff --git a/doc/constituents_overhaul_20260513T0733.md b/doc/constituents_overhaul_20260513T0733.md deleted file mode 100644 index 2adb34dd..00000000 --- a/doc/constituents_overhaul_20260513T0733.md +++ /dev/null @@ -1,836 +0,0 @@ -# CCPP Constituents — Architecture Review & Overhaul Discussion - -**Authors:** Dom Heinzeller (lead), Claude (assistant) -**Date drafted:** 2026-05-12 -**Last revised:** 2026-05-13 -**Intended audience:** CCPP framework team, CAM-SIMA team -**Status:** Discussion document — no decisions are final. Proposals -A/B/C below remain pending the upcoming meeting; the bug fix from -Proposal A (the `ccpt_deallocate` ownership flag) and the capgen-ng -internal cleanup from Proposal B (§4.8) have landed; the missing -setters from Proposal A and the `is_match` relaxation from Proposal B -have not. - ---- - -## Executive summary - -CCPP's "constituent" mechanism — how schemes declare and how the framework -manages tracer species like water vapor, cloud liquid, prescribed ozone, -etc. — has grown organically over the last few years. The result works, -but it carries: - -- **A latent framework bug** in `ccpp_constituent_prop_mod` that crashes on - teardown of explicitly-registered (target-passed) constituent property - arrays. Fixed in capgen-ng's framework copy 2026-05-12; needs to land - upstream. -- **Architectural confusion** about which properties are *physics-portable* - (the scheme owns them) versus *host-configuration* (the host owns them). - Today schemes are forced to supply host-specific values (`diag_name` is - the worst offender) at `%instantiate` time. -- **Setter API gaps**: properties that the host wants to override after - scheme-side registration (`advected`, `diagnostic_name`, `default_value`) - have no setters; `is_match` is overly strict about properties hosts - should be free to change. -- **Two registration models** coexist — original capgen's auto-clone of - is_constituent scheme args, and capgen-ng's explicit register-phase + - host-side declaration. Capgen-ng deliberately dropped auto-clone. - -This document is a structured brief for a discussion this week. It does -NOT pre-commit to any decision; it lays out what exists, what's broken, -what we audited, and what proposals are on the table. - ---- - -## Table of contents - -1. [How original capgen handles constituents](#1-how-original-capgen-handles-constituents) -2. [How capgen-ng handles constituents](#2-how-capgen-ng-handles-constituents) -3. [What CAM-SIMA actually needs (audit)](#3-what-cam-sima-actually-needs-audit) -4. [Bugs and design flaws](#4-bugs-and-design-flaws) -5. [Property classification (Class A vs Class B)](#5-property-classification-class-a-vs-class-b) -6. [What to remove, replace, improve](#6-what-to-remove-replace-improve) -7. [Open design questions](#7-open-design-questions) -8. [Three proposals — minimal / clean / deep](#8-three-proposals--minimal--clean--deep) -9. [Appendix: framework setter inventory](#9-appendix-framework-setter-inventory) - ---- - -## 1. How original capgen handles constituents - -### 1.1 Mental model - -Original capgen treats constituents as a **separate scope** between -suite and host: - -``` -group → suite → ConstituentVarDict → host -``` - -A scheme arg flagged `constituent = True` in metadata is matched first -against group/suite/ConstituentVarDict, and only against host as a last -resort. The ConstituentVarDict is a synthetic dictionary whose entries -are auto-created by `find_variable()` when a scheme metadata declares a -constituent dependency. - -### 1.2 Auto-clone of `is_constituent` scheme args - -Every scheme arg with non-default `advected`, `constituent`, or -`molar_mass` is treated as a *registration*. The generator emits, into -the host cap, a routine `_constituents_ccpp_create_constituent_array` -that: - -1. Allocates a `ccpp_constituent_properties_t` pointer per scheme arg. -2. Calls `%instantiate(...)` populating fields **from the scheme - metadata directly** — `std_name`, `long_name`, `diagnostic_name`, - `units`, `default_value`, `advected`, `vertical_dim`, etc. (See - `scripts/constituents.py:565`.) -3. Adds it to the model constituents object via `%new_field`. - -After this auto-clone runs, the host's hand-written -`host_constituents(:)` array is appended, then `%lock_table` finalizes -the hash table. - -### 1.3 The host-cap-owned `ccpp_model_constituents_obj` - -Original capgen generates **one** `ccpp_model_constituents_obj` per -generator invocation, declared module-level in `_ccpp_cap.F90`. -Single global; not per-instance. (CAM-SIMA runs one host per -executable, so single-instance is fine for them.) - -### 1.4 Scheme-side `%instantiate` registration (the other path) - -A scheme may also register constituents via a register-phase argument: - -```fortran -type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) -``` - -The scheme allocates the array, calls `%instantiate` per entry, and -returns it. Original capgen wires this through a per-suite -"dynamic constituents" buffer and merges it during host-cap setup, -alongside the auto-cloned set. - -So original capgen really supports **three** registration sources: - -- Host: hand-written `host_constituents(:)` arg. -- Suite-dynamic: register-phase scheme args. -- Suite-static: auto-cloned from any `is_constituent` consumer. - -All three flow into one `%new_field` table. - -### 1.5 Lifecycle - -- `_ccpp_register_constituents(host_constituents, ...)` runs the - three-source merge. -- `ccpp_initialize_constituent_ptr(const_obj, ...)` (in - `ccpp_scheme_utils`) caches a pointer for the `ccpp_constituent_index` - lookup. -- Phase entry points access `vars_layer` / `vars_layer_tend` via cached - `index_of_` integers. - -### 1.6 What's good about original capgen's approach - -- Schemes declare a constituent dependency once in metadata; no manual - Fortran registration ever needed for "static" tracers. -- Host doesn't have to enumerate every species every scheme wants. -- Works for CAM-SIMA's current scheme catalog. - -### 1.7 What's painful about original capgen's approach - -- The auto-clone path is **invisible** to anyone reading the scheme - Fortran — the registration happens in generated code. -- `ConstituentVarDict` is a synthetic scope, conceptually subtle, and - doesn't generalize cleanly to multi-instance. -- The auto-clone path lifts `diagnostic_name` and `default_value` from - scheme metadata, but those values are often host-specific (see §4.4). -- Three sources of registration with overlap mean two registrations of - the same `std_name` may collide; original capgen relies on - `is_match` (units, advected, thermo_active, water_species) to dedup, - which means schemes accidentally diverge on `advected` and trip the - "incompatible constituent" error. - ---- - -## 2. How capgen-ng handles constituents - -### 2.1 Mental model - -No synthetic scope. Constituents are *one of four* sources for any -scheme arg: - -``` -control | host | suite | constituent -``` - -The resolver classifies each scheme arg into exactly one source. A -`constituent` source means the value will be accessed at runtime as -`ccpp_model_constituents_obj()%vars_layer(:, :, index_of_)` -(or `%vars_layer_tend(...)` for `tendency_of_` outputs). - -### 2.2 The four scheme-author rules - -(See `doc/constituents.md` for full details; this is the summary.) - -1. **Register** — register-phase scheme args of type - `ccpp_constituent_properties_t(:), intent=out, allocatable` declare - new constituents the scheme contributes. -2. **Consume** — physics-phase scheme args with `advected=true` (or - `molar_mass=...` or `constituent=true`) and `intent=in/inout`, with - the constituent's standard name, read the base species. -3. **Produce a tendency** — physics-phase scheme args with - `constituent=true`, `intent=out`, and standard name - `tendency_of_`, write the tendency. -4. **Mismatched combinations are errors** — `intent=out` on a base - constituent, or `intent=in` on a tendency, are codegen-time errors. - -### 2.3 Two registration sources (no auto-clone) - -- **Host**: hand-written `host_constituents(:)`, passed into - `ccpp_register_constituents(host_constituents, instance_number, ...)`. -- **Suite-dynamic**: register-phase scheme args, accumulated into a - per-suite buffer `_dynamic_constituents(:)` by `_register`, - drained into `ccpp_model_constituents_obj(inst)` by - `ccpp_register_constituents`. - -The auto-clone-from-metadata path is deliberately **gone**. If a scheme -declares `advected=true` on an arg but no source registers that -standard name, capgen-ng now emits a runtime check during -`ccpp_initialize_constituents` that errors with the missing name. - -### 2.4 Per-instance state - -Everything is per-instance: - -```fortran -type(ccpp_model_constituents_t), allocatable :: ccpp_model_constituents_obj(:) - ! indexed by instance_number -``` - -All host-facing entry points take `instance_number`: - -``` -ccpp_register_constituents (host_constituents, instance_number, errflg, errmsg) -ccpp_initialize_constituents (ncols, num_layers, instance_number, errflg, errmsg) -ccpp_number_constituents (num_flds, advected, instance_number, errflg, errmsg) -ccpp_gather_constituents (const_array, instance_number, errflg, errmsg) -ccpp_update_constituents (const_array, instance_number, errflg, errmsg) -ccpp_const_get_index (stdname, const_index, instance_number, errflg, errmsg) -ccpp_constituents_array (instance_number) => pointer -ccpp_advected_constituents_array (instance_number) => pointer -ccpp_model_const_properties (instance_number) => pointer -ccpp_deallocate_dynamic_constituents (instance_number, ...) -``` - -`ccpp_is_scheme_constituent(var_name, ...)` and the -`ccpp_model_const_stdnames(:)` parameter array are NOT per-instance — -the standard-name catalog is identical across instances. - -### 2.5 Lifecycle - -``` -ccpp_register(suite_name, instance_number, ...) - └─ _register → packs scheme-dynamic constituents into - _dynamic_constituents (shared buffer, - first instance wins) - ↓ -ccpp_register_constituents(host_constituents, instance_number, ...) - └─ initialize_table(num_host_consts + num_suite_consts) - └─ new_field(host_consts ...) - └─ new_field(_dynamic_constituents ...) - └─ lock_table - ↓ -ccpp_initialize_constituents(ncols, num_layers, instance_number, ...) - └─ lock_data (allocates vars_layer, vars_layer_tend, vars_minvalue) - └─ ccpp_initialize_constituent_ptr (first instance wins; documented limit) - └─ %const_index('') for each enumerated constituent - └─ post-lookup int_unassigned check → clear error message - ↓ -ccpp_init(suite_name, instance_number, ...) - └─ _init → binds module-level pointers - ↓ -... physics phases ... - ↓ -ccpp_final(suite_name, instance_number, ...) - └─ _final → nullifies + last-to-leave deallocates - ↓ -ccpp_deallocate_dynamic_constituents(instance_number, ...) - └─ ccp_model_constituents_obj(inst)%reset - ↓ (in _final, last-to-leave) - deallocate(_dynamic_constituents) -``` - -### 2.6 What's good - -- Explicit. Every constituent registration is visible in someone's - Fortran source. -- Multi-instance from day one. -- The "four rules" are small enough to fit on a slide. -- Resolver-time + codegen-time + runtime checks catch the most common - mistakes. - -### 2.7 What's still painful - -Covered in §4. - ---- - -## 3. What CAM-SIMA actually needs (audit) - -### 3.1 Scheme-side registration usage - -We audited `EXT/cam-sima/atmospheric_physics/schemes/` for use of the -register-phase `ccpp_constituent_properties_t(:)` pattern: - -| Scheme | File | Registers | -|---|---|---| -| RRTMGP constituents | `schemes/rrtmgp/rrtmgp_constituents.meta` | radiative-active species | -| MUSICA chemistry | `schemes/musica/musica_ccpp.meta` | chemical species from MUSICA | -| Prescribed aerosols | `schemes/chemistry/prescribed_aerosols.meta` | aerosol species | -| Prescribed ozone | `schemes/chemistry/prescribed_ozone.meta` | ozone | - -**Total: 4 of 128 schemes** in the atmospheric_physics tree use -scheme-side registration. The other 124 only **consume** constituents -(`advected=true` + `intent=in/inout` in metadata, accessed via the -framework's `vars_layer`). - -This is a small enough number that an alternative "host-only -registration" model is feasible: move those 4 register calls into the -host (or into helper modules the host calls), and the rest of the -catalog only consumes. - -### 3.2 Host-side patterns - -`EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` wraps -the framework setters and exposes: - -- `const_set_thermo_active(const_obj | const_ind, value)` -- `const_set_water_species(const_obj | const_ind, value)` -- `const_set_minimum(...)` - -CAM-SIMA actively **calls these setters at runtime** — schemes don't -supply `thermo_active` at instantiate time; the host configures it -afterwards. This is direct evidence that the "post-instantiation -override" pattern is real and used today, and that the framework's -setter API is load-bearing. - -### 3.3 What CAM-SIMA does **not** do - -- It does not rely on auto-clone for `diag_name`. The scheme-side - register calls in the 4 schemes do supply `diag_name`, but those - values are CAM-SIMA's; a different host would need different ones. -- It does not use `ccpp_constituent_index` (the - `ccpp_scheme_utils`-singleton-based lookup) extensively — most - access goes through the framework's `index_of_` integers. - -### 3.4 What CAM-SIMA's host-cap-owned constituent object looks like - -Because original capgen generates **one** `ccpp_model_constituents_obj` -per generator invocation, and CAM-SIMA uses one generator invocation per -executable, CAM-SIMA effectively runs single-instance today. A -multi-instance CAM-SIMA (sub-columns, ensembles) would expose the -single-global limitation immediately. - ---- - -## 4. Bugs and design flaws - -This section lists known issues across the three layers (framework, -original capgen, capgen-ng). Items marked **(FIXED)** were resolved -2026-05-12 and either are or will be PRs; items marked **(OPEN)** are -intentionally left for this discussion. - -### 4.1 Framework: `ccpt_deallocate` ownership bug (FIXED in capgen-ng tree, needs upstream PR) - -- **Location**: `src/ccpp_constituent_prop_mod.F90`, `ccpt_deallocate` - + `ccpt_set`. -- **Symptom**: `free(): invalid size` crash when - `ccp_model_const_reset` is called on a properly-locked table whose - entries came from pointer-assigned targets (the common pattern - under capgen-ng's explicit registration; also potentially under - original capgen's `host_constituents` path). -- **Root cause**: `ccpt_set` does pointer assignment (`this%prop => - const_ptr`); `ccpt_deallocate` does an unconditional - `deallocate(this%prop)`. The deallocate is correct only when the - caller allocated `const_ptr` on the heap and transferred ownership. -- **Why it didn't surface earlier**: original capgen's advection test - only calls `deallocate` once between a *failing* register and a - *successful* one — at that point `lock_table` has not populated - `const_metadata`, so the broken inner loop is skipped. Capgen-ng - triggers it because its teardown calls `reset` after a successful - lock. -- **Fix landed 2026-05-12**: added `framework_owns_me` private flag on - `ccpp_constituent_properties_t` (default `.false.`) with - `is_framework_owned()` getter and `set_framework_owned(value)` - setter; `ccpt_deallocate` now only deallocates when the flag is set. - Original capgen's auto-clone path in `scripts/constituents.py` - updated to call `set_framework_owned(.true.)` after `allocate`. - Diffs in `src/ccpp_constituent_prop_mod.F90` (and capgen-ng's - parallel copy) + `scripts/constituents.py`. -- **Status**: framework tests pass; capgen-ng unit-test suite (1127 passing - as of 2026-05-13) is green. Still needs upstream PR to ccpp-framework + - original ccpp-capgen. - -### 4.2 Framework: missing setters (OPEN) - -| Property | Optional in `%instantiate`? | Has setter? | `is_match`-checked? | -|---|---|---|---| -| `std_name` | required | — | (lookup key) | -| `long_name` | required | — | no | -| `diag_name` | required | **NO** | no | -| `units` | required | — | **yes** | -| `vertical_dim` | required | — | no | -| `advected` | optional (default .false.) | **NO** | **yes** | -| `default_value` | optional | **NO** | no | -| `min_value` | optional | `set_minimum` | no | -| `molar_mass` | optional | `set_molar_mass` | no | -| `water_species` | optional (default .false.) | `set_water_species` | **yes** | -| `mixing_ratio_type`| optional | **NO** | no | -| `thermo_active` | not in instantiate | `set_thermo_active` | **yes** | -| `const_index` | internal | `set_const_index` | no | - -**Pain points**: - -- `advected` is `is_match`-checked AND has no setter. Once registered, - immutable. If a scheme and the host disagree, you get the - "incompatible constituent" error and you cannot reconcile from - Fortran. -- `diag_name` is required (cannot be omitted at instantiate) AND has - no setter. A scheme must pick a value at registration time; that - value is then frozen. -- `default_value` is silently optional. If omitted, the constituent - array initializes to `huge(real)` and downstream comparisons fail - in surprising ways (we burnt half a day on this 2026-05-12). -- `thermo_active` is the only property in the "post-instantiate-only" - shape: it has a setter but isn't a `%instantiate` arg. The - asymmetry is confusing. - -### 4.3 Framework: `is_match` is too strict (OPEN) - -`is_match` (in `ccp_is_match`) checks `units`, `advected`, -`thermo_active`, `water_species`. Three of those four (`advected`, -`thermo_active`, `water_species`) are properties the host legitimately -overrides post-registration. Two registrations of the same `std_name` -with the same `units` but different `advected` should be a -duplicate-dedup (host wins), not a hard error. - -### 4.4 Framework: `diag_name` portability problem (OPEN) - -Diagnostic output names are host-specific. CAM-SIMA names cloud -liquid mixing ratio `CLDLIQ`; UFS would call it something else. Yet -`%instantiate` makes `diag_name` a *required* arg, forcing schemes to -either: - -- Pick a host-specific value (couples the scheme to a host), or -- Pick a "neutral" default that no host's diagnostic tooling - recognizes. - -The current de-facto pattern in CAM-SIMA scheme code is to pick a -CAM-SIMA-flavoured value and ship it. Any port to UFS would need to -either monkey-patch or fork the scheme. - -A clean fix: -1. Make `diag_name` optional at `%instantiate` (default to empty - string or `std_name`). -2. Add `set_diagnostic_name(value)` setter. -3. Host overrides per-registration after `ccpp_register_constituents`. - -### 4.5 Original capgen: implicit registration (OPEN — observation) - -The auto-clone path is generator magic. Reading scheme metadata -doesn't tell you whether the scheme's args result in registration; you -have to know that `advected=true` triggers it. This is a documentation -+ comprehension problem more than a bug. - -### 4.6 Original capgen: single-instance `ccpp_model_constituents_obj` (OPEN — limitation) - -The host cap declares one global. Multi-instance hosts would need to -either generate one cap per instance or restructure. - -### 4.7 Original capgen: `ConstituentVarDict` complexity (OPEN — observation) - -The synthetic scope between suite and host serves correctness but -adds a code path that most contributors don't read. If we drop it -(capgen-ng has), the variable-matching algorithm shrinks. - -### 4.8 Capgen-ng: `_FRAMEWORK_CONST_DIM_INPUTS` cleanup (LANDED 2026-05-13) - -`generator/static_api.py` no longer carries the hand-curated frozenset of -standard names; framework-constituent dimension references now ride on a -dedicated `used_const_dim_std_names` field on `ResolvedArg`. Closes the -"hand-curated → structured field" REVISIT note that was in the code. - -### 4.9 Capgen-ng: no codegen-time cross-check of scheme registration (OPEN) - -The resolver knows every `is_constituent` arg's standard name (in -`SuiteResolution.constituent_index_names`) but doesn't know what each -scheme's `_register` subroutine actually `%instantiate`s. Today's -guarantee is a runtime check (the `int_unassigned` validation we -added 2026-05-12). Stronger options: - -- (a) New metadata attribute `registers_std_names = a, b, c` on - register-phase tables; codegen errors at generation time. -- (b) Parse scheme `_register` Fortran for `%instantiate(std_name=…)` - calls and cross-check. -- (c) Keep runtime check as authoritative, document the gap. - -### 4.10 Capgen-ng: scheme-metadata `diagnostic_name` for is_constituent args is host-specific (OPEN) - -Same issue as §4.4 but in capgen-ng's metadata layer. Today's -`diagnostic_name` attribute on a scheme metadata arg flows into -`datatable.xml` and is then trusted as "the" diagnostic name. If we -adopt setter-based class-B overrides, this attribute should either be -dropped for constituent args or marked as a default-only hint. - -### 4.11 Capgen-ng: `ccpp_scheme_utils` singleton (OPEN — documented limit) - -`ccpp_initialize_constituent_ptr(const_obj)` stores a single module-level -pointer. Schemes that use `ccpp_constituent_index(stdname)` get that -pointer back. Under multi-instance, only the first instance's -pointer is retained — `ccpp_constituent_index` queries from -within a scheme will always reflect instance 1. CAM-SIMA's 4 -scheme-registering schemes don't rely on this; documented in -`doc/constituents.md` §8. Real fix requires either threading -`instance_number` through `ccpp_constituent_index` (interface -change) or maintaining a per-instance pointer table. - ---- - -## 5. Property classification (Class A vs Class B) - -Proposed in `design_constituents_mutability.md` 2026-05-12. Each -constituent property is conceptually owned by either the scheme -(physics-portable, immutable once instantiated) or the host -(host-configuration, mutable post-instantiation). - -### Class A — scheme-intrinsic (immutable) - -| Property | Why class A | -|---|---| -| `std_name` | Identity. Cannot change. | -| `long_name` | Human-readable name of the *species*. Not host-specific. | -| `units` | Physics correctness. `is_match`-checked. | -| `vertical_dim` | Scheme's structural expectation (interface vs layer). | -| `molar_mass` | Physical constant of the species. | -| `default_value` | (Debatable — see §7) Scheme-appropriate initial value. | - -### Class B — host-configuration (mutable post-instantiation) - -| Property | Why class B | -|---|---| -| `advected` | Whether the host's dycore advects this — host decision. | -| `diag_name` | Host-specific diagnostic system name. | -| `thermo_active` | Host model configuration. | -| `min_value` | Host runtime guardrail. | -| `water_species` | (Borderline — see §7) Physical classification but also host-config. | -| `mixing_ratio_type` | (Borderline — see §7) Depends on dycore convention. | - -### Consequences if adopted - -- `is_match` should check **only class A**. Today it checks 3 of 4 - class-B properties. -- Class B properties need setters. Today - `advected`, `diag_name`, (and `mixing_ratio_type` if it stays - class B) have none. -- `%instantiate` can demote class B from "required + optional" to - "all optional with sane defaults" — `diag_name=''`, - `advected=.false.`, etc. Schemes wouldn't need to set them at all. - ---- - -## 6. What to remove, replace, improve - -### Remove (or stop requiring) - -- **Scheme-metadata `diagnostic_name` on is_constituent args** — host - will override. Keep the attribute valid on non-constituent args - (where it's host tooling documentation, no portability issue). -- **`is_match` checks on advected / water_species / thermo_active** — - class B should not block dedup. -- **The `diag_name` requirement at `%instantiate`** — demote to - optional with `''` default. -- **(Not adopting)** Original capgen's auto-clone path. Already gone - in capgen-ng; this discussion does not propose bringing it back. - Listed for completeness because the option is in memory. - -### Replace - -- **`ConstituentVarDict`** as a concept — capgen-ng already runs - without it. If the framework or future generator code references - it, dropping is fine. -- **Single-global `ccpp_model_constituents_obj`** — capgen-ng's - per-instance array is the replacement. Original capgen could be - retrofitted, but the priority depends on whether multi-instance - enters the original capgen's roadmap. - -### Improve - -- **Add the missing setters**: `set_advected`, `set_diagnostic_name`, - `set_default_value` (if `default_value` becomes class B), - `set_mixing_ratio_type` (if class B). -- **Add a convenience routine** like - `ccpp_get_constituent_props_by_std_name(stdname, instance_number, prop_ptr, errflg, errmsg)` - so hosts can lookup a single constituent's property wrapper by - name without iterating. -- **Codegen-time cross-check** of scheme `_register` calls vs - metadata declarations (preferred: §4.9 option (a) — new - `registers_std_names` attr). -- **Document the lifecycle** clearly. `doc/constituents.md` is - ~960 lines; targeted additions for "register-then-override" - workflow once the new setters land. -- **Capgen-ng-internal cleanup** (LANDED 2026-05-13): replaced - `_FRAMEWORK_CONST_DIM_INPUTS` with a `used_const_dim_std_names` - field on `ResolvedArg`. - ---- - -## 7. Open design questions - -These are the calls we need to make in the meeting. - -### Q1. `default_value` — class A or class B? - -- **Class A argument**: the scheme knows what the species - should be initialized to (zero for "starts empty"; small positive - for "starts at background"); the host doesn't typically override. -- **Class B argument**: hosts may want non-default starting values - (chemistry runs with prescribed initial profiles). -- **Today's reality**: framework has no setter, so it's de-facto - class A. The advection-test issue 2026-05-12 surfaced because we - removed the `default_value=0._kind_phys` from cld_liq.F90's - scheme-side register and had no way to put it back; restoring it - in the scheme fixed the test but cements the class-A treatment. -- **Recommendation**: leave class A for now. Revisit when a real - host-override use case appears. - -### Q2. `water_species` — class A or class B? - -- The current `is_match` check on `water_species` treats it as - identity-defining (class A semantics). But the actual *meaning* of - the bit is mostly host bookkeeping ("does the dycore treat this as - water?"). CAM-SIMA has a `set_water_species` wrapper and uses it. -- **Recommendation**: class B, with the caveat that schemes whose - numerics depend on a constituent *being* water should declare that - in metadata as a hard requirement (different mechanism — not the - `is_match` machinery). - -### Q3. `mixing_ratio_type` — class A or class B? - -- The scheme's calculations assume `wrt_dry` or `wrt_moist`; this - feels class A. -- But hosts using different dycores might want to interpret the - same `std_name` differently — feels class B. -- **Recommendation**: class A. The mismatch should manifest as - different `std_name`s (`cloud_liquid_dry_mixing_ratio` vs - `cloud_liquid_wet_mixing_ratio`), not the same name with a runtime - override. Need cam-sima input. - -### Q4. After `is_match` relaxation: what happens on disagreement? - -- If two registrations of the same std_name agree on class A but - disagree on class B (e.g., `advected=.false.` from a scheme, - `advected=.true.` from the host), the second registration's class - B values should win without error. Effectively: the host overrides - the scheme. -- Order matters: today the host appends *after* the dynamic - constituents. Should we reverse so the host appends *first*? - Probably not — the "first registration wins on class A; host - setters override class B" model is conceptually clearer. -- **Recommendation**: silently dedup on matching class A; for class - B disagreements, the *later* registration's class B values are - ignored. Hosts use setters to override after registration - finalizes. - -### Q5. Should `%instantiate` accept class-B args at all? - -- **Option Y**: keep `%instantiate` accepting class B args (with - defaults). Schemes can supply them as hints; hosts can override. - Backward-compatible. -- **Option N**: remove class-B args from `%instantiate`. Schemes - *must* leave them to the host. Breaks the 4 cam-sima - scheme-registering schemes. -- **Recommendation**: option Y. The cost of breaking 4 schemes for - marginal clarity isn't worth it. - -### Q6. `ccpp_scheme_utils` singleton - -- Today: `ccpp_initialize_constituent_ptr(const_obj)` stores one - pointer module-wide. First instance wins. -- Fix options: - - (a) Maintain a per-instance pointer table; threading - `instance_number` through `ccpp_constituent_index`. - - (b) Document the limitation, route around it (no scheme uses - `ccpp_constituent_index` under multi-instance — capgen-ng - already enforces `index_of_` everywhere). -- **Recommendation**: (b). It's a one-line doc note and zero code - change. - ---- - -## 8. Three proposals — minimal / clean / deep - -### Proposal A — bugfix only - -**Scope**: -- Land the `ccpt_deallocate` ownership fix (done 2026-05-12). -- Update `scripts/constituents.py` for original capgen's auto-clone - path to pass `owned=.true.` (done). -- Add the three missing setters (`set_advected`, - `set_diagnostic_name`, `set_default_value`) without changing - semantics. Doesn't touch `is_match` or `%instantiate`. -- Document the gaps in `doc/constituents.md`. - -**Cost**: ~50 lines framework code + tests. No cam-sima changes -required. - -**Benefit**: closes the immediate bug, gives hosts the override -mechanism they need today (specifically for `diag_name`), unblocks -the advection test's deferred-property pattern. - -**Limit**: leaves `is_match` strict — hosts that disagree with a -scheme on `advected` still hit the "incompatible constituent" error. - -### Proposal B — class A/B split + setters - -**Scope** (in addition to A): -- Relax `is_match` to check only class A (`units` and possibly - `mixing_ratio_type`). -- Make all class-B properties optional in `%instantiate` with sane - defaults; deprecate (but keep accepting) class-B kwargs. -- Adopt the recommendation in Q4: silently dedup; host setters - override. -- Update `doc/constituents.md` with the register-then-override - workflow. -- (capgen-ng) Reject `diagnostic_name` on `is_constituent=True` - scheme args at parse time, or downgrade it to a default-only hint. -- (capgen-ng) **DONE 2026-05-13**: replaced `_FRAMEWORK_CONST_DIM_INPUTS` - with a `ResolvedArg.used_const_dim_std_names` field. - -**Cost**: ~150 lines framework + ~50 lines capgen-ng + tests. -CAM-SIMA host code can stay as-is (the 4 scheme-side registrations -continue to work with their existing class-B values; they're just -not enforced anymore). Optional: tidy the 4 schemes to pass class-A -only. - -**Benefit**: physics schemes become genuinely portable across -hosts. The class-B override pattern that CAM-SIMA already uses for -`thermo_active` and `water_species` generalizes. - -**Limit**: does not change the registration model (still -explicit-only in capgen-ng, still auto-clone in original capgen). - -### Proposal C — host-only registration - -**Scope** (in addition to B): -- Move the 4 cam-sima scheme-side register calls into a CAM-SIMA - helper module called from `cam_comp.F90`'s initialization. -- Drop register-phase `ccpp_constituent_properties_t(:)` support - from capgen-ng (and possibly original capgen). Schemes only - consume constituents; only the host registers. -- Codegen-time enforcement: any `advected=true` scheme arg whose - std_name is not in the host's enumeration → codegen error. -- Eliminates the `_dynamic_constituents` per-suite buffer - entirely. - -**Cost**: ~300 lines code total; requires coordinated PRs across -ccpp-framework, ccpp-capgen, ccpp-capgen-ng, atmospheric_physics, and -CAM-SIMA. The 4 schemes need their `_register` routines deleted (or -made no-ops); the host needs a new helper. - -**Benefit**: one source of truth for what constituents exist -(the host). Removes the auto-clone / scheme-register conceptual -overlap. Simplifies generator and runtime. - -**Limit**: changes the contract for the 4 scheme authors. Risk of -breaking yet-undiscovered downstream users of the scheme-side -registration model. - -### Comparison - -| Aspect | A | B | C | -|---|---|---|---| -| Lines changed | ~50 | ~200 | ~500+ | -| Coordination needed | framework only | framework + capgen-ng | framework + both generators + cam-sima | -| Fixes the crash | yes | yes | yes | -| Fixes `diag_name` portability | yes (host overrides) | yes | yes | -| Relaxes `is_match` | no | yes | yes | -| Removes scheme-side register | no | no | yes | -| Risk to existing CAM-SIMA workflows | none | low | medium | - -### Recommendation - -**Adopt A immediately (mostly done), aim for B over the next 4–6 -weeks, table C until the framework PR for B is in and we have a -clearer signal on whether the scheme-side register pattern is worth -keeping.** - ---- - -## 9. Appendix: framework setter inventory - -(For reference during the meeting. Reproduced from -`design_constituents_mutability.md`.) - -`ccpp_constituent_properties_t` methods (`src/ccpp_constituent_prop_mod.F90`): - -``` -Instantiation - procedure :: instantiate ! takes std_name, long_name, diag_name (REQUIRED), - ! units, vertical_dim, plus optional - ! advected, default_value, min_value, molar_mass, - ! water_species, mixing_ratio_type - procedure :: deallocate - -Getters (subset) - procedure :: standard_name - procedure :: long_name - procedure :: diagnostic_name - procedure :: units - procedure :: vertical_dimension - procedure :: is_advected - procedure :: is_thermo_active - procedure :: is_water_species - procedure :: is_mass_mixing_ratio - procedure :: is_volume_mixing_ratio - procedure :: is_number_concentration - procedure :: is_dry / is_moist / is_wet - procedure :: minimum - procedure :: molar_mass - procedure :: default_value - procedure :: has_default - procedure :: is_framework_owned ! NEW 2026-05-12 - -Setters (changes after instantiate) - procedure :: set_const_index - procedure :: set_thermo_active - procedure :: set_water_species - procedure :: set_minimum - procedure :: set_molar_mass - procedure :: set_framework_owned ! NEW 2026-05-12 - procedure :: set_advected ! GAP - procedure :: set_diagnostic_name ! GAP - procedure :: set_default_value ! GAP (or keep class A) - procedure :: set_mixing_ratio_type ! GAP (if class B) - -Identity / equality - procedure :: equivalent ! full equality - procedure :: is_match ! checks units + (class-B props ← too strict) -``` - -`ccpp_constituent_prop_ptr_t` is the pointer wrapper. Has parallel -setters that delegate to the underlying `ccpp_constituent_properties_t`. - ---- - -## Cross-references - -- `doc/constituents.md` — capgen-ng's user-facing constituents reference. -- `design_constituent_api.md` (memory) — capgen-ng's per-instance option-A design. -- `design_constituents_mutability.md` (memory) — extended design notes incl. class A/B classification. -- `project_implementation_status.md` (memory) — current implementation state and deferred items. -- `scripts/constituents.py` — original capgen's host-cap generator. -- `src/ccpp_constituent_prop_mod.F90` — framework. -- `capgen-ng/generator/host_constituents.py` — capgen-ng's host-side module emitter. -- `capgen-ng/generator/suite_resolver.py` (`_resolve_constituent_arg`) — capgen-ng's resolver routing. -- `EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` — CAM-SIMA's host-side wrappers around framework setters. - diff --git a/doc/constituents_overhaul_20260519T0905.md b/doc/constituents_overhaul_20260519T0905.md deleted file mode 100644 index 2bf6fad8..00000000 --- a/doc/constituents_overhaul_20260519T0905.md +++ /dev/null @@ -1,947 +0,0 @@ -# CCPP Constituents — Architecture Review & Overhaul Discussion - -**Authors:** Dom Heinzeller (lead), Claude (assistant) -**Date drafted:** 2026-05-12 -**Last revised:** 2026-05-18 -**Intended audience:** CCPP framework team, CAM-SIMA team -**Status:** Discussion document — no decisions are final. Proposals -A/B/C below remain pending the upcoming meeting; the bug fix from -Proposal A (the `ccpt_deallocate` ownership flag) and the capgen-ng -internal cleanup from Proposal B (§4.8) have landed; the missing -setters from Proposal A and the `is_match` relaxation from Proposal B -have not. Independent of A/B/C, the per-suite dynamic_constituents -buffer was made per-instance on 2026-05-18 to fix a multi-instance -mutation conflict — see §4.13. - ---- - -## Executive summary - -CCPP's "constituent" mechanism — how schemes declare and how the framework -manages tracer species like water vapor, cloud liquid, prescribed ozone, -etc. — has grown organically over the last few years. The result works, -but it carries: - -- **A latent framework bug** in `ccpp_constituent_prop_mod` that crashes on - teardown of explicitly-registered (target-passed) constituent property - arrays. Fixed in capgen-ng's framework copy 2026-05-12; needs to land - upstream. -- **Architectural confusion** about which properties are *physics-portable* - (the scheme owns them) versus *host-configuration* (the host owns them). - Today schemes are forced to supply host-specific values (`diag_name` is - the worst offender) at `%instantiate` time. -- **Setter API gaps**: properties that the host wants to override after - scheme-side registration (`advected`, `diagnostic_name`, `default_value`) - have no setters; `is_match` is overly strict about properties hosts - should be free to change. -- **Two registration models** coexist — original capgen's auto-clone of - is_constituent scheme args, and capgen's/capgen-ng's explicit register-phase + - host-side declaration. Capgen-ng deliberately dropped auto-clone. - -This document is a structured brief for a discussion this week. It does -NOT pre-commit to any decision; it lays out what exists, what's broken, -what we audited, and what proposals are on the table. - ---- - -## Table of contents - -1. [How original capgen handles constituents](#1-how-original-capgen-handles-constituents) -2. [How capgen-ng handles constituents](#2-how-capgen-ng-handles-constituents) -3. [What CAM-SIMA actually needs (audit)](#3-what-cam-sima-actually-needs-audit) -4. [Bugs and design flaws](#4-bugs-and-design-flaws) -5. [Property classification (Class A vs Class B)](#5-property-classification-class-a-vs-class-b) -6. [What to remove, replace, improve](#6-what-to-remove-replace-improve) -7. [Open design questions](#7-open-design-questions) -8. [Three proposals — minimal / clean / deep](#8-three-proposals--minimal--clean--deep) -9. [Appendix: framework setter inventory](#9-appendix-framework-setter-inventory) - ---- - -## 1. How original capgen handles constituents - -### 1.1 Mental model - -Original capgen treats constituents as a **separate scope** between -suite and host: - -``` -group → suite → ConstituentVarDict → host -``` - -A scheme arg flagged `constituent = True` in metadata is matched first -against group/suite/ConstituentVarDict, and only against host as a last -resort. The ConstituentVarDict is a synthetic dictionary whose entries -are auto-created by `find_variable()` when a scheme metadata declares a -constituent dependency. - -### 1.2 Auto-clone of `is_constituent` scheme args - -Every scheme arg with non-default `advected`, `constituent`, or -`molar_mass` is treated as a *registration*. The generator emits, into -the host cap, a routine `_constituents_ccpp_create_constituent_array` -that: - -1. Allocates a `ccpp_constituent_properties_t` pointer per scheme arg. -2. Calls `%instantiate(...)` populating fields **from the scheme - metadata directly** — `std_name`, `long_name`, `diagnostic_name`, - `units`, `default_value`, `advected`, `vertical_dim`, etc. (See - `scripts/constituents.py:565`.) -3. Adds it to the model constituents object via `%new_field`. - -After this auto-clone runs, the host's hand-written -`host_constituents(:)` array is appended, then `%lock_table` finalizes -the hash table. - -### 1.3 The host-cap-owned `ccpp_model_constituents_obj` - -Original capgen generates **one** `ccpp_model_constituents_obj` per -generator invocation, declared module-level in `_ccpp_cap.F90`. -Single global; not per-instance. (CAM-SIMA runs one host per -executable, so single-instance is fine for them.) - -### 1.4 Scheme-side `%instantiate` registration (the other path) - -A scheme may also register constituents via a register-phase argument: - -```fortran -type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) -``` - -The scheme allocates the array, calls `%instantiate` per entry, and -returns it. Original capgen wires this through a per-suite -"dynamic constituents" buffer and merges it during host-cap setup, -alongside the auto-cloned set. - -So original capgen really supports **three** registration sources: - -- Host: hand-written `host_constituents(:)` arg. -- Suite-dynamic: register-phase scheme args. -- Suite-static: auto-cloned from any `is_constituent` consumer. - -All three flow into one `%new_field` table. - -### 1.5 Lifecycle - -- `_ccpp_register_constituents(host_constituents, ...)` runs the - three-source merge. -- `ccpp_initialize_constituent_ptr(const_obj, ...)` (in - `ccpp_scheme_utils`) caches a pointer for the `ccpp_constituent_index` - lookup. -- Phase entry points access `vars_layer` / `vars_layer_tend` via cached - `index_of_` integers. - -### 1.6 What's good about original capgen's approach - -- Schemes declare a constituent dependency once in metadata; no manual - Fortran registration ever needed for "static" tracers. -- Host doesn't have to enumerate every species every scheme wants. -- Works for CAM-SIMA's current scheme catalog. - -### 1.7 What's painful about original capgen's approach - -- The auto-clone path is **invisible** to anyone reading the scheme - Fortran — the registration happens in generated code. -- `ConstituentVarDict` is a synthetic scope, conceptually subtle, and - doesn't generalize cleanly to multi-instance. -- The auto-clone path lifts `diagnostic_name` and `default_value` from - scheme metadata, but those values are often host-specific (see §4.4). -- Three sources of registration with overlap mean two registrations of - the same `std_name` may collide; original capgen relies on - `is_match` (units, advected, thermo_active, water_species) to dedup, - which means schemes accidentally diverge on `advected` and trip the - "incompatible constituent" error. - ---- - -## 2. How capgen-ng handles constituents - -### 2.1 Mental model - -No synthetic scope. Constituents are *one of four* sources for any -scheme arg: - -``` -control | host | suite | constituent -``` - -The resolver classifies each scheme arg into exactly one source. A -`constituent` source means the value will be accessed at runtime as -`ccpp_model_constituents_obj()%vars_layer(:, :, index_of_)` -(or `%vars_layer_tend(...)` for `tendency_of_` outputs). - -### 2.2 The four scheme-author rules - -(See `doc/constituents.md` for full details; this is the summary.) - -1. **Register** — register-phase scheme args of type - `ccpp_constituent_properties_t(:), intent=out, allocatable` declare - new constituents the scheme contributes. -2. **Consume** — physics-phase scheme args with `advected=true` (or - `molar_mass=...` or `constituent=true`) and `intent=in/inout`, with - the constituent's standard name, read the base species. -3. **Produce a tendency** — physics-phase scheme args with - `constituent=true`, `intent=out`, and standard name - `tendency_of_`, write the tendency. -4. **Mismatched combinations are errors** — `intent=out` on a base - constituent, or `intent=in` on a tendency, are codegen-time errors. - -### 2.3 Two registration sources (no auto-clone) - -- **Host**: hand-written `host_constituents(:)`, passed into - `ccpp_register_constituents(host_constituents, instance_number, ...)`. -- **Suite-dynamic**: register-phase scheme args, accumulated into a - per-suite buffer `_dynamic_constituents(:)` by `_register`, - drained into `ccpp_model_constituents_obj(inst)` by - `ccpp_register_constituents`. - -The auto-clone-from-metadata path is deliberately **gone**. If a scheme -declares `advected=true` on an arg but no source registers that -standard name, capgen-ng now emits a runtime check during -`ccpp_initialize_constituents` that errors with the missing name. - -### 2.4 Per-instance state - -Everything is per-instance: - -```fortran -type(ccpp_model_constituents_t), allocatable :: ccpp_model_constituents_obj(:) - ! indexed by instance_number -``` - -All host-facing entry points take `instance_number`: - -``` -ccpp_register_constituents (host_constituents, instance_number, errflg, errmsg) -ccpp_initialize_constituents (ncols, num_layers, instance_number, errflg, errmsg) -ccpp_number_constituents (num_flds, advected, instance_number, errflg, errmsg) -ccpp_gather_constituents (const_array, instance_number, errflg, errmsg) -ccpp_update_constituents (const_array, instance_number, errflg, errmsg) -ccpp_const_get_index (stdname, const_index, instance_number, errflg, errmsg) -ccpp_constituents_array (instance_number) => pointer -ccpp_advected_constituents_array (instance_number) => pointer -ccpp_model_const_properties (instance_number) => pointer -ccpp_deallocate_dynamic_constituents (instance_number, ...) -``` - -`ccpp_is_scheme_constituent(var_name, ...)` and the -`ccpp_model_const_stdnames(:)` parameter array are NOT per-instance — -the standard-name catalog is identical across instances. - -### 2.5 Lifecycle - -``` -ccpp_register(suite_name, instance_number, ...) - └─ _register → packs scheme-dynamic constituents into - _dynamic_constituents(instance)%items - (per-instance wrapper-DDT array; each instance - allocates and fills its own slot — see §4.13) - ↓ -ccpp_register_constituents(host_constituents, instance_number, ...) - └─ initialize_table(num_host_consts + num_suite_consts) - └─ new_field(host_consts ...) - └─ new_field(_dynamic_constituents ...) - └─ lock_table - ↓ -ccpp_initialize_constituents(ncols, num_layers, instance_number, ...) - └─ lock_data (allocates vars_layer, vars_layer_tend, vars_minvalue) - └─ ccpp_initialize_constituent_ptr (first instance wins; documented limit) - └─ %const_index('') for each enumerated constituent - └─ post-lookup int_unassigned check → clear error message - ↓ -ccpp_init(suite_name, instance_number, ...) - └─ _init → binds module-level pointers - ↓ -... physics phases ... - ↓ -ccpp_final(suite_name, instance_number, ...) - └─ _final → nullifies + last-to-leave deallocates - ↓ -ccpp_deallocate_dynamic_constituents(instance_number, ...) - └─ ccp_model_constituents_obj(inst)%reset - ↓ (in _final, last-to-leave) - deallocate(_dynamic_constituents) -``` - -### 2.6 What's good - -- Explicit. Every constituent registration is visible in someone's - Fortran source. -- Multi-instance from day one. -- The "four rules" are small enough to fit on a slide. -- Resolver-time + codegen-time + runtime checks catch the most common - mistakes. - -### 2.7 What's still painful - -Covered in §4. - ---- - -## 3. What CAM-SIMA actually needs (audit) - -### 3.1 Scheme-side registration usage - -We audited `EXT/cam-sima/atmospheric_physics/schemes/` for use of the -register-phase `ccpp_constituent_properties_t(:)` pattern: - -| Scheme | File | Registers | -|---|---|---| -| RRTMGP constituents | `schemes/rrtmgp/rrtmgp_constituents.meta` | radiative-active species | -| MUSICA chemistry | `schemes/musica/musica_ccpp.meta` | chemical species from MUSICA | -| Prescribed aerosols | `schemes/chemistry/prescribed_aerosols.meta` | aerosol species | -| Prescribed ozone | `schemes/chemistry/prescribed_ozone.meta` | ozone | - -**Total: 4 of 128 schemes** in the atmospheric_physics tree use -scheme-side registration. The other 124 only **consume** constituents -(`advected=true` + `intent=in/inout` in metadata, accessed via the -framework's `vars_layer`). - -This is a small enough number that an alternative "host-only -registration" model is feasible: move those 4 register calls into the -host (or into helper modules the host calls), and the rest of the -catalog only consumes. - -### 3.2 Host-side patterns - -`EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` wraps -the framework setters and exposes: - -- `const_set_thermo_active(const_obj | const_ind, value)` -- `const_set_water_species(const_obj | const_ind, value)` -- `const_set_minimum(...)` - -CAM-SIMA actively **calls these setters at runtime** — schemes don't -supply `thermo_active` at instantiate time; the host configures it -afterwards. This is direct evidence that the "post-instantiation -override" pattern is real and used today, and that the framework's -setter API is load-bearing. - -### 3.3 What CAM-SIMA does **not** do - -- It does not rely on auto-clone for `diag_name`. The scheme-side - register calls in the 4 schemes do supply `diag_name`, but those - values are CAM-SIMA's; a different host would need different ones. -- It does not use `ccpp_constituent_index` (the - `ccpp_scheme_utils`-singleton-based lookup) extensively — most - access goes through the framework's `index_of_` integers. - -### 3.4 What CAM-SIMA's host-cap-owned constituent object looks like - -Because original capgen generates **one** `ccpp_model_constituents_obj` -per generator invocation, and CAM-SIMA uses one generator invocation per -executable, CAM-SIMA effectively runs single-instance today. A -multi-instance CAM-SIMA (sub-columns, ensembles) would expose the -single-global limitation immediately. - ---- - -## 4. Bugs and design flaws - -This section lists known issues across the three layers (framework, -original capgen, capgen-ng). Items marked **(FIXED)** were resolved -2026-05-12 and either are or will be PRs; items marked **(OPEN)** are -intentionally left for this discussion. - -### 4.1 Framework: `ccpt_deallocate` ownership bug (FIXED in capgen-ng tree, needs upstream PR) - -- **Location**: `src/ccpp_constituent_prop_mod.F90`, `ccpt_deallocate` - + `ccpt_set`. -- **Symptom**: `free(): invalid size` crash when - `ccp_model_const_reset` is called on a properly-locked table whose - entries came from pointer-assigned targets (the common pattern - under capgen-ng's explicit registration; also potentially under - original capgen's `host_constituents` path). -- **Root cause**: `ccpt_set` does pointer assignment (`this%prop => - const_ptr`); `ccpt_deallocate` does an unconditional - `deallocate(this%prop)`. The deallocate is correct only when the - caller allocated `const_ptr` on the heap and transferred ownership. -- **Why it didn't surface earlier**: original capgen's advection test - only calls `deallocate` once between a *failing* register and a - *successful* one — at that point `lock_table` has not populated - `const_metadata`, so the broken inner loop is skipped. Capgen-ng - triggers it because its teardown calls `reset` after a successful - lock. -- **Fix landed 2026-05-12**: added `framework_owns_me` private flag on - `ccpp_constituent_properties_t` (default `.false.`) with - `is_framework_owned()` getter and `set_framework_owned(value)` - setter; `ccpt_deallocate` now only deallocates when the flag is set. - Original capgen's auto-clone path in `scripts/constituents.py` - updated to call `set_framework_owned(.true.)` after `allocate`. - Diffs in `src/ccpp_constituent_prop_mod.F90` (and capgen-ng's - parallel copy) + `scripts/constituents.py`. -- **Status**: framework tests pass; capgen-ng unit-test suite (1127 passing - as of 2026-05-13) is green. Still needs upstream PR to ccpp-framework + - original ccpp-capgen. - -### 4.2 Framework: missing setters (OPEN) - -| Property | Optional in `%instantiate`? | Has setter? | `is_match`-checked? | -|---|---|---|---| -| `std_name` | required | — | (lookup key) | -| `long_name` | required | — | no | -| `diag_name` | required | **NO** | no | -| `units` | required | — | **yes** | -| `vertical_dim` | required | — | no | -| `advected` | optional (default .false.) | **NO** | **yes** | -| `default_value` | optional | **NO** | no | -| `min_value` | optional | `set_minimum` | no | -| `molar_mass` | optional | `set_molar_mass` | no | -| `water_species` | optional (default .false.) | `set_water_species` | **yes** | -| `mixing_ratio_type`| optional | **NO** | no | -| `thermo_active` | not in instantiate | `set_thermo_active` | **yes** | -| `const_index` | internal | `set_const_index` | no | - -**Pain points**: - -- `advected` is `is_match`-checked AND has no setter. Once registered, - immutable. If a scheme and the host disagree, you get the - "incompatible constituent" error and you cannot reconcile from - Fortran. -- `diag_name` is required (cannot be omitted at instantiate) AND has - no setter. A scheme must pick a value at registration time; that - value is then frozen. -- `default_value` is silently optional. If omitted, the constituent - array initializes to `huge(real)` and downstream comparisons fail - in surprising ways (we burnt half a day on this 2026-05-12). -- `thermo_active` is the only property in the "post-instantiate-only" - shape: it has a setter but isn't a `%instantiate` arg. The - asymmetry is confusing. - -### 4.3 Framework: `is_match` is too strict (OPEN) - -`is_match` (in `ccp_is_match`) checks `units`, `advected`, -`thermo_active`, `water_species`. Three of those four (`advected`, -`thermo_active`, `water_species`) are properties the host legitimately -overrides post-registration. Two registrations of the same `std_name` -with the same `units` but different `advected` should be a -duplicate-dedup (host wins), not a hard error. - -### 4.4 Framework: `diag_name` portability problem (OPEN) - -Diagnostic output names are host-specific. CAM-SIMA names cloud -liquid mixing ratio `CLDLIQ`; UFS would call it something else. Yet -`%instantiate` makes `diag_name` a *required* arg, forcing schemes to -either: - -- Pick a host-specific value (couples the scheme to a host), or -- Pick a "neutral" default that no host's diagnostic tooling - recognizes. - -The current de-facto pattern in CAM-SIMA scheme code is to pick a -CAM-SIMA-flavoured value and ship it. Any port to UFS would need to -either monkey-patch or fork the scheme. - -A clean fix: -1. Make `diag_name` optional at `%instantiate` (default to empty - string or `std_name`). -2. Add `set_diagnostic_name(value)` setter. -3. Host overrides per-registration after `ccpp_register_constituents`. - -### 4.5 Original capgen: implicit registration (OPEN — observation) - -The auto-clone path is generator magic. Reading scheme metadata -doesn't tell you whether the scheme's args result in registration; you -have to know that `advected=true` triggers it. This is a documentation -+ comprehension problem more than a bug. - -### 4.6 Original capgen: single-instance `ccpp_model_constituents_obj` (OPEN — limitation) - -The host cap declares one global. Multi-instance hosts would need to -either generate one cap per instance or restructure. - -### 4.7 Original capgen: `ConstituentVarDict` complexity (OPEN — observation) - -The synthetic scope between suite and host serves correctness but -adds a code path that most contributors don't read. If we drop it -(capgen-ng has), the variable-matching algorithm shrinks. - -### 4.8 Capgen-ng: `_FRAMEWORK_CONST_DIM_INPUTS` cleanup (LANDED 2026-05-13) - -`generator/static_api.py` no longer carries the hand-curated frozenset of -standard names; framework-constituent dimension references now ride on a -dedicated `used_const_dim_std_names` field on `ResolvedArg`. Closes the -"hand-curated → structured field" REVISIT note that was in the code. - -### 4.9 Capgen-ng: no codegen-time cross-check of scheme registration (OPEN) - -The resolver knows every `is_constituent` arg's standard name (in -`SuiteResolution.constituent_index_names`) but doesn't know what each -scheme's `_register` subroutine actually `%instantiate`s. Today's -guarantee is a runtime check (the `int_unassigned` validation we -added 2026-05-12). Stronger options: - -- (a) New metadata attribute `registers_std_names = a, b, c` on - register-phase tables; codegen errors at generation time. -- (b) Parse scheme `_register` Fortran for `%instantiate(std_name=…)` - calls and cross-check. -- (c) Keep runtime check as authoritative, document the gap. - -### 4.10 Capgen-ng: scheme-metadata `diagnostic_name` for is_constituent args is host-specific (OPEN) - -Same issue as §4.4 but in capgen-ng's metadata layer. Today's -`diagnostic_name` attribute on a scheme metadata arg flows into -`datatable.xml` and is then trusted as "the" diagnostic name. If we -adopt setter-based class-B overrides, this attribute should either be -dropped for constituent args or marked as a default-only hint. - -### 4.11 Capgen-ng: `ccpp_scheme_utils` singleton (OPEN — documented limit) - -`ccpp_initialize_constituent_ptr(const_obj)` stores a single module-level -pointer. Schemes that use `ccpp_constituent_index(stdname)` get that -pointer back. Under multi-instance, only the first instance's -pointer is retained — `ccpp_constituent_index` queries from -within a scheme will always reflect instance 1. CAM-SIMA's 4 -scheme-registering schemes don't rely on this; documented in -`doc/constituents.md` §8. Real fix requires either threading -`instance_number` through `ccpp_constituent_index` (interface -change) or maintaining a per-instance pointer table. - -### 4.12 Capgen-ng: drop `diagnostic_name_fixed`, keep only `diagnostic_name` (OPEN — proposed simplification) - -Today the metadata layer carries two mutually-exclusive scheme-arg -attributes: - -- `diagnostic_name = X` — emits `diagnostic_name="X"` in `datatable.xml`; - defaults to `local_name` when absent. -- `diagnostic_name_fixed = Y` — emits `diagnostic_name_fixed="Y"` in - `datatable.xml`; the `diagnostic_name` slot stays empty (no - auto-default to `local_name`). - -The behavioral difference is purely *which attribute name* host -tooling sees in `datatable.xml` — both attributes carry the same kind -of value (a Fortran-identifier-shaped string), and both are passed -through unmodified. `_fixed` is a signal to the host "use verbatim, do -not decorate or transform"; but `diagnostic_name = X` already means -exactly that — the cap code never decorates the value, and any host -tooling that wants to decorate would have to opt in by parsing a -separate attribute (or by syntactic convention on the value itself). - -**Proposal:** Remove `diagnostic_name_fixed` from the metadata layer -and the parser. Keep `diagnostic_name` with the existing defaulting -rule (explicit → use it; absent → fall back to `local_name`). Hosts -that today rely on the `_fixed` semantic ("don't auto-default to -`local_name`") get the same outcome by simply *setting* -`diagnostic_name` to the desired exact value. - -Touchpoints to retire: - -- `metadata/parse_tools/parse_checkers.py::check_diagnostic_fixed` and - the mutual-exclusion block at the top of `check_diagnostic_id`. -- `metadata_table.py::MetaVar._KNOWN_ATTRS` entry and the - `@property diagnostic_name` fallback that returns `''` when - `_diagnostic_name_fixed` is set. -- `generator/datatable.py:267-269` emission of the - `diagnostic_name_fixed` XML attribute. -- Existing unit-test coverage for `diagnostic_name_fixed` becomes - obsolete and is removed (not migrated). - -**Why it's worth doing as part of the overhaul:** the attribute has no -unique semantics that `diagnostic_name` can't express, and dropping it -shrinks the metadata-layer surface area at the same time the -`set_diagnostic_name(value)` framework setter (§4.4 / §4.10) is being -added on the framework side. Hosts that want runtime override get -`set_diagnostic_name`; hosts that want metadata-declared values get -`diagnostic_name`. There is no third use case that needs `_fixed`. - -**Risk:** non-CCPP-ng metadata in the wild may carry -`diagnostic_name_fixed`. Mitigation: a one-line legacy-mode rewrite -(`metadata/legacy_compat.py`) translates the deprecated attribute to -`diagnostic_name` at parse time with a loud warning, identical in -spirit to the existing `horizontal_loop_extent → horizontal_dimension` -shim. Remove the rewrite once known consumers are migrated. - -### 4.13 Capgen-ng: per-suite `dynamic_constituents` buffer was shared across instances (FIXED 2026-05-18) - -- **Location**: `capgen-ng/generator/host_constituents.py` (buffer - declaration + `ccpp_register_constituents` iteration); - `capgen-ng/generator/suite_cap.py::_register_lines` (the two-pass - count→allocate→pack inside `_register`). -- **Symptom**: with two or more instances and any register-phase - scheme that produces constituents, the second per-instance - `ccpp_register_constituents` call fails with `ccp_set_const_index - ccpp_constituent_properties_t const index is already set`. -- **Root cause**: the per-suite buffer - `_dynamic_constituents(:)` was declared as a single shared - 1-D array of `ccpp_constituent_properties_t`, filled exactly once on - first instance entry (`.not. allocated(buf)` gate). - `ccpp_register_constituents` then iterates that shared buffer per - instance and calls `%new_field(const_prop)` on each property - object. `%new_field` calls `ccp_set_const_index`, which **mutates - the property object** by writing `const_ind`. Instance 1 set - `const_ind` on every shared object; instance 2's call tripped the - "set exactly once" guard. -- **Latent companion bug**: the same shared-mutation pattern means - that once Proposal B's class-B setters (`set_advected`, - `set_diagnostic_name`, `set_water_species` per-instance, etc.) are - exercised, instance 1's setter call would silently corrupt instance - 2's view of the property. No "already set" guard exists on those - setters today. -- **Why it didn't surface earlier**: the advection end-to-end test is - single-instance; the instances end-to-end test has no constituents. - Surfaced by the new `instances_advection` combined test - (`end-to-end-tests/instances_advection/`) on first run. -- **Fix landed 2026-05-18**: the per-suite buffer is now a wrapper-DDT - array indexed by `instance_number`: - ```fortran - type :: ccpp_dyn_const_buffer_t - type(ccpp_constituent_properties_t), allocatable :: items(:) - end type - type(ccpp_dyn_const_buffer_t), allocatable, target :: _dynamic_constituents(:) - ``` - The outer array is allocated to `number_of_instances` on first call; - each instance independently runs the two-pass count+pack into its - own `%items` slot. `ccpp_register_constituents` iterates - `_dynamic_constituents(instance)%items` so each instance's - `new_field` calls operate on **distinct** property objects. - Scheme `_register` routines are now called N times instead of once - (negligible cost — typical register bodies are a few `%instantiate` - calls), in exchange for clean per-instance isolation. -- **Cost**: ~50 lines across the two generator emitters, plus updates - to six pinned unit tests. No CAM-SIMA / NEPTUNE / SCM coordination - needed (host-facing API unchanged). -- **Status**: framework tests pass; full unit-test suite (1319 tests) - is green; all 10 end-to-end tests pass. -- **Position relative to Proposals A/B/C**: orthogonal — none of the - three proposed touching the buffer. Independently adopted. - ---- - -## 5. Property classification (Class A vs Class B) - -Proposed in `design_constituents_mutability.md` 2026-05-12. Each -constituent property is conceptually owned by either the scheme -(physics-portable, immutable once instantiated) or the host -(host-configuration, mutable post-instantiation). - -### Class A — scheme-intrinsic (immutable) - -| Property | Why class A | -|---|---| -| `std_name` | Identity. Cannot change. | -| `long_name` | Human-readable name of the *species*. Not host-specific. | -| `units` | Physics correctness. `is_match`-checked. | -| `vertical_dim` | Scheme's structural expectation (interface vs layer). | -| `molar_mass` | Physical constant of the species. | -| `default_value` | (Debatable — see §7) Scheme-appropriate initial value. | - -### Class B — host-configuration (mutable post-instantiation) - -| Property | Why class B | -|---|---| -| `advected` | Whether the host's dycore advects this — host decision. | -| `diag_name` | Host-specific diagnostic system name. | -| `thermo_active` | Host model configuration. | -| `min_value` | Host runtime guardrail. | -| `water_species` | (Borderline — see §7) Physical classification but also host-config. | -| `mixing_ratio_type` | (Borderline — see §7) Depends on dycore convention. | - -### Consequences if adopted - -- `is_match` should check **only class A**. Today it checks 3 of 4 - class-B properties. -- Class B properties need setters. Today - `advected`, `diag_name`, (and `mixing_ratio_type` if it stays - class B) have none. -- `%instantiate` can demote class B from "required + optional" to - "all optional with sane defaults" — `diag_name=''`, - `advected=.false.`, etc. Schemes wouldn't need to set them at all. - ---- - -## 6. What to remove, replace, improve - -### Remove (or stop requiring) - -- **Scheme-metadata `diagnostic_name` on is_constituent args** — host - will override. Keep the attribute valid on non-constituent args - (where it's host tooling documentation, no portability issue). -- **`is_match` checks on advected / water_species / thermo_active** — - class B should not block dedup. -- **The `diag_name` requirement at `%instantiate`** — demote to - optional with `''` default. -- **(Not adopting)** Original capgen's auto-clone path. Already gone - in capgen-ng; this discussion does not propose bringing it back. - Listed for completeness because the option is in memory. - -### Replace - -- **`ConstituentVarDict`** as a concept — capgen-ng already runs - without it. If the framework or future generator code references - it, dropping is fine. -- **Single-global `ccpp_model_constituents_obj`** — capgen-ng's - per-instance array is the replacement. Original capgen could be - retrofitted, but the priority depends on whether multi-instance - enters the original capgen's roadmap. - -### Improve - -- **Add the missing setters**: `set_advected`, `set_diagnostic_name`, - `set_default_value` (if `default_value` becomes class B), - `set_mixing_ratio_type` (if class B). -- **Add a convenience routine** like - `ccpp_get_constituent_props_by_std_name(stdname, instance_number, prop_ptr, errflg, errmsg)` - so hosts can lookup a single constituent's property wrapper by - name without iterating. -- **Codegen-time cross-check** of scheme `_register` calls vs - metadata declarations (preferred: §4.9 option (a) — new - `registers_std_names` attr). -- **Document the lifecycle** clearly. `doc/constituents.md` is - ~960 lines; targeted additions for "register-then-override" - workflow once the new setters land. -- **Capgen-ng-internal cleanup** (LANDED 2026-05-13): replaced - `_FRAMEWORK_CONST_DIM_INPUTS` with a `used_const_dim_std_names` - field on `ResolvedArg`. - ---- - -## 7. Open design questions - -These are the calls we need to make in the meeting. - -### Q1. `default_value` — class A or class B? - -- **Class A argument**: the scheme knows what the species - should be initialized to (zero for "starts empty"; small positive - for "starts at background"); the host doesn't typically override. -- **Class B argument**: hosts may want non-default starting values - (chemistry runs with prescribed initial profiles). -- **Today's reality**: framework has no setter, so it's de-facto - class A. The advection-test issue 2026-05-12 surfaced because we - removed the `default_value=0._kind_phys` from cld_liq.F90's - scheme-side register and had no way to put it back; restoring it - in the scheme fixed the test but cements the class-A treatment. -- **Recommendation**: leave class A for now. Revisit when a real - host-override use case appears. - -### Q2. `water_species` — class A or class B? - -- The current `is_match` check on `water_species` treats it as - identity-defining (class A semantics). But the actual *meaning* of - the bit is mostly host bookkeeping ("does the dycore treat this as - water?"). CAM-SIMA has a `set_water_species` wrapper and uses it. -- **Recommendation**: class B, with the caveat that schemes whose - numerics depend on a constituent *being* water should declare that - in metadata as a hard requirement (different mechanism — not the - `is_match` machinery). - -### Q3. `mixing_ratio_type` — class A or class B? - -- The scheme's calculations assume `wrt_dry` or `wrt_moist`; this - feels class A. -- But hosts using different dycores might want to interpret the - same `std_name` differently — feels class B. -- **Recommendation**: class A. The mismatch should manifest as - different `std_name`s (`cloud_liquid_dry_mixing_ratio` vs - `cloud_liquid_wet_mixing_ratio`), not the same name with a runtime - override. Need cam-sima input. - -### Q4. After `is_match` relaxation: what happens on disagreement? - -- If two registrations of the same std_name agree on class A but - disagree on class B (e.g., `advected=.false.` from a scheme, - `advected=.true.` from the host), the second registration's class - B values should win without error. Effectively: the host overrides - the scheme. -- Order matters: today the host appends *after* the dynamic - constituents. Should we reverse so the host appends *first*? - Probably not — the "first registration wins on class A; host - setters override class B" model is conceptually clearer. -- **Recommendation**: silently dedup on matching class A; for class - B disagreements, the *later* registration's class B values are - ignored. Hosts use setters to override after registration - finalizes. - -### Q5. Should `%instantiate` accept class-B args at all? - -- **Option Y**: keep `%instantiate` accepting class B args (with - defaults). Schemes can supply them as hints; hosts can override. - Backward-compatible. -- **Option N**: remove class-B args from `%instantiate`. Schemes - *must* leave them to the host. Breaks the 4 cam-sima - scheme-registering schemes. -- **Recommendation**: option Y. The cost of breaking 4 schemes for - marginal clarity isn't worth it. - -### Q6. `ccpp_scheme_utils` singleton - -- Today: `ccpp_initialize_constituent_ptr(const_obj)` stores one - pointer module-wide. First instance wins. -- Fix options: - - (a) Maintain a per-instance pointer table; threading - `instance_number` through `ccpp_constituent_index`. - - (b) Document the limitation, route around it (no scheme uses - `ccpp_constituent_index` under multi-instance — capgen-ng - already enforces `index_of_` everywhere). -- **Recommendation**: (b). It's a one-line doc note and zero code - change. - ---- - -## 8. Three proposals — minimal / clean / deep - -### Proposal A — bugfix only - -**Scope**: -- Land the `ccpt_deallocate` ownership fix (done 2026-05-12). -- Update `scripts/constituents.py` for original capgen's auto-clone - path to pass `owned=.true.` (done). -- Add the three missing setters (`set_advected`, - `set_diagnostic_name`, `set_default_value`) without changing - semantics. Doesn't touch `is_match` or `%instantiate`. -- Document the gaps in `doc/constituents.md`. - -**Cost**: ~50 lines framework code + tests. No cam-sima changes -required. - -**Benefit**: closes the immediate bug, gives hosts the override -mechanism they need today (specifically for `diag_name`), unblocks -the advection test's deferred-property pattern. - -**Limit**: leaves `is_match` strict — hosts that disagree with a -scheme on `advected` still hit the "incompatible constituent" error. - -### Proposal B — class A/B split + setters - -**Scope** (in addition to A): -- Relax `is_match` to check only class A (`units` and possibly - `mixing_ratio_type`). -- Make all class-B properties optional in `%instantiate` with sane - defaults; deprecate (but keep accepting) class-B kwargs. -- Adopt the recommendation in Q4: silently dedup; host setters - override. -- Update `doc/constituents.md` with the register-then-override - workflow. -- (capgen-ng) Reject `diagnostic_name` on `is_constituent=True` - scheme args at parse time, or downgrade it to a default-only hint. -- (capgen-ng) **DONE 2026-05-13**: replaced `_FRAMEWORK_CONST_DIM_INPUTS` - with a `ResolvedArg.used_const_dim_std_names` field. - -**Cost**: ~150 lines framework + ~50 lines capgen-ng + tests. -CAM-SIMA host code can stay as-is (the 4 scheme-side registrations -continue to work with their existing class-B values; they're just -not enforced anymore). Optional: tidy the 4 schemes to pass class-A -only. - -**Benefit**: physics schemes become genuinely portable across -hosts. The class-B override pattern that CAM-SIMA already uses for -`thermo_active` and `water_species` generalizes. - -**Limit**: does not change the registration model (still -explicit-only in capgen-ng, still auto-clone in original capgen). - -### Proposal C — host-only registration - -**Scope** (in addition to B): -- Move the 4 cam-sima scheme-side register calls into a CAM-SIMA - helper module called from `cam_comp.F90`'s initialization. -- Drop register-phase `ccpp_constituent_properties_t(:)` support - from capgen-ng (and possibly original capgen). Schemes only - consume constituents; only the host registers. -- Codegen-time enforcement: any `advected=true` scheme arg whose - std_name is not in the host's enumeration → codegen error. -- Eliminates the `_dynamic_constituents` per-suite buffer - entirely. - -**Cost**: ~300 lines code total; requires coordinated PRs across -ccpp-framework, ccpp-capgen, ccpp-capgen-ng, atmospheric_physics, and -CAM-SIMA. The 4 schemes need their `_register` routines deleted (or -made no-ops); the host needs a new helper. - -**Benefit**: one source of truth for what constituents exist -(the host). Removes the auto-clone / scheme-register conceptual -overlap. Simplifies generator and runtime. - -**Limit**: changes the contract for the 4 scheme authors. Risk of -breaking yet-undiscovered downstream users of the scheme-side -registration model. - -### Comparison - -| Aspect | A | B | C | -|---|---|---|---| -| Lines changed | ~50 | ~200 | ~500+ | -| Coordination needed | framework only | framework + capgen-ng | framework + both generators + cam-sima | -| Fixes the crash | yes | yes | yes | -| Fixes `diag_name` portability | yes (host overrides) | yes | yes | -| Relaxes `is_match` | no | yes | yes | -| Removes scheme-side register | no | no | yes | -| Risk to existing CAM-SIMA workflows | none | low | medium | - -### Recommendation - -**Adopt A immediately (mostly done), aim for B over the next 4–6 -weeks, table C until the framework PR for B is in and we have a -clearer signal on whether the scheme-side register pattern is worth -keeping.** - ---- - -## 9. Appendix: framework setter inventory - -(For reference during the meeting. Reproduced from -`design_constituents_mutability.md`.) - -`ccpp_constituent_properties_t` methods (`src/ccpp_constituent_prop_mod.F90`): - -``` -Instantiation - procedure :: instantiate ! takes std_name, long_name, diag_name (REQUIRED), - ! units, vertical_dim, plus optional - ! advected, default_value, min_value, molar_mass, - ! water_species, mixing_ratio_type - procedure :: deallocate - -Getters (subset) - procedure :: standard_name - procedure :: long_name - procedure :: diagnostic_name - procedure :: units - procedure :: vertical_dimension - procedure :: is_advected - procedure :: is_thermo_active - procedure :: is_water_species - procedure :: is_mass_mixing_ratio - procedure :: is_volume_mixing_ratio - procedure :: is_number_concentration - procedure :: is_dry / is_moist / is_wet - procedure :: minimum - procedure :: molar_mass - procedure :: default_value - procedure :: has_default - procedure :: is_framework_owned ! NEW 2026-05-12 - -Setters (changes after instantiate) - procedure :: set_const_index - procedure :: set_thermo_active - procedure :: set_water_species - procedure :: set_minimum - procedure :: set_molar_mass - procedure :: set_framework_owned ! NEW 2026-05-12 - procedure :: set_advected ! GAP - procedure :: set_diagnostic_name ! GAP - procedure :: set_default_value ! GAP (or keep class A) - procedure :: set_mixing_ratio_type ! GAP (if class B) - -Identity / equality - procedure :: equivalent ! full equality - procedure :: is_match ! checks units + (class-B props ← too strict) -``` - -`ccpp_constituent_prop_ptr_t` is the pointer wrapper. Has parallel -setters that delegate to the underlying `ccpp_constituent_properties_t`. - ---- - -## Cross-references - -- `doc/constituents.md` — capgen-ng's user-facing constituents reference. -- `design_constituent_api.md` (memory) — capgen-ng's per-instance option-A design. -- `design_constituents_mutability.md` (memory) — extended design notes incl. class A/B classification. -- `project_implementation_status.md` (memory) — current implementation state and deferred items. -- `scripts/constituents.py` — original capgen's host-cap generator. -- `src/ccpp_constituent_prop_mod.F90` — framework. -- `capgen-ng/generator/host_constituents.py` — capgen-ng's host-side module emitter. -- `capgen-ng/generator/suite_resolver.py` (`_resolve_constituent_arg`) — capgen-ng's resolver routing. -- `EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` — CAM-SIMA's host-side wrappers around framework setters. - diff --git a/doc/migration_20260513T0733.md b/doc/migration_20260513T0733.md deleted file mode 100644 index a2e631df..00000000 --- a/doc/migration_20260513T0733.md +++ /dev/null @@ -1,545 +0,0 @@ -# Migrating from ccpp-prebuild / ccpp-capgen to capgen-ng - -This document captures the **user-facing differences** a host model author -or scheme author needs to know when moving metadata, suite XML, and host -Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to -**capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and -`doc/redesign_analysis.md` (analysis of the old systems). - -*Last revised: 2026-05-13.* Current unit-test suite: 1127 passing. - -**Repository layout** (post-2026-05-13 cleanup): tooling lives under -`capgen-ng/` (top-level of this repo). Unit tests live at the top -level in `unit-tests/`; end-to-end tests in `end-to-end-tests/`. Run -the unit suite from the repo root with `python -m pytest unit-tests/`. - -## Table of contents - -1. [Metadata format changes](#1-metadata-format-changes) - 1. [1.8 `horizontal_loop_extent` → `horizontal_dimension`](#18-horizontal_loop_extent--horizontal_dimension) -2. [Suite definition file (SDF) changes](#2-suite-definition-file-sdf-changes) -3. [Host Fortran requirements](#3-host-fortran-requirements) -4. [Generator CLI and build integration](#4-generator-cli-and-build-integration) -5. [Generated cap layout — what's new and what changed](#5-generated-cap-layout--whats-new-and-what-changed) -6. [Framework changes (constituents)](#6-framework-changes-constituents) - 1. [6.3 Host metadata wins over auto-provisioning](#63-host-metadata-wins-over-auto-provisioning-2026-05-12) -7. [Validator (`ccpp_validator.py`)](#7-validator) -8. [Known gaps and deferred items](#8-known-gaps-and-deferred-items) - ---- - -## 1. Metadata format changes - -### 1.1 Table types - -Four `type =` values in `[ccpp-table-properties]`: - -| Type | Contents | -|---------|---------------------------------------------------------| -| `control` | Control variables passed as ``ccpp_physics_*`` args. | -| `host` | Host-model variables imported via `use`. | -| `ddt` | Derived-type definitions. | -| `scheme` | Scheme metadata. | - -The legacy `type = module` (capgen) becomes `type = host`. The legacy -`TYPEDEFS_NEW_METADATA` Python dict (prebuild) is replaced by `type = ddt` -tables. See `doc/redesign_prompt.md` §3.2. - -### 1.2 New table-property attributes - -All optional inside the `[ccpp-table-properties]` block: - -| Attribute | Applies to | Description | -|-----------------------|-----------------------|-------------| -| `module_name` | scheme, host, ddt | Fortran module name; overrides "module name = table name" when they differ. | -| `dependencies` | any | Comma-separated list of file paths to compile. **May appear multiple times** in one block (new this session); single occurrence still accepted. | -| `dependencies_path` | any | Relative base for `dependencies` entries. Single-valued. | -| `source_path` | any | Relative path to the Fortran source. Single-valued. | -| `kind_spec` | any | `:=>spec` (or shorthand). May appear multiple times. | - -Example with multi-line dependencies (real CCPP physics pattern): - -``` -[ccpp-table-properties] - name = GFS_rrtmg_setup - type = scheme - module_name = GFS_rrtmg_setup # optional when names match - dependencies_path = ../../ - dependencies = tools/mpiutil.F90 - dependencies = hooks/machine.F - dependencies = Radiation/RRTMG/radlw_main.F90,Radiation/RRTMG/radsw_main.F90 -``` - -### 1.3 New per-variable attributes - -Inside a `[ var_name ]` section. All optional. - -| Attribute | Type | Default | Notes | -|------------------|------|---------|-------| -| `top_at_one` | bool | `False` | When host and scheme disagree, generator emits a vertical-flip transform with reverse-stride subscript on the host side. Meaningless on variables without a vertical dimension. | -| `constituent` | bool | `False` | Scheme metadata only. Marks the var as a constituent reference. | -| `advected` | bool | `False` | Scheme metadata only. | -| `molar_mass` | float | `0.0` | Scheme metadata only. | -| `diagnostic_name` | str | (defaults to `local_name`) | Host-tooling hint; mutually exclusive with `diagnostic_name_fixed`. | - -### 1.4 Sliced local names with long subscript indices - -Local names with array slices may carry CCPP standard names as subscript -tokens: - -``` -[ dqdt(:,:,index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array) ] - standard_name = ... -``` - -The 63-char Fortran-identifier limit is enforced only on the base -identifier (`dqdt`), not on subscript tokens (which are CCPP standard -names resolved at codegen time and routinely exceed 63 chars). - -### 1.5 Unit strings: bare vs explicit positive exponent - -`m2` and `m+2` (or any `` vs `+` -combo) are normalised internally and treated as equivalent. Pre-existing -unit-conversion entries don't need to be duplicated; either spelling -matches. - -### 1.6 Improved error messages - -- **Duplicate standard name**: error message now lists both colliding - access paths and hints at the "sibling DDT instance" pattern (when - applicable). -- **Subcycle bound unresolved**: error names the std_name and points - at the control/host metadata as the fix. -- **Instance-dim used without `instance_number`**: error explains the - paired-opt-in requirement (see §1.7). - -### 1.7 Optional `instance_number` / `number_of_instances` pair - -These two control variables are now **paired optional**: - -- Declare **both** (`instance_number` in `type=control`, - `number_of_instances` in `type=host`) → multi-instance API. -- Declare **neither** → single-instance API. Public entry points drop - `instance_number`; internal per-instance arrays size to length 1. -- Declare exactly one → hard error from the validator. - -Hosts that don't need multi-instance bookkeeping can drop both declarations. - -### 1.8 `horizontal_loop_extent` → `horizontal_dimension` - -ccpp-prebuild / original ccpp-capgen used `horizontal_loop_extent` as -the horizontal-axis std name in scheme metadata. capgen-ng uses -`horizontal_dimension` uniformly — the run-vs-non-run distinction -isn't expressed in scheme metadata anymore (host passes -`horizontal_loop_begin`/`horizontal_loop_end` as control vars and the -generated cap slices accordingly). - -Migration paths: - -1. **Edit the metadata** (recommended) — search-and-replace - `horizontal_loop_extent` → `horizontal_dimension` in every scheme - `.meta` you maintain. -2. **Use `--legacy-mode`** (transient) — pass `--legacy-mode` to both - `ccpp_capgen_ng.py` and `ccpp_validator.py` and the rename happens - at parse time. A loud warning banner prints at startup so the - rewrite is never invisible. This shim *will be removed*; treat - it as a runway, not a destination. - ---- - -## 2. Suite definition file (SDF) changes - -### 2.1 Schema v2.0 with nested-suite expansion - -Capgen-ng parses v2.0 SDFs and expands `` references -recursively at parse time. See `doc/redesign_prompt.md` §3 and the -`suite_v2_0.xsd` schema. - -### 2.2 `` with CCPP standard-name loop bound - -```xml - - effr_pre - -``` - -The `loop=` attribute accepts: - -- **Integer literal** (`loop="3"`) — emitted verbatim. -- **CCPP standard name** (`loop="num_subcycles_for_effr"`) — resolved - against host/control metadata; supports DDT-component access paths - (e.g. `phys_state%num_subcycles`). -- **Absent / empty** — treated as `loop="1"`. - -The loop-bound standard name is automatically included in the -introspection inputs list (`ccpp_physics_suite_variables` and -`_suite_host_data`). - -### 2.3 Nested `` elements - -```xml - - - effr_calc - - -``` - -Nested subcycles produce nested `do` loops in the generated cap. Loop -counter variables follow the convention: - -- Outermost / single-level: `ccpp_loop_counter`. -- Each deeper level: `ccpp_loop_counter_2`, `ccpp_loop_counter_3`, ... - -Effective iteration count = product of every level's `loop=` value. -`effr_calc` in the example runs 3·2 = 6 times. - -### 2.4 Suite-level `` and `` schemes - -```xml - - my_init_scheme - ... - my_final_scheme - -``` - -- Each element contains a **single** scheme name as text content. - Multiple `` children inside ``/`` is a schema - violation. (Group-shaped lists belong inside ``.) -- The named scheme's `init` / `final` phase metadata is resolved like - any other scheme phase; missing-phase metadata is a generator error. -- The scheme call is emitted inside `_init` / `_final` - with USE for the scheme module + per-arg host modules, and the - standard errflg check. -- Call ordering: - - `_init`: after all group `state_alloc` and - `suite_data_init_fields`, **before** the `CCPP_SUITE_FRAMEWORK_INITIALIZED` - state transition. - - `_final`: before the `CCPP_SUITE_UNREGISTERED` transition. - -**Accepted spellings**: `` and `` only. Legacy spellings -**``** (typo), **``** (correct long form), and -**``** are rejected with a clear error pointing at the -canonical short form. - -To exercise: - -1. Declare a scheme with `init` and/or `final` phases in its metadata - (minimal sig — just `errmsg` + `errflg` — is fine). -2. Reference it in the SDF as shown above. -3. Add the scheme's `.F90` to your build's source list. - ---- - -## 3. Host Fortran requirements - -### 3.1 Required control variables - -Every host's `type=control` table must declare: - -| Standard name | Fortran type | Purpose | -|-----------------------------------|--------------|-----------------------------------| -| `suite_name` | character | Drives suite dispatch | -| `horizontal_loop_begin` | integer | Lower chunk-bound | -| `horizontal_loop_end` | integer | Upper chunk-bound | -| `thread_number` | integer | Current thread | -| `number_of_threads` | integer | Total threads | -| `number_of_physics_threads` | integer | Physics-internal budget | -| `ccpp_error_code` | integer | Error flag | -| `ccpp_error_message` | character | Error message | - -Optional (paired — see §1.7): - -| Standard name | Fortran type | Table type | Purpose | -|-------------------------|--------------|------------|--------------------------------| -| `instance_number` | integer | control | Current instance index | -| `number_of_instances` | integer | host | Total instance count | - -### 3.2 Required entry-point call sequence - -``` -ccpp_register(suite_name, errflg, errmsg, [instance_number]) - └── per scheme that declares a register phase -ccpp_init(suite_name, errflg, errmsg, [instance_number]) - └── per scheme that declares an init phase -ccpp_physics_init(...) - └── physics phase routines per group: - ccpp_physics_init - ccpp_physics_timestep_init - ccpp_physics_run ← run-loop phase - ccpp_physics_timestep_final - ccpp_physics_final -ccpp_final(suite_name, errflg, errmsg, [instance_number]) -``` - -`instance_number` appears in every signature only when the host -declares the `instance_number` / `number_of_instances` pair (§1.7). - -### 3.3 Host module convention - -The Fortran module that exports a host metadata table's variables is -typically named after the table. When that's not the case, use the -`module_name` table-property override (§1.2): - -``` -[ccpp-table-properties] - name = test_host_data - type = host - module_name = mod_test_host_data -``` - ---- - -## 4. Generator CLI and build integration - -### 4.1 `ccpp_capgen_ng.py` invocation - -``` -python ccpp_capgen_ng.py \ - --host-files [,,...] \ - --scheme-files [,,...] \ - --suites [,,...] \ - --host-name \ - --output-root /ccpp \ - [--kind-type =[:]] \ - [--legacy-mode] \ - [--verbose] [--verbose] -``` - -`--kind-type` syntax: `=[:]`. When `:` is -omitted, `` must be an ISO_FORTRAN_ENV constant (REAL32/REAL64/...) -and the module defaults to `iso_fortran_env`. `kind_phys` is -auto-defaulted to `iso_fortran_env:REAL64` when not supplied. - -`--legacy-mode` (transient migration shim, will be removed): silently -rewrites legacy CCPP standard names that ccpp-prebuild / original -ccpp-capgen used to their capgen-ng equivalents at parse time. -Currently translates `horizontal_loop_extent` → `horizontal_dimension`. -Prints a loud warning banner at startup so the rewrite is never -invisible. Available on both `ccpp_capgen_ng.py` and `ccpp_validator.py` -(keep the flag consistent between the two when both are invoked from -CMake). All translation logic is isolated in -`metadata/legacy_compat.py` and tagged with `# legacy-compat:` comments -at every touchpoint, so the shim can be cleanly removed when migration -is complete. - -### 4.2 `ccpp_datafile.py` query CLI - -Generated `datatable.xml` carries: - -- `` — generated outputs (utilities/host_files/suite_files). -- `` — `.meta` and expanded SDF. -- `` — per-scheme call lists. -- `` — host/api/suite/group dictionaries. - -Query via `ccpp_datafile.py -- `. Flags include -`--dependencies`, `--capgen-files`, `--host-files`, `--utility-files`, -`--suite-list`, `--required-variables `, `--input-variables `, -`--output-variables `, `--host-variables`, `--show`. - -### 4.3 CMake helpers - -`cmake/ccpp_capgen.cmake` and `cmake/ccpp_validator.cmake` provide the -`ccpp_capgen(...)` and `ccpp_validator(...)` macros. `ccpp_datafile(...)` -queries datatable.xml at configure time. - ---- - -## 5. Generated cap layout — what's new and what changed - -### 5.1 Output files - -Always generated: - -- `ccpp_kinds.F90` — kind parameters. Listed under ``. -- `ccpp_static_api.F90` — public host-facing entry points + introspection routines. -- `ccpp__cap.F90` — per-suite dispatcher. -- `ccpp___cap.F90` — per-group phase implementations. -- `ccpp__data.F90` — suite-owned interstitial DDT + module-level array. -- `ccpp__types.F90` — pointer-wrapper types for optional args. -- `ccpp_.meta` — inspection artifact; matches the generated cap. -- `datatable.xml` — build-system + host-introspection metadata. - -When any scheme registers constituents: - -- `ccpp_host_constituents.F90` — owns `ccpp_model_constituents_obj(:)` - and the host-facing constituent API. - -### 5.2 Per-suite data: TARGET on the instance array - -`ccpp_suite_data(:)` carries the `TARGET` attribute: - -```fortran -type(ccpp__data_t), allocatable, target, public :: ccpp_suite_data(:) -``` - -This makes every `ccpp_suite_data(i)%component(...)` subobject a valid -pointer-assignment target — needed for transformation temps and -optional-arg pointer wrappers. - -### 5.3 Variable transformations - -The generator emits three kinds of transform on a per-arg basis: - -| Transform | Trigger | -|------------------|--------------------------------------------------| -| Unit conversion | `host.units != scheme.units` with a registered conversion entry. | -| Kind conversion | `host.kind != scheme.kind` (different strings). | -| Vertical flip | `host.top_at_one != scheme.top_at_one` on a var with a vertical dim. | - -These compose. A scheme arg that needs unit + flip emits a single -combined assignment through a transformation temp: - -```fortran -temp_l = 1.0E-3_kind_phys*host_var(lb:ub, nlev:1:-1) ! unit conversion: kind_phys to kind_phys; vertical flip (top_at_one mismatch) -call scheme_run(temp=temp_l, ...) -host_var(lb:ub, nlev:1:-1) = 1.0E+3_kind_phys*temp_l ! ... reverse ... -``` - -Identity unit conversions (registered for dimensionally-equivalent -spellings like `J kg-1 ↔ m2 s-2`, formula `'{var}'`) are not labelled -"unit conversion" in the comment. - -### 5.4 Subcycle emission - -```fortran -integer :: ccpp_loop_counter -integer :: ccpp_loop_counter_2 -... -do ccpp_loop_counter = 1, phys_state%num_subcycles ! outer - call scheme_pre(...) - do ccpp_loop_counter_2 = 1, 2 ! inner - call scheme_calc(...) - end do -end do -``` - -### 5.5 State machine - -Per-instance integer state arrays: - -- `ccpp_suite_state(:)` — suite-level (UNREGISTERED / REGISTERED / - FRAMEWORK_INITIALIZED). -- `ccpp_group_state(:)` — group-level (UNINITIALIZED / INITIALIZED / - IN_TIMESTEP). - -Single-instance hosts get length-1 arrays indexed with literal `1`. -See `doc/redesign_prompt.md` §7. - ---- - -## 6. Framework changes (constituents) - -### 6.1 `ccpp_constituent_prop_mod` ownership flag - -(Framework PR — needs upstream merge.) Adds: - -- `framework_owns_me` private flag on `ccpp_constituent_properties_t`, - default `.false.`. -- `set_framework_owned(value)` setter (call before - `obj%new_field(const_prop, ...)` when transferring ownership). -- `is_framework_owned()` getter. -- `ccpt_deallocate` only frees when the flag is set; otherwise just - nullifies. - -Backward-compatible. Original capgen's auto-clone path in -`scripts/constituents.py` has been updated to call the setter. - -### 6.2 capgen-ng constituent API - -(See `doc/constituents.md` for the full reference.) Highlights: - -- One `ccpp_model_constituents_obj(:)` array per generator invocation, - sized to `number_of_instances`. -- Host-facing API: - - `ccpp_register_constituents(host_constituents, instance_number, ...)` - - `ccpp_initialize_constituents(ncols, num_layers, instance_number, ...)` - - `ccpp_const_get_index(stdname, const_index, instance_number, ...)` - - `ccpp_constituents_array(instance_number) → pointer` - - `ccpp_advected_constituents_array(instance_number) → pointer` - - `ccpp_model_const_properties(instance_number) → pointer` - - `ccpp_number_constituents(num_flds, advected, instance_number, ...)` - - `ccpp_gather_constituents`, `ccpp_update_constituents` - - `ccpp_is_scheme_constituent(var_name, ...)` (not per-instance) -- Scheme-side registration: four rules — register-phase - `ccpp_constituent_properties_t(:)` arg, consume base via - `advected=true intent=in/inout`, produce tendency via - `constituent=true intent=out` + `tendency_of_` std name, mismatches - are codegen errors. - -### 6.3 Host metadata wins over auto-provisioning (2026-05-12) - -If the host declares a framework-named standard name -(`ccpp_constituents` / `ccpp_constituent_tendencies` / -`ccpp_constituent_properties` / `number_of_ccpp_constituents` / -`index_of_`) as a regular host variable, the resolver uses the -host's declaration and skips capgen-ng auto-provisioning. Matters -most for legacy hosts (GFS / SCM) that own their own tracer -indices — e.g. `[ntcw]` with `standard_name = -index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array` -resolves to the host's short local name `ntcw`, not a parallel -module-level integer named after the full standard name (which -would also blow Fortran's 63-char identifier limit). See -`doc/constituents.md` §3. - -Active design review for the next constituents iteration: -`doc/constituents_overhaul.md` (Class A vs Class B property -classification, three reform proposals). - ---- - -## 7. Validator - -`capgen-ng/ccpp_validator.py` — standalone Fortran-vs-metadata checker. -Today validates **scheme** metadata against scheme Fortran files -(subroutine signatures, optional args, paren-aware decl splitting). - -Continuation-line handling covers both free-form (`&` at trailing end -of prior line only) and fixed-form / dual-form (`&` at both ends, with -the leading marker at column 6). Comment-only and blank lines -interleaved between continuation lines are skipped as Fortran 90+ -permits. - -When the signature parser finds a subroutine but extracts zero args -while metadata declares many, the "Argument count mismatch" error -appends a HINT pointing at the parser rather than masquerading as a -real mismatch — common cause is an unsupported signature feature. - -**Known gap**: host-metadata validation is not yet implemented. When -invoked with non-scheme `.meta` files, the validator silently filters -to zero schemes and reports "Validation passed." Slated for revisit -after the e2e test suite settles (`unit_conv` + `variable_transform` -complete). See `project_validator_host_check_deferred.md` (memory). - ---- - -## 8. Known gaps and deferred items - -| Item | Status | -|--------------------------------------------|-----------------------------------------------| -| `ccpp_loop_counter` standard name inside nested subcycles | Maps to OUTERMOST loop var. None of cam-sima uses this; revisit if a scheme needs the innermost value. | -| Validator host-metadata check | Deferred; revisit after e2e tests stabilise. | -| Constituents overhaul (Class A/B + setters) | Discussion doc at `doc/constituents_overhaul.md`. | -| Framework setters: `set_advected`, `set_diagnostic_name`, `set_default_value` | Deferred; depends on constituents-overhaul decision. | -| Codegen-time scheme-registration cross-check | Deferred; would require new `registers_std_names` metadata attr. | -| `_FRAMEWORK_CONST_DIM_INPUTS` cleanup | **Done 2026-05-13**: hand-curated frozenset gone; framework-constituent dim refs ride on a dedicated `used_const_dim_std_names` field on `ResolvedArg`. | -| Suppress `ccpp_host_constituents.F90` when unused | Deferred; currently emitted for every build even when no scheme/host actually exercises the constituent system. Now *correct* (empty) for SCM-style hosts thanks to the host-wins rule, but still dead code. See `design_constituent_host_wins.md`. | -| Python linter / formatter pass | Deferred; pick `ruff` and apply across `capgen-ng/`. | -| Generated Fortran ↔ Codee formatter idempotency | Deferred; emitted `.F90` must round-trip cleanly through the project's Codee Fortran formatter. | -| `fortran_to_metadata` developer utility | Deferred; bootstraps a `.meta` skeleton from an existing `.F90` subroutine. | -| `--legacy-mode` shim removal | Transient; remove `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and every `# legacy-compat:` touchpoint when scheme metadata has migrated. | -| Original capgen auto-clone path | Intentionally dropped in favour of explicit registration; kept in memory as "Option B" fallback. | - ---- - -## Cross-references - -- `doc/redesign_prompt.md` — original design specification (sections - marked "historic" where the implementation has evolved). -- `doc/redesign_analysis.md` — analysis of the legacy ccpp-prebuild + - ccpp-capgen toolchains. -- `doc/constituents.md` — full constituents reference for capgen-ng. -- `doc/constituents_overhaul.md` — architecture review and reform - proposals for the next iteration. - diff --git a/doc/migration_20260519T0905.md b/doc/migration_20260519T0905.md deleted file mode 100644 index d69f41c3..00000000 --- a/doc/migration_20260519T0905.md +++ /dev/null @@ -1,729 +0,0 @@ -# Migrating from ccpp-prebuild / ccpp-capgen to capgen-ng - -This document captures the **user-facing differences** a host model author -or scheme author needs to know when moving metadata, suite XML, and host -Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to -**capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and -`doc/redesign_analysis.md` (analysis of the old systems). - -*Last revised: 2026-05-14 (end-of-day).* Current unit-test suite: 1229 passing. - -**Repository layout** (post-2026-05-13 cleanup): tooling lives under -`capgen-ng/` (top-level of this repo). Unit tests live at the top -level in `unit-tests/`; end-to-end tests in `end-to-end-tests/`. Run -the unit suite from the repo root with `python -m pytest unit-tests/`. - -## Table of contents - -1. [Metadata format changes](#1-metadata-format-changes) - 1. [1.8 `horizontal_loop_extent` → `horizontal_dimension`](#18-horizontal_loop_extent--horizontal_dimension) -2. [Suite definition file (SDF) changes](#2-suite-definition-file-sdf-changes) -3. [Host Fortran requirements](#3-host-fortran-requirements) -4. [Generator CLI and build integration](#4-generator-cli-and-build-integration) -5. [Generated cap layout — what's new and what changed](#5-generated-cap-layout--whats-new-and-what-changed) -6. [Framework changes (constituents)](#6-framework-changes-constituents) - 1. [6.3 Host metadata wins over auto-provisioning](#63-host-metadata-wins-over-auto-provisioning-2026-05-12) -7. [Validator (`ccpp_validator.py`)](#7-validator) -8. [Known gaps and deferred items](#8-known-gaps-and-deferred-items) - ---- - -## 1. Metadata format changes - -### 1.1 Table types - -Four `type =` values in `[ccpp-table-properties]`: - -| Type | Contents | -|---------|---------------------------------------------------------| -| `control` | Control variables passed as ``ccpp_physics_*`` args. | -| `host` | Host-model variables imported via `use`. | -| `ddt` | Derived-type definitions. | -| `scheme` | Scheme metadata. | - -The legacy `type = module` (capgen) becomes `type = host`. The legacy -`TYPEDEFS_NEW_METADATA` Python dict (prebuild) is replaced by `type = ddt` -tables. See `doc/redesign_prompt.md` §3.2. - -### 1.2 New table-property attributes - -All optional inside the `[ccpp-table-properties]` block: - -| Attribute | Applies to | Description | -|-----------------------|-----------------------|-------------| -| `module_name` | scheme, host, ddt | Fortran module name; overrides "module name = table name" when they differ. | -| `dependencies` | any | Comma-separated list of file paths to compile. **May appear multiple times** in one block (new this session); single occurrence still accepted. | -| `dependencies_path` | any | Relative base for `dependencies` entries. Single-valued. | -| `source_path` | any | Relative path to the Fortran source. Single-valued. | -| `kind_spec` | any | `:=>spec` (or shorthand). May appear multiple times. | - -Example with multi-line dependencies (real CCPP physics pattern): - -``` -[ccpp-table-properties] - name = GFS_rrtmg_setup - type = scheme - module_name = GFS_rrtmg_setup # optional when names match - dependencies_path = ../../ - dependencies = tools/mpiutil.F90 - dependencies = hooks/machine.F - dependencies = Radiation/RRTMG/radlw_main.F90,Radiation/RRTMG/radsw_main.F90 -``` - -### 1.3 New per-variable attributes - -Inside a `[ var_name ]` section. All optional. - -| Attribute | Type | Default | Notes | -|------------------|------|---------|-------| -| `top_at_one` | bool | `False` | When host and scheme disagree, generator emits a vertical-flip transform with reverse-stride subscript on the host side. Meaningless on variables without a vertical dimension. | -| `constituent` | bool | `False` | Scheme metadata only. Marks the var as a constituent reference. | -| `advected` | bool | `False` | Scheme metadata only. | -| `molar_mass` | float | `0.0` | Scheme metadata only. | -| `diagnostic_name` | str | (defaults to `local_name`) | Host-tooling hint; mutually exclusive with `diagnostic_name_fixed`. | - -### 1.4 Sliced local names with long subscript indices - -Local names with array slices may carry CCPP standard names as subscript -tokens: - -``` -[ dqdt(:,:,index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array) ] - standard_name = ... -``` - -The 63-char Fortran-identifier limit is enforced only on the base -identifier (`dqdt`), not on subscript tokens (which are CCPP standard -names resolved at codegen time and routinely exceed 63 chars). - -### 1.5 Unit strings: bare vs explicit positive exponent - -`m2` and `m+2` (or any `` vs `+` -combo) are normalized internally and treated as equivalent. Pre-existing -unit-conversion entries don't need to be duplicated; either spelling -matches. - -### 1.6 Improved error messages - -- **Duplicate standard name**: error message now lists both colliding - access paths and hints at the "sibling DDT instance" pattern (when - applicable). -- **Subcycle bound unresolved**: error names the std_name and points - at the control/host metadata as the fix. -- **Instance-dim used without `instance_number`**: error explains the - paired-opt-in requirement (see §1.7). - -### 1.7 Optional `instance_number` / `number_of_instances` pair - -These two control variables are now **paired optional** and both live -in the host's `type=control` table (symmetric with the -`thread_number` / `number_of_threads` pair): - -- Declare **both** in `type=control` → multi-instance API. Both flow - as control dummies through every lifecycle and physics-phase - signature. -- Declare **neither** → single-instance API. Public entry points drop - both args; internal per-instance arrays size to length 1. -- Declare exactly one → hard error from the validator. -- Declare `number_of_instances` in `type=host` → hard error - (must be `type=control`). - -Hosts that don't need multi-instance bookkeeping can drop both declarations. - -### 1.8 Deprecated standard names rewritten by `--legacy-mode` - -`--legacy-mode` is a transient migration shim that rewrites a small -set of deprecated standard names to their canonical capgen-ng -equivalents at parse time. The full table currently covers: - -| Deprecated (legacy) | Canonical (capgen-ng) | -|--------------------------------|--------------------------| -| `horizontal_loop_extent` | `horizontal_dimension` | -| `number_of_openmp_threads` | `number_of_threads` | - -Why each entry: - -* `horizontal_loop_extent` — ccpp-prebuild / original ccpp-capgen used - this for the horizontal-axis std name in scheme metadata. capgen-ng - uses `horizontal_dimension` uniformly; the run-vs-non-run distinction - isn't expressed in scheme metadata anymore (host passes - `horizontal_loop_begin` / `horizontal_loop_end` as control vars and - the generated cap slices accordingly). -* `number_of_openmp_threads` — legacy CCPP-physics hosts (CCPP-SCM - 17p8 in particular) size per-thread DDT containers by - `number_of_openmp_threads` (e.g. `physics%Interstitial`). capgen-ng - uses `number_of_threads`, which matches the `thread_number` control - variable, so the registered scalar-index dim table can substitute - `physics%Interstitial(thread_number)%…` automatically (see §3.4). - -The rewrite fires for both standard-name attributes AND dimension -tokens (so a host's `dimensions = (number_of_openmp_threads)` becomes -`dimensions = (number_of_threads)` before any further processing). - -Migration paths: - -1. **Edit the metadata** (recommended) — search-and-replace the - legacy names in every host / scheme `.meta` you maintain. -2. **Use `--legacy-mode`** (transient) — pass `--legacy-mode` to both - `ccpp_capgen_ng.py` and `ccpp_validator.py` and the renames happen - at parse time. A loud warning banner prints at startup, listing - every pair the shim is rewriting, so the substitution is never - invisible. This shim *will be removed*; treat it as a runway, - not a destination. - ---- - -## 2. Suite definition file (SDF) changes - -### 2.1 Schema v2.0 with nested-suite expansion - -Capgen-ng parses v2.0 SDFs and expands `` references -recursively at parse time. See `doc/redesign_prompt.md` §3 and the -`suite_v2_0.xsd` schema. - -### 2.2 `` with CCPP standard-name loop bound - -```xml - - effr_pre - -``` - -The `loop=` attribute accepts: - -- **Integer literal** (`loop="3"`) — emitted verbatim. -- **CCPP standard name** (`loop="num_subcycles_for_effr"`) — resolved - against host/control metadata; supports DDT-component access paths - (e.g. `phys_state%num_subcycles`). -- **Absent / empty** — treated as `loop="1"`. - -The loop-bound standard name is automatically included in the -introspection inputs list (`ccpp_physics_suite_variables` and -`_suite_host_data`). - -### 2.3 Nested `` elements - -```xml - - - effr_calc - - -``` - -Nested subcycles produce nested `do` loops in the generated cap. Loop -counter variables follow the convention: - -- Outermost / single-level: `ccpp_loop_counter`. -- Each deeper level: `ccpp_loop_counter_2`, `ccpp_loop_counter_3`, ... - -Effective iteration count = product of every level's `loop=` value. -`effr_calc` in the example runs 3·2 = 6 times. - -### 2.3.1 Passing the loop counter / extent to a scheme - -A scheme inside a `` block may consume the current iteration -counter and the total iteration count via two CCPP standard names: - -| Standard name | Fortran type | Meaning | -|----------------------|--------------|----------------------------------------------------------| -| `ccpp_loop_counter` | integer | Current subcycle iteration (1 … `ccpp_loop_extent`) | -| `ccpp_loop_extent` | integer | Total iterations — the `loop=` value on the `` | - -These are **loop-context control variables**: the host model does **not** -declare them. capgen-ng emits them automatically as locals in the -generated group cap (the `do` loop's induction variable for the counter, -the loop bound for the extent), and resolves any scheme arg requesting -them against those locals. - -Example scheme metadata fragment: - -``` -[iter] - standard_name = ccpp_loop_counter - units = index - dimensions = () - type = integer - intent = in -[niter] - standard_name = ccpp_loop_extent - units = index - dimensions = () - type = integer - intent = in -``` - -Place the scheme in a `` in the SDF: - -```xml - - sfc_diff - GFS_surface_loop_control_part1 - sfc_nst - -``` - -The generated group cap will emit `do ccpp_loop_counter = 1, 2` and call -the scheme with `iter = ccpp_loop_counter, niter = 2` (or the loop's -resolved local name when `loop=` is used). - -**Scope is the subcycle body.** A scheme that requests -`ccpp_loop_counter` / `ccpp_loop_extent` but is NOT inside a -`` block raises a clear parse-time error pointing at this -contract. - -**Nested-subcycle nuance** (see §8): nested-subcycle schemes that ask -for `ccpp_loop_counter` currently get the **outermost** loop's counter, -not the innermost. None of the in-tree physics catalogs use the -inner-counter case yet; revisit when one needs it. - -### 2.4 Suite-level `` and `` schemes - -```xml - - my_init_scheme - ... - my_final_scheme - -``` - -- Each element contains a **single** scheme name as text content. - Multiple `` children inside ``/`` is a schema - violation. (Group-shaped lists belong inside ``.) -- The named scheme's `init` / `final` phase metadata is resolved like - any other scheme phase; missing-phase metadata is a generator error. -- The scheme call is emitted inside `_init` / `_final` - with USE for the scheme module + per-arg host modules, and the - standard errflg check. -- Call ordering: - - `_init`: after all group `state_alloc` and - `suite_data_init_fields`, **before** the `CCPP_SUITE_FRAMEWORK_INITIALIZED` - state transition. - - `_final`: before the `CCPP_SUITE_UNREGISTERED` transition. - -**Accepted spellings**: `` and `` only. Legacy spellings -**``** (typo), **``** (correct long form), and -**``** are rejected with a clear error pointing at the -canonical short form. - -To exercise: - -1. Declare a scheme with `init` and/or `final` phases in its metadata - (minimal sig — just `errmsg` + `errflg` — is fine). -2. Reference it in the SDF as shown above. -3. Add the scheme's `.F90` to your build's source list. - ---- - -## 3. Host Fortran requirements - -### 3.1 Required control variables - -Every host's `type=control` table must declare: - -| Standard name | Fortran type | Purpose | -|-----------------------------------|--------------|-----------------------------------| -| `suite_name` | character | Drives suite dispatch | -| `horizontal_loop_begin` | integer | Lower chunk-bound | -| `horizontal_loop_end` | integer | Upper chunk-bound | -| `thread_number` | integer | Current thread | -| `number_of_threads` | integer | Total threads | -| `number_of_physics_threads` | integer | Physics-internal budget | -| `ccpp_error_code` | integer | Error flag | -| `ccpp_error_message` | character | Error message | - -Optional (paired — see §1.7): - -| Standard name | Fortran type | Table type | Purpose | -|-------------------------|--------------|------------|--------------------------------| -| `instance_number` | integer | control | Current instance index | -| `number_of_instances` | integer | control | Total instance count | - -### 3.2 Required entry-point call sequence - -``` -ccpp_register(suite_name, errflg, errmsg, [instance_number, number_of_instances]) - └── per scheme that declares a register phase -ccpp_init(suite_name, errflg, errmsg, [instance_number, number_of_instances]) - └── per scheme that declares an init phase -ccpp_physics_init(...) - └── physics phase routines per group: - ccpp_physics_init - ccpp_physics_timestep_init - ccpp_physics_run ← run-loop phase - ccpp_physics_timestep_final - ccpp_physics_final -ccpp_final(suite_name, errflg, errmsg, [instance_number, number_of_instances]) -``` - -The `(instance_number, number_of_instances)` pair appears in every -signature only when the host declares it (§1.7). Both flow uniformly -through lifecycle and physics-phase calls; the framework consumes -`number_of_instances` only at register/init time but carries it -elsewhere for API symmetry with `(thread_number, number_of_threads)`. - -### 3.3 Host module convention - -The Fortran module that exports a host metadata table's variables is -typically named after the table. When that's not the case, use the -`module_name` table-property override (§1.2): - -``` -[ccpp-table-properties] - name = test_host_data - type = host - module_name = mod_test_host_data -``` - -### 3.4 Registered scalar-index dimensions - -A small set of CCPP standard-name dimensions are *registered*: each -one is a count that capgen-ng auto-collapses to a paired scalar index -variable at every access site. - -| Count dim (in `dimensions = (...)`) | Index var (capgen-ng substitutes) | -|---|---| -| `number_of_instances` | `instance_number` | -| `number_of_threads` | `thread_number` | - -**Where these may appear**: ONLY on container DDT-instance variables in -the access path. Example: - -``` -[Interstitial] - standard_name = GFS_interstitial_type_instance - type = GFS_interstitial_type - dimensions = (number_of_threads) -``` - -Every scheme that reaches into `Interstitial%` will see the -generator emit `physics%Interstitial(thread_number)%` at the -call site — no metadata work required on the scheme side. - -**Two rules govern this:** - -1. *(generalized)* A container DDT-instance variable may carry any - registered scalar-index dim — single (`(number_of_threads)`) or - paired (`(number_of_instances, number_of_threads)`). Dims that - AREN'T registered flow through the normal slice machinery - (`horizontal_loop_begin:horizontal_loop_end`, `1:vertical_*`, …) - just like flat-array dims. -2. *(enforced — hard parse-time error)* A **leaf** variable - (intrinsic-typed or `external:` — the kind a scheme binds to) - **MUST NOT** declare a registered scalar-index dim. If you write:: - - [my_array] - type = real | kind = kind_phys - dimensions = (number_of_threads, horizontal_dimension) # ILLEGAL - - capgen-ng will reject it at parse time with a message pointing - at the wrap-in-DDT remediation pattern. Wrap the leaf in a - container DDT instead. - -The registered table lives in -[`capgen-ng/metadata/registered_dimensions.py`](../capgen-ng/metadata/registered_dimensions.py). -It carries a four-step recipe at the top of the file for adding new -pairings. - ---- - -## 4. Generator CLI and build integration - -### 4.1 `ccpp_capgen_ng.py` invocation - -``` -python ccpp_capgen_ng.py \ - --host-files [,,...] \ - --scheme-files [,,...] \ - --suites [,,...] \ - --host-name \ - --output-root /ccpp \ - [--kind-type =[:]] \ - [--legacy-mode] \ - [--verbose] [--verbose] -``` - -`--kind-type` syntax: `=[:]`. When `:` is -omitted, `` must be an ISO_FORTRAN_ENV constant (REAL32/REAL64/...) -and the module defaults to `iso_fortran_env`. `kind_phys` is -auto-defaulted to `iso_fortran_env:REAL64` when not supplied. - -`--legacy-mode` (transient migration shim, will be removed): silently -rewrites a small set of deprecated CCPP standard names to their -capgen-ng equivalents at parse time — see §1.8 for the full table -(`horizontal_loop_extent` → `horizontal_dimension`, -`number_of_openmp_threads` → `number_of_threads`). The rewrite fires -for both standard-name attributes AND dimension tokens. Prints a -loud warning banner at startup, enumerating every pair the shim is -rewriting, so the substitution is never invisible. Available on both -`ccpp_capgen_ng.py` and `ccpp_validator.py` (keep the flag consistent -between the two when both are invoked from CMake). All translation -logic is isolated in `metadata/legacy_compat.py` and tagged with -`# legacy-compat:` comments at every touchpoint, so the shim can be -cleanly removed when migration is complete. - -### 4.2 `ccpp_datafile.py` query CLI - -Generated `datatable.xml` carries: - -- `` — generated outputs (utilities/host_files/suite_files). -- `` — `.meta` and expanded SDF. -- `` — per-scheme call lists, **scoped to schemes that are - actually referenced by the loaded suites** (group phase calls + the - suite-level ``/`` hooks). Scheme metadata files passed - on the CLI but never referenced are silently dropped. -- `` — `dependencies = …` from host/control/ddt tables - (always) plus the same per-scheme list as `` (filtered to - the used set). Build systems that compile against - `ccpp_datafile.py --dependencies` therefore only pull in scheme deps - for compiled schemes; missing transitive deps in scheme metadata - surface as link errors and should be fixed in the `.meta` file. -- `` — host/api/suite/group dictionaries. - -Query via `ccpp_datafile.py -- `. Flags include -`--dependencies`, `--capgen-files`, `--host-files`, `--utility-files`, -`--suite-files`, `--scheme-files`, `--suite-list`, -`--required-variables `, `--input-variables `, -`--output-variables `, `--host-variables`, `--show`. - -`--suite-files` returns capgen-generated cap files (`ccpp__cap.F90`, -etc.). `--scheme-files` returns the **user-supplied scheme `.F90` sources** -that the loaded suites actually reference — the filtered compile manifest. -Each used scheme's source is resolved as `/.` -(extension preference order: `.F90`, `.f90`, `.F`, `.f`); missing files are -warned about and the canonical `.F90` guess is emitted so the build-system -query stays useful. - -### 4.3 CMake helpers - -`cmake/ccpp_capgen.cmake` and `cmake/ccpp_validator.cmake` provide the -`ccpp_capgen(...)` and `ccpp_validator(...)` macros. `ccpp_datafile(...)` -queries datatable.xml at configure time. - -### 4.4 No-op regeneration preserves mtimes - -Every generated file (caps, `datatable.xml`, `ccpp_kinds.F90`, expanded -SDFs, `.meta` artifacts) goes through `write_if_changed`: the new content -is staged to a sibling temp file under the output root and atomically -replaces the target only when the bytes actually differ. Reruns with -identical inputs therefore leave on-disk mtimes untouched, so CMake / -Make / Ninja do not trigger a downstream rebuild cascade. Matches the -behavior of legacy `ccpp-prebuild` / `ccpp-capgen`. The staging temp -file lives in the target's parent directory (always under -`--output-root`), so no `/tmp` access is required. - ---- - -## 5. Generated cap layout — what's new and what changed - -### 5.1 Output files - -Always generated: - -- `ccpp_kinds.F90` — kind parameters. Listed under ``. -- `_ccpp_cap.F90` — public host-facing entry points + introspection routines. - Filename and emitted `module _ccpp_cap` name are both driven by the - required `--host-name ` CLI argument so multiple host integrations - can co-exist in one executable. The public sub names inside - (`ccpp_register`, `ccpp_init`, `ccpp_physics_*`, `ccpp_final`) are - unchanged regardless of ``. -- `ccpp__cap.F90` — per-suite dispatcher. -- `ccpp___cap.F90` — per-group phase implementations. -- `ccpp__data.F90` — suite-owned interstitial DDT + module-level array. -- `ccpp__types.F90` — pointer-wrapper types for optional args. -- `ccpp__data.meta` — inspection artifact; pairs with `ccpp__data.F90` (`.meta` ↔ `.F90` filename convention). -- `datatable.xml` — build-system + host-introspection metadata. - -When any scheme registers constituents: - -- `ccpp_host_constituents.F90` — owns `ccpp_model_constituents_obj(:)` - and the host-facing constituent API. - -### 5.2 Per-suite data: TARGET on the instance array - -`ccpp_suite_data(:)` carries the `TARGET` attribute: - -```fortran -type(ccpp__data_t), allocatable, target, public :: ccpp_suite_data(:) -``` - -This makes every `ccpp_suite_data(i)%component(...)` subobject a valid -pointer-assignment target — needed for transformation temps and -optional-arg pointer wrappers. - -### 5.3 Variable transformations - -The generator emits three kinds of transform on a per-arg basis: - -| Transform | Trigger | -|------------------|--------------------------------------------------| -| Unit conversion | `host.units != scheme.units` with a registered conversion entry. | -| Kind conversion | `host.kind != scheme.kind` (different strings). | -| Vertical flip | `host.top_at_one != scheme.top_at_one` on a var with a vertical dim. | - -These compose. A scheme arg that needs unit + flip emits a single -combined assignment through a transformation temp: - -```fortran -temp_l = 1.0E-3_kind_phys*host_var(lb:ub, nlev:1:-1) ! unit conversion: kind_phys to kind_phys; vertical flip (top_at_one mismatch) -call scheme_run(temp=temp_l, ...) -host_var(lb:ub, nlev:1:-1) = 1.0E+3_kind_phys*temp_l ! ... reverse ... -``` - -Identity unit conversions (registered for dimensionally-equivalent -spellings like `J kg-1 ↔ m2 s-2`, formula `'{var}'`) are not labeled -"unit conversion" in the comment. - -### 5.4 Subcycle emission - -```fortran -integer :: ccpp_loop_counter -integer :: ccpp_loop_counter_2 -... -do ccpp_loop_counter = 1, phys_state%num_subcycles ! outer - call scheme_pre(...) - do ccpp_loop_counter_2 = 1, 2 ! inner - call scheme_calc(...) - end do -end do -``` - -### 5.5 State machine - -Per-instance integer state arrays: - -- `ccpp_suite_state(:)` — suite-level (UNREGISTERED / REGISTERED / - FRAMEWORK_INITIALIZED). -- `ccpp_group_state(:)` — group-level (UNINITIALIZED / INITIALIZED / - IN_TIMESTEP). - -Single-instance hosts get length-1 arrays indexed with literal `1`. -See `doc/redesign_prompt.md` §7. - -**Idempotent entry points.** `ccpp_physics_init`, `ccpp_physics_final`, and -`ccpp_final` are all silently idempotent — repeat calls return cleanly with -`errflg=0` rather than erroring. `ccpp_physics_final` additionally silent-skips -when issued *after* `ccpp_final` has torn the suite down (state array -deallocated on the last instance, or `== UNREGISTERED` on any other instance). -The other physics phases (`timestep_init`, `run`, `timestep_final`) still -hard-error on a state mismatch. `ccpp_init` does *not* silent-skip when the -state array is unallocated — there, "not allocated" really does mean -"you forgot `ccpp_register`" and continues to be a hard error. - ---- - -## 6. Framework changes (constituents) - -### 6.1 `ccpp_constituent_prop_mod` ownership flag - -(Framework PR — needs upstream merge.) Adds: - -- `framework_owns_me` private flag on `ccpp_constituent_properties_t`, - default `.false.`. -- `set_framework_owned(value)` setter (call before - `obj%new_field(const_prop, ...)` when transferring ownership). -- `is_framework_owned()` getter. -- `ccpt_deallocate` only frees when the flag is set; otherwise just - nullifies. - -Backward-compatible. Original capgen's auto-clone path in -`scripts/constituents.py` has been updated to call the setter. - -### 6.2 capgen-ng constituent API - -(See `doc/constituents.md` for the full reference.) Highlights: - -- One `ccpp_model_constituents_obj(:)` array per generator invocation, - sized to `number_of_instances`. -- Host-facing API: - - `ccpp_register_constituents(host_constituents, instance_number, ...)` - - `ccpp_initialize_constituents(ncols, num_layers, instance_number, ...)` - - `ccpp_const_get_index(stdname, const_index, instance_number, ...)` - - `ccpp_constituents_array(instance_number) → pointer` - - `ccpp_advected_constituents_array(instance_number) → pointer` - - `ccpp_model_const_properties(instance_number) → pointer` - - `ccpp_number_constituents(num_flds, advected, instance_number, ...)` - - `ccpp_gather_constituents`, `ccpp_update_constituents` - - `ccpp_is_scheme_constituent(var_name, ...)` (not per-instance) -- Scheme-side registration: four rules — register-phase - `ccpp_constituent_properties_t(:)` arg, consume base via - `advected=true intent=in/inout`, produce tendency via - `constituent=true intent=out` + `tendency_of_` std name, mismatches - are codegen errors. - -### 6.3 Host metadata wins over auto-provisioning (2026-05-12) - -If the host declares a framework-named standard name -(`ccpp_constituents` / `ccpp_constituent_tendencies` / -`ccpp_constituent_properties` / `number_of_ccpp_constituents` / -`index_of_`) as a regular host variable, the resolver uses the -host's declaration and skips capgen-ng auto-provisioning. Matters -most for legacy hosts (GFS / SCM) that own their own tracer -indices — e.g. `[ntcw]` with `standard_name = -index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array` -resolves to the host's short local name `ntcw`, not a parallel -module-level integer named after the full standard name (which -would also blow Fortran's 63-char identifier limit). See -`doc/constituents.md` §3. - -Active design review for the next constituents iteration: -`doc/constituents_overhaul.md` (Class A vs Class B property -classification, three reform proposals). - ---- - -## 7. Validator - -`capgen-ng/ccpp_validator.py` — standalone Fortran-vs-metadata checker. -Today validates **scheme** metadata against scheme Fortran files -(subroutine signatures, optional args, paren-aware decl splitting). - -Continuation-line handling covers both free-form (`&` at trailing end -of prior line only) and fixed-form / dual-form (`&` at both ends, with -the leading marker at column 6). Comment-only and blank lines -interleaved between continuation lines are skipped as Fortran 90+ -permits. - -When the signature parser finds a subroutine but extracts zero args -while metadata declares many, the "Argument count mismatch" error -appends a HINT pointing at the parser rather than masquerading as a -real mismatch — common cause is an unsupported signature feature. - -**Known gap**: host-metadata validation is not yet implemented. When -invoked with non-scheme `.meta` files, the validator silently filters -to zero schemes and reports "Validation passed." Slated for revisit -after the e2e test suite settles (`unit_conv` + `variable_transform` -complete). See `project_validator_host_check_deferred.md` (memory). - ---- - -## 8. Known gaps and deferred items - -| Item | Status | -|--------------------------------------------|-----------------------------------------------| -| `ccpp_loop_counter` standard name inside nested subcycles | Maps to OUTERMOST loop var. None of cam-sima uses this; revisit if a scheme needs the innermost value. | -| Validator host-metadata check | Deferred; revisit after e2e tests stabilize. | -| Constituents overhaul (Class A/B + setters) | Discussion doc at `doc/constituents_overhaul.md`. | -| Framework setters: `set_advected`, `set_diagnostic_name`, `set_default_value` | Deferred; depends on constituents-overhaul decision. | -| Codegen-time scheme-registration cross-check | Deferred; would require new `registers_std_names` metadata attr. | -| `_FRAMEWORK_CONST_DIM_INPUTS` cleanup | **Done 2026-05-13**: hand-curated frozenset gone; framework-constituent dim refs ride on a dedicated `used_const_dim_std_names` field on `ResolvedArg`. | -| Suppress `ccpp_host_constituents.F90` when unused | Deferred; currently emitted for every build even when no scheme/host actually exercises the constituent system. Now *correct* (empty) for SCM-style hosts thanks to the host-wins rule, but still dead code. See `design_constituent_host_wins.md`. | -| Python linter / formatter pass | Deferred; pick `ruff` and apply across `capgen-ng/`. | -| Generated Fortran ↔ Codee formatter idempotency | Deferred; emitted `.F90` must round-trip cleanly through the project's Codee Fortran formatter. | -| `fortran_to_metadata` developer utility | Deferred; bootstraps a `.meta` skeleton from an existing `.F90` subroutine. | -| `--legacy-mode` shim removal | Transient; remove `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and every `# legacy-compat:` touchpoint when scheme metadata has migrated. | -| `ccpp_datafile.py` query CLI rework | Deferred (2026-05-13); collapse `--host-files` / `--suite-files` / `--utility-files` into `--capgen-files`, then repurpose `--host-files` as a filtered list of **input** host metadata files (parallel to `--scheme-files`). Most hosts pack all host data into a handful of shared files, so the filtering pay-off is small — the draw is API symmetry. | -| Original capgen auto-clone path | Intentionally dropped in favor of explicit registration; kept in memory as "Option B" fallback. | - ---- - -## Cross-references - -- `doc/redesign_prompt.md` — original design specification (sections - marked "historic" where the implementation has evolved). -- `doc/redesign_analysis.md` — analysis of the legacy ccpp-prebuild + - ccpp-capgen toolchains. -- `doc/constituents.md` — full constituents reference for capgen-ng. -- `doc/constituents_overhaul.md` — architecture review and reform - proposals for the next iteration. - diff --git a/doc/redesign_analysis_20260513T0733.md b/doc/redesign_analysis_20260513T0733.md deleted file mode 100644 index 67252a5f..00000000 --- a/doc/redesign_analysis_20260513T0733.md +++ /dev/null @@ -1,2639 +0,0 @@ -# CCPP Framework Code Generator — Technical Analysis for Redesign - -*Analysis date: 2026-05-04. Clarifications added: 2026-05-05.* - -This document is a deep-dive technical analysis of the two existing CCPP Framework code generators — -`ccpp-prebuild` and `ccpp-capgen` — produced as input to a planned complete redesign. -It covers execution flow, data structures, feature sets, build system integration, and -key architectural differences. - ---- - -## Table of Contents - -1. [Background and motivation](#1-background-and-motivation) -2. [ccpp-prebuild — detailed analysis](#2-ccpp-prebuild--detailed-analysis) -3. [ccpp-capgen — detailed analysis](#3-ccpp-capgen--detailed-analysis) -4. [Shared infrastructure](#4-shared-infrastructure) -5. [Feature comparison](#5-feature-comparison) -6. [Build system integration](#6-build-system-integration) -7. [Key architectural differences](#7-key-architectural-differences) -8. [Design considerations for the redesign](#8-design-considerations-for-the-redesign) -9. [Real-world example: CCPP Single Column Model (SCM)](#9-real-world-example-ccpp-single-column-model-scm) -10. [Real-world example: CAM-SIMA (capgen)](#10-real-world-example-cam-sima-capgen) -11. [Real-world example: UFS Weather Model (prebuild)](#11-real-world-example-ufs-weather-model-prebuild) -12. [Real-world example: Navy NEPTUNE (prebuild, restricted)](#12-real-world-example-navy-neptune-prebuild-restricted) -13. [Cross-cutting design decision: how host data enters the cap chain](#13-cross-cutting-design-decision-how-host-data-enters-the-cap-chain) - ---- - -## 1. Background and motivation - -The CCPP Framework is a code generator that analyzes metadata describing variables required -by physical parameterizations in numerical weather prediction (NWP) models, compares them -against metadata provided by a host model, and generates Fortran interface ("cap") code that -connects the two. - -There are two generations of the generator: - -**`ccpp-prebuild`** (`scripts/ccpp_prebuild.py`): -- Simple, mostly procedural Python -- Used in: NOAA UFS Weather Model, Navy NEPTUNE, CCPP-SCM -- Extremely reliable in research, development, and operations -- Fewer capabilities; simpler design - -**`ccpp-capgen`** (`scripts/ccpp_capgen.py`): -- Highly complex, object-oriented Python taken to the extreme -- Used in: NCAR CAM-SIMA (still mostly a research/development model) -- Many advanced features designed but never implemented (funding/priority gaps) -- Notoriously difficult to develop; no remaining team member fully understands it - -**The original plan** was to update `ccpp-capgen` with missing features from `ccpp-prebuild` -and transition all models to it. **This plan has been abandoned** in favor of a complete -redesign that draws the best lessons from both generations. - -The immediate trigger for abandoning capgen was the failure — after considerable effort by -three developers — to make capgen pass DDT arguments to group caps the way prebuild does. -This is the root cause of capgen's severe performance problem (seconds for prebuild, -10+ minutes for capgen on the same suite set) and of its broken handling of optional -variables under Fortran compiler debugging flags. - ---- - -## 2. ccpp-prebuild — detailed analysis - -### 2.1 Command-line arguments and configuration - -Entry point: `scripts/ccpp_prebuild.py`, `main()`. - -Arguments parsed by `argparse`: - -| Argument | Required | Purpose | -|---|---|---| -| `--config` | yes | Path to host-model Python config module | -| `--suites` | no | Comma-separated suite names (without `.xml`) | -| `--builddir` | no | Override build directory from config | -| `--namespace` | no | Appended to static API module name | -| `--debug` | no | Insert Fortran array-size checks in generated caps | -| `--clean` | no | Remove generated files and exit | -| `--verbose` | no | Set logging to DEBUG | - -The `--config` file is a plain Python module imported dynamically via `importlib`. -Key variables it must define: - -| Config variable | Purpose | -|---|---| -| `VARIABLE_DEFINITION_FILES` | List of host-model Fortran sources with metadata hooks | -| `SCHEME_FILES` | List of physics scheme Fortran sources | -| `CAPS_DIR` | Output directory for generated cap `.F90` files | -| `SUITES_DIR` | Directory containing suite definition XML files | -| `STATIC_API_DIR` | Output directory for `ccpp_static_api.F90` | -| `TYPEDEFS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for typedef build snippets | -| `SCHEMES_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for scheme build snippets | -| `CAPS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for cap build snippets | -| `HTML_VARTABLE_FILE`, `LATEX_VARTABLE_FILE` | Documentation output paths | -| `TYPEDEFS_NEW_METADATA` | Optional: dict enabling DDT member name translation bridge | - -The config file can contain arbitrary Python expressions — computed file lists, -conditional logic, environment-variable lookups — making it very flexible. - -### 2.2 Step-by-step execution pipeline - -``` -1. Import config module dynamically via importlib - -2. gather_variable_definitions() - for each file in VARIABLE_DEFINITION_FILES: - parse_variable_tables(file) [metadata_parser.py] - → metadata_define: OrderedDict[standard_name → [mkcap.Var]] - -3. collect_physics_subroutines() - for each file in SCHEME_FILES: - parse_scheme_tables(file) [metadata_parser.py] - → metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] - → arguments_request: OrderedDict[scheme → OrderedDict[subroutine → [std_names]]] - → dependencies_request: OrderedDict[scheme → [abs_paths]] - → schemes_in_files: OrderedDict[scheme → abs_path] - -4. compare_metadata() [batch matching] - for each std_name in metadata_request: - check exists in metadata_define - check type/kind/rank compatibility - register unit conversions in var.actions - copy local_name as var.target - → metadata: OrderedDict[std_name → [Var]] (targets and actions set) - -5. check_optional_arguments() [warnings only] - -6. For each requested suite XML: - Suite.parse(xml) [mkstatic.py] → Suite + Group objects - Group.write() → ccpp___cap.F90 - Suite.write() → ccpp__cap.F90 - -7. API.write() [mkstatic.py] - → ccpp_static_api[_].F90 - -8. Write build-system snippets [mkcap.py writers] - → CCPP_CAPS.cmake/mk/sh - → CCPP_SCHEMES.cmake/mk/sh - → CCPP_TYPEDEFS.cmake/mk/sh - → CCPP_API.cmake/sh - -9. mkdoc.metadata_to_html() → HTML variable table - mkdoc.metadata_to_latex() → LaTeX variable table -``` - -### 2.3 Data structures — the "flat dict" model - -Everything in prebuild lives in flat Python `OrderedDict` structures. There is no object -hierarchy; variables are simple Python objects with plain attributes. - -```python -# Top-level data containers -metadata_define: OrderedDict[standard_name → [mkcap.Var]] # 1 Var per std_name -metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] # N Vars (one per scheme×subroutine) -arguments_request: OrderedDict[scheme_name → OrderedDict[subroutine_name → [std_names]]] -dependencies_request: OrderedDict[scheme_name → [abs_paths]] -schemes_in_files: OrderedDict[scheme_name → abs_path] -``` - -`mkcap.Var` attributes: - -| Attribute | Type | Description | -|---|---|---| -| `standard_name` | str | CF-convention unique identifier | -| `long_name` | str | Human-readable description | -| `units` | str | Physical units | -| `local_name` | str | Fortran local name (may be DDT member reference) | -| `type` | str | Fortran type (real, integer, logical, or DDT name) | -| `kind` | str | Fortran kind parameter | -| `dimensions` | list[str] | Dimension standard names | -| `intent` | str | in / out / inout | -| `active` | str | `'T'`, `'F'`, or expression string | -| `optional` | str | `'T'` or `'F'` | -| `pointer` | bool | Whether Fortran POINTER attribute needed | -| `target` | str | Set during matching: the host model local_name | -| `actions` | dict | `{'in': fn, 'out': fn}` for unit conversions | -| `container` | str | Encoded provenance: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | - -**Performance note on `container` and `target`**: these two attributes act as a lookup -cache computed once during the `compare_metadata()` batch step. The `container` string -encodes where each variable lives in the host model (module and, if applicable, the -DDT member chain). The `target` records the resolved Fortran local name. Both are -computed once and then used directly during Fortran cap generation — no further dictionary -lookups are needed. This is a major contributor to prebuild's speed advantage. - -### 2.4 Metadata parsing and the bridge to capgen - -`metadata_parser.py` is a shared module that acts as a bridge. It detects whether a -metadata section in a Fortran source file uses the old pipe-delimited format (deprecated, -warning emitted) or the new `.meta` format (triggered by `!! \htmlinclude .html` -in the Fortran source comment hook). - -For `.meta` files, `read_new_metadata()` in `metadata_parser.py`: -1. Calls capgen's `metadata_table.parse_metadata_file()` → `[MetadataTable]` -2. Converts each `metavar.Var` to a `mkcap.Var` -3. Normalizes `active` to `'T'`/`'F'`/expression, `optional` to `'T'`/`'F'` - -The `TYPEDEFS_NEW_METADATA` config variable (when provided) triggers an additional -pass via `convert_local_name_from_new_metadata()` which translates flat -standard-name-style local names into DDT member references such as -`Atm(blk_no)%q(:,:,:,graupel_index)`. This is the bridge that makes the newer -`.meta` format work with the older DDT-heavy host model code. - -### 2.5 Variable matching — `compare_metadata()` - -A single batch function processes all matching. For each standard name in `metadata_request`: - -1. Check it exists in `metadata_define` — error if missing -2. Check there is exactly one definition — error if ambiguous -3. Call `var.compatible(other_var)` — checks equality of `standard_name`, `type`, `kind`, and rank -4. Register unit conversions: if units differ, `var.convert_from()` / `var.convert_to()` - stores a conversion function in `var.actions` -5. Check `active` attribute: if host variable is conditionally allocated and scheme variable - is not `optional`, issue a warning (not an error) -6. Copy `local_name` from the define side as `var.target` -7. Build module use list from container strings - -Result: `metadata` dict where each `Var` has `.target` set to the host model local name -and `.actions` populated with any needed unit conversion functions. - -### 2.6 Generated Fortran files - -#### Group cap: `ccpp___cap.F90` - -One module per group. For each CCPP stage (tsinit, init, run, tsfinal, finalize), a subroutine: - -```fortran -module ccpp_suite_A_physics_cap - use scheme_module, only: scheme_run - use host_module_A, only: ddt_A ! DDT, not flat fields - use host_module_B, only: ddt_B - implicit none - contains - - subroutine suite_A_physics_run_cap(ddt_A, ddt_B, im, iaend, ierr, ...) - type(ddt_A_type), intent(inout), target :: ddt_A ! entire DDT passed - type(ddt_B_type), intent(inout), target :: ddt_B - integer, intent(in) :: im, iaend ! loop bounds - integer, intent(out) :: ierr - logical, save :: initialized(200) = .false. - ! optional variable: local pointer, conditionally associated - real(kind_phys), pointer :: opt_var(:) => null() - if (ddt_A%active_flag) then - opt_var => ddt_A%opt_field - end if - ! unit conversion: local variable - real(kind_phys) :: converted_var(im) - converted_var(:) = ddt_B%field(:im) * conversion_factor - ! fixed-index extraction: local pointer for a specific tracer - real(kind_phys), pointer :: q_water_vapor(:,:) => null() - q_water_vapor => ddt_A%q(:,:,ntqv) ! ntqv = water vapor index in tracer array - ! call scheme with loop-bound application and extracted variables at the call site - call scheme_run( & - arg1 = ddt_A%field1(1:im), & ! horizontal loop-bound applied here - arg2 = ddt_A%field2(1:im,:), & ! loop-bound + all levels - qv = q_water_vapor(1:im,:), & ! specific tracer, loop-bound applied - arg3 = converted_var, & ! unit-converted local var - opt_arg = opt_var, & ! optional pointer - ...) - if (ierr /= 0) return - end subroutine -end module -``` - -Key points: -- **DDTs are passed as arguments, not flat fields.** Hundreds of variables arrive as - one or a small number of DDT arguments. This is the fundamental architectural choice - that makes prebuild fast and safe with compiler debugging flags. -- **Two distinct "subsetting" operations happen at the scheme call site:** - 1. *Loop-bound application*: horizontal range `1:im` (or `im` for scalar extents) - applied in the scheme call argument expressions. - 2. *Fixed-index extraction*: a specific element along one dimension is selected, - e.g. `q_water_vapor => ddt%q(:,:,ntqv)` extracts the water vapor tracer from the - full tracer array. A local pointer (or local variable for unit conversions) is - declared just before the scheme call and passed as the scheme argument. The group - cap always receives the full data; these extractions are local to the group cap. -- **Optional variables** are handled by declaring a local `pointer` variable and - conditionally associating it with the DDT field based on the `active` expression. - An unassociated pointer is passed to the scheme if the variable is inactive. This - avoids compiler exceptions when mandatory debugging flags are enabled, because the - unallocated field is never directly referenced — only the already-null pointer is. -- `logical :: initialized(200), save` — per-instance initialization tracking. The - 200 is the maximum number of complete model instances that can coexist in memory - simultaneously (used in ensemble approaches where multiple copies of the full model - state live in memory at once). Each instance has its own initialization flag. -- For the `run` phase, `im` and `iaend` (or similar) carry `horizontal_loop_begin` - and `horizontal_loop_end`, enabling OpenMP thread-level parallelism where each - thread processes a horizontal slice. -- Explicit keyword argument passing in scheme calls. -- Unit conversion: a local variable is declared and populated before the call; the - local variable is then passed to the scheme. -- Error check after each scheme call; returns immediately on error. -- `--debug` flag inserts Fortran array-size assertions. - -#### Suite cap: `ccpp__cap.F90` - -Imports all group cap functions and exposes one function per stage that chains group calls. - -#### Static API: `ccpp_static_api[_].F90` - -A single Fortran module `ccpp_static_api` with one subroutine per stage: - -```fortran -subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) - character(len=*), intent(in) :: suite_name, group_name - select case(trim(suite_name)) - case('suite_A') - select case(trim(group_name)) - case('physics') - call suite_A_physics_run_cap(cdata, ierr) - ... - end select - ... - end select -end subroutine -``` - -This is the **single entry point** the host model calls. The host model passes `suite_name` -and `group_name` at runtime; the static API dispatches to the appropriate cap function. - -### 2.7 Build system snippet files generated - -Six output files (Makefile, CMakefile, shell source) for three variable sets: - -| File | Content | -|---|---| -| `CCPP_CAPS.cmake` | `set(CAPS /abs/path/cap1.F90 /abs/path/cap2.F90 ...)` | -| `CCPP_SCHEMES.cmake` | `set(SCHEMES /abs/path/scheme1.F90 ...)` | -| `CCPP_TYPEDEFS.cmake` | `set(TYPEDEFS module1 module2 ...)` (module names, not paths) | -| `CCPP_API.cmake` | `set(API /abs/path/ccpp_static_api.F90)` | - -All files are written as `.tmp` first and compared against the existing version; they are -replaced only if the content changed, which avoids unnecessary recompilation of downstream -Fortran targets. - -### 2.8 What `mkcap.py`, `mkstatic.py`, and `mkdoc.py` each do - -**`mkcap.py`**: -- Defines the `mkcap.Var` class (prebuild's variable data class) -- Defines six file-writer classes: `CapsMakefile`, `CapsCMakefile`, `CapsSourcefile`, - `SchemesMakefile`, `SchemesCMakefile`, `SchemesSourcefile`, `TypedefsMakefile`, - `TypedefsCMakefile`, `TypedefsSourcefile` -- Each writer has a `write(file_list)` method that produces a formatted include file -- Does NOT generate any Fortran - -**`mkstatic.py`**: -- Defines `Suite`, `Group`, `Subcycle` classes that parse suite definition XML and - generate Fortran caps -- `Suite.parse()`: reads SDF XML via `xml.etree.ElementTree`, builds `Group` and - `Subcycle` objects -- `Suite.write()`: drives cap generation for all groups and the suite-level cap -- `Group.write()`: generates the group cap Fortran — argument list construction, - module `use` statements, unit conversion code, scheme calls, error handling -- Defines `API` class: generates the static API Fortran module (suite_name/group_name - dispatch switch) -- `CCPP_SUITE_VARIABLES` dict: mandatory variables always included (error message, - error code, loop counter, loop extent) -- Helper functions `extract_parents_and_indices_from_local_name()` and - `extract_dimensions_from_local_name()` handle complex DDT member access like - `Atm(blk_no)%q(:,:,:,graupel_index)` — these are critical for DDT-heavy host models - -**`mkdoc.py`**: -- `metadata_to_html()`: produces an HTML table of all host-model provided variables - (standard_name, long_name, units, rank, type, kind, source, local_name) -- `metadata_to_latex()`: produces a LaTeX table combining host-defined and scheme-requested - variables, annotating which schemes use each variable and whether unit conversion is needed -- Informational outputs only; do not affect the build - ---- - -## 3. ccpp-capgen — detailed analysis - -### 3.1 Command-line arguments - -Entry point: `scripts/ccpp_capgen.py`, `_main_func()`. -Arguments parsed via `framework_env.parse_command_line()` into a `CCPPFrameworkEnv` object: - -| Argument | Required | Purpose | -|---|---|---| -| `--host-files` | yes | `.meta` files or `.txt` indirect file lists | -| `--scheme-files` | yes | Same format | -| `--suites` | yes | `.xml` SDF files or `.txt` lists | -| `--output-root` | no | Directory for generated files | -| `--host-name` | no | If given, generates a host cap | -| `--ccpp-datafile` | no | Path for datatable XML (default: `datatable.xml`) | -| `--kind-type` | no (repeatable) | Fortran kind mappings, syntax `=[:]`. Module defaults to `iso_fortran_env` for ISO_FORTRAN_ENV specs. Examples: `kind_phys=REAL64`, `kind_phys=my_host_kinds:kind_r8`. If omitted, `kind_phys=iso_fortran_env:REAL64` is injected. | -| `--preproc-directives` | no | Fortran preprocessor macros | -| `--use-error-obj` | no | Use error object instead of scalar error variables | -| `--force-overwrite` | no | Always regenerate output | -| `--clean` | no | Remove files listed in datatable and exit | -| `--verbose` | no (repeatable) | Increase log verbosity | - -`CCPPFrameworkEnv` (defined in `framework_env.py`) consolidates all settings into typed -properties and stores a `kind_dict` mapping CCPP kind names to `[kind_spec, module]` pairs. - -### 3.2 Step-by-step execution pipeline - -``` -1. create_file_list() - expand .txt indirect file lists, validate .meta extensions - -2. register_ddts(scheme_files) - pre-scan all scheme .meta files - register DDT type names via register_fortran_ddt_name() - (so the host parser can recognize them as non-intrinsic types) - -3. parse_host_model_files() - for each host .meta file: - metadata_table.parse_metadata_file() → [MetadataTable] - find_associated_fortran_file() → matching .F90 path - parse_fortran_file() → Fortran declarations (via fortran_tools) - check_fortran_against_metadata() → cross-validation (type, kind, rank, intent) - accumulate MetadataSection headers: DDT, module, host types - -4. HostModel(table_dict, host_name, run_env) - process DDT headers: → DDTLibrary + ddt_dict (VarDictionary) - process module/host headers: → main VarDictionary + __var_locations - add ConstituentVarDict synthetically for ccpp_model_constituents_t - -5. API(sdfs, host_model, scheme_headers, run_env) - for each SDF XML: - Suite construction: - auto-create 5 phase groups: register, initialize, timestep_initial, - timestep_final, finalize - parse elements → Group objects (RUN_PHASE_NAME) - parse / tags → Scheme objects in full-phase groups - Suite.analyze(host_model, scheme_library, ddt_library, run_env): - Group.analyze() → Scheme.analyze(): - for each scheme argument: - VarDictionary.find_variable() [scope chain search] - Var.compatible() [→ VarCompatObj with transformations] - loop dim substitution for _run phase - register constituent if constituent=True - variable promotion: group outputs → suite level if needed by later group - -6. ccpp_api.write(outdir, run_env) - suite cap .F90 per suite - group caps (embedded or separate) - host cap .F90 (if --host-name given) - ccpp_kinds.F90 - -7. generate_ccpp_datatable() → datatable.xml -``` - -### 3.3 Object hierarchy - -``` -API (ccpp_suite.py) - └── Suite (extends VarDictionary) [one per SDF XML] - parent → ConstituentVarDict (extends VarDictionary) - parent → API - ├── Group (suite_objects.py, extends VarDictionary) [one per ] - │ call_list: CallList (extends VarDictionary) - │ ├── Subcycle (suite_objects.py) - │ │ └── Scheme (suite_objects.py, extends SuiteObject) - │ └── Scheme (for full-phase groups: init, register, etc.) - └── (auto groups: register, initialize, timestep_initial, - timestep_final, finalize) - -HostModel (host_model.py, extends VarDictionary) - ├── ddt_lib: DDTLibrary - │ └── {ddt_name → MetadataSection} - ├── ddt_dict: VarDictionary (all DDT field variables, expanded) - └── loop_vars: VarDictionary (run-time dimension variables) - -VarDictionary (metavar.py) - ├── {standard_name → Var} - └── parent_dict → VarDictionary ← scope chain for find_variable() - -Var (metavar.py) - └── __prop_dict: {property_name → validated_value} - -VarDDT (ddt_library.py, extends Var) - └── __field: Var | VarDDT ← recursive DDT traversal chain -``` - -### 3.4 Variable matching — scope-chain and VarCompatObj - -Unlike prebuild's single batch `compare_metadata()`, capgen performs incremental, -scope-aware matching during the suite analysis phase. - -For each scheme argument in `Scheme.analyze()`: -1. `VarDictionary.find_variable(standard_name)` — searches scope chain: - local group dict → suite dict → ConstituentVarDict → host model dict -2. `Var.compatible(other, run_env)` returns a `VarCompatObj` — not a bool. - `VarCompatObj` carries: - - Whether the variables are equivalent (no transformation needed) - - Whether they are compatible with transformations (unit conversion, dimension - substitution, `top_at_one` flip) - - The reason for any incompatibility (for error messages) -3. For `_run` phase: `horizontal_dimension` is automatically substituted with - `horizontal_loop_begin:horizontal_loop_end` -4. For `constituent = True` variables: auto-registered in `ConstituentVarDict`; - allocation/management code is generated -5. Variable promotion: if a Group produces a variable needed by a later Group, it is - promoted to Suite-level scope - -`VarCompatObj` compatibility considers: -- Type equality -- Kind equality (with ISO kind aliases) -- Units compatibility (triggers unit conversion if compatible) -- Dimension substitutability (horizontal loop vs. full dimension, vertical extent) -- `top_at_one` orientation (triggers flip if needed) -- `protected` status (cannot be an output if protected) -- `CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` - from `var_props.py` - -### 3.5 `metavar.Var` properties - -`metavar.Var` stores all properties in a validated `__prop_dict`. Properties: - -**Specification properties** (all metadata contexts): - -| Property | Type | Notes | -|---|---|---| -| `local_name` | str | Valid Fortran identifier | -| `standard_name` | str | CF-convention, lowercase+underscores | -| `long_name` | str | Human-readable description | -| `units` | str | Physical units string | -| `dimensions` | list | Dimension standard names or `()` | -| `type` | str | Intrinsic or registered DDT name | -| `kind` | str | Fortran kind parameter | -| `active` | str | Conditional allocation expression | -| `optional` | bool | Whether scheme can handle missing var | -| `protected` | bool | Cannot be written by schemes | -| `allocatable` | bool | Has ALLOCATABLE attribute | -| `state_variable` | bool | Persists across timesteps | -| `persistence` | str | `timestep` or `run` | -| `default_value` | str | Fortran expression | -| `diagnostic_name` | str | Diagnostic output name | -| `target` | bool | Has TARGET attribute | -| `polymorphic` | bool | CLASS(*) type | -| `top_at_one` | bool | Vertical ordering: top at index 1 | - -**Scheme-only properties**: - -| Property | Type | Notes | -|---|---|---| -| `intent` | str | in / out / inout | - -**Constituent properties**: - -| Property | Type | Notes | -|---|---|---| -| `constituent` | bool | Is a CCPP-managed constituent (tracer) | -| `advected` | bool | Is advected by the dynamical core | -| `molar_mass` | float | Molecular weight (positive) | - -### 3.6 Capgen-only features - -**Fortran cross-validation** (`check_fortran_against_metadata()`): -- Parses the actual `.F90` file alongside the `.meta` file -- Checks that every metadata entry matches the real Fortran declaration: - variable count, local_name, type, kind, intent (for schemes), dimension rank/names -- Catches bugs where metadata was updated but the Fortran source was not (or vice versa) - -**State machine** (`ccpp_state_machine.py`, `state_machine.py`): -- `CCPP_STATE_MACH`: a `StateMachine` instance with 6 transitions -- Valid state sequence: `register → uninitialized → initialized → in_time_step` -- Suite caps include a `character(len=16) :: ccpp_suite_state` variable -- State-checking code at the start of each phase function enforces correct call ordering -- `CCPP_STATE_MACH.function_match()` uses compiled regex to identify which CCPP phase - a subroutine name belongs to - -**Constituent variable support** (`constituents.py`): -- `ConstituentVarDict` (extends `VarDictionary`) manages traceable species (tracers) -- When a scheme declares `constituent = True`, `find_variable()` auto-creates the variable -- Allocation code for the constituent array is auto-generated -- Constants: `CONST_DDT_NAME = "ccpp_model_constituents_t"`, - `CONST_PROP_TYPE = "ccpp_constituent_properties_t"` - -**DDT library** (`ddt_library.py`): -- `VarDDT(Var)`: represents a DDT field variable at any nesting level -- Traversal chain: `VarDDT → VarDDT → ... → Var` (innermost is the actual leaf field) -- `DDTLibrary`: dictionary of DDT `MetadataSection` objects -- `collect_ddt_fields()` expands DDT variables into component fields in `ddt_dict` - -**Host cap generation** (`host_cap.py`): -- Generated only when `--host-name` is given -- Produces `_ccpp_cap.F90` -- Subroutines: `_ccpp_physics_(api_vars)` - that call into suite cap functions - -**`ccpp_kinds.F90`**: -- Simple Fortran module `ccpp_kinds`. **Always generated**, even when no `--kind-type` - is supplied (in that case `kind_phys=iso_fortran_env:REAL64` is injected - automatically and an INFO log line is emitted). -- One `use , only: ` line per module (modules sorted; specs deduped per - module). Each kind is then re-exported as - `integer, parameter, public :: = `. -- Supports host-supplied kind modules: `--kind-type kind_phys=my_host_kinds:kind_r8` - emits `use my_host_kinds, only: kind_r8` and - `integer, parameter, public :: kind_phys = kind_r8`. -- Listed in `` of `datatable.xml` (matches original capgen) so - the build system picks it up via `ccpp_datafile.py --ccpp-files`. -- USEd by all generated Fortran files that declare kind-typed variables — the group - cap, the suite types module, and the suite data module. The static API and suite - cap have no kind references and do not USE it. - -**Datatable XML** (`ccpp_datafile.py`): -- Produced after generation; lists all generated files, scheme entries, variable properties, - suite configurations -- Queryable by the build system via `ccpp_datafile.py --suite-files` etc. -- Supports `--clean` workflow: reads the file list, removes all generated files, deletes itself -- `DatatableReport` class provides a programmatic query API - -**In-memory database** (`ccpp_database_obj.py`): -- `CCPPDatabaseObj`: wraps `HostModel` and `API` for programmatic access to capgen results -- Returned when `capgen()` is called with `return_db=True` -- Provides `host_model_dict()`, `suite_list()`, `constituent_dictionary(suite)` - -**Variable tracking tool** (`ccpp_track_variables.py`): -- Standalone diagnostic: traces a specific variable through a suite, showing which schemes - use it and with what intent -- Uses prebuild's `import_config` and capgen's `Suite`/`parse_metadata_file` together - -**Fortran-to-metadata tool** (`ccpp_fortran_to_metadata.py`): -- Standalone utility: parses annotated Fortran source files and generates skeleton `.meta` - files — used to bootstrap new scheme metadata - ---- - -## 4. Shared infrastructure - -### 4.1 Module sharing map - -| Module | Used by prebuild | Used by capgen | Notes | -|---|---|---|---| -| `metadata_parser.py` | yes | partial | **Bridge module**: calls capgen's parser, returns mkcap.Var | -| `metadata_table.py` | via bridge | yes (primary) | Native `.meta` format parser | -| `metavar.py` | no | yes | Primary `Var` class, `VarDictionary` | -| `var_props.py` | no | yes | `VariableProperty`, `VarCompatObj`, dimension constants | -| `mkcap.py` | yes | no | `mkcap.Var` class + build-snippet writers | -| `mkstatic.py` | yes | no | Suite/Group/API Fortran generators | -| `mkdoc.py` | yes | no | HTML/LaTeX documentation generators | -| `common.py` | yes | partial | `CCPP_STAGES`, container encoding | -| `framework_env.py` | dummy instance | yes (primary) | `CCPPFrameworkEnv` | -| `file_utils.py` | no | yes | `create_file_list`, `move_modified_files` | -| `code_block.py` | no | yes | Structured Fortran output | -| `ddt_library.py` | no | yes | `DDTLibrary`, `VarDDT` | -| `host_model.py` | no | yes | `HostModel` class | -| `host_cap.py` | no | yes | Host cap generation | -| `ccpp_suite.py` | no | yes | `Suite`, `API` classes | -| `suite_objects.py` | no | yes | `Scheme`, `Group`, `Subcycle`, `CallList` | -| `constituents.py` | no | yes | `ConstituentVarDict` | -| `ccpp_datafile.py` | no | yes | Datatable XML | -| `ccpp_database_obj.py` | no | yes | `CCPPDatabaseObj` | -| `ccpp_state_machine.py` | no | yes | `CCPP_STATE_MACH` | -| `state_machine.py` | no | yes | `StateMachine` base class | -| `ccpp_fortran_to_metadata.py` | no | yes | Fortran→metadata bootstrap tool | -| `ccpp_track_variables.py` | partial | partial | Uses both worlds | - -**The key architectural debt**: `metadata_parser.py` is a prebuild module that internally -calls capgen's `metadata_table.parse_metadata_file()` and converts results to `mkcap.Var` -objects. This creates a one-way dependency (prebuild → capgen's parser infrastructure) -while presenting a prebuild-style API to `ccpp_prebuild.py`. It exists only because -prebuild predates the `.meta` format. - -### 4.2 The `.meta` file format - -The `.meta` format is the native format for capgen and the expected format for all new -scheme development. The Fortran source file contains a comment hook pointing to the `.meta` -file: - -```fortran -!! \section arg_table_scheme_name_run Argument Table -!! \htmlinclude scheme_name_run.html -``` - -The `.meta` file itself uses an INI-style format: - -```ini -[ccpp-table-properties] - name = scheme_name - type = scheme - source_path = ../src - dependencies_path = ../some/path - dependencies = utility_module.F90, another.F90 - -[ccpp-arg-table] - name = scheme_name_run - type = scheme -[ im ] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - type = integer - dimensions = () - intent = in -[ dz ] - standard_name = layer_thickness - long_name = thickness of each model layer - units = m - type = real - kind = kind_phys - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - intent = in -``` - -Multiple `[ccpp-arg-table]` sections are allowed in a scheme file (one per phase: -`_init`, `_run`, `_finalize`, `_timestep_init`, `_timestep_finalize`). -Singleton tables (DDT, module, host) allow only one section. - -The three table-level properties in `[ccpp-table-properties]` that carry path information: - -| Property | Purpose | Resolution | -|---|---|---| -| `source_path` | Relative path from the `.meta` file's directory to the directory containing the corresponding `.F90` Fortran source file | `os.path.normpath(os.path.join(meta_dir, source_path))`. Defaults to `meta_dir` when absent. | -| `dependencies_path` | Optional subdirectory relative to `meta_dir`; used as the base directory for resolving entries in `dependencies` | `os.path.normpath(os.path.join(meta_dir, dependencies_path))`. Defaults to `meta_dir` when absent. | -| `dependencies` | Comma-separated list of dependency file names or relative paths | Each entry resolved via `os.path.normpath(os.path.join(dep_base, entry))` where `dep_base` is the resolved `dependencies_path`. The value `none` is ignored. | - -**Implementation note — `flush_table_props` pattern:** The INI parser processes the -`[ccpp-table-properties]` and `[ccpp-arg-table]` headers in one streaming pass. Extra -table-level keys (`source_path`, `dependencies_path`, `dependencies`) are collected in a -`pending_props` dict alongside `name` and `type`. The parser must apply these properties -to the `MetadataTable` object — via a `flush_table_props()` call — at every -state-transition point (first `[ccpp-arg-table]` header, next `[ccpp-table-properties]` -header, and end-of-file) **before** resetting `pending_props`. Without this, the extra -properties are silently discarded. - -### 4.3 Variable property validation (`var_props.py`) - -`VariableProperty` encapsulates a single metadata property with its name, Python type, -optionality, default, valid-value constraints, and a check function. Check functions used: - -| Checker | What it validates | -|---|---| -| `check_local_name` | Valid Fortran identifier | -| `check_cf_standard_name` | Lowercase, underscores, alphanumeric only | -| `check_fortran_type` | Intrinsic type or registered DDT name | -| `check_units` | Valid unit string (normalizes `+` in exponents) | -| `check_dimensions` | Valid dimension specification | -| `check_default_value` | Valid Fortran expression | -| `check_molar_mass` | Positive float (for constituents) | - -`CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` -in `var_props.py` define the recognized dimension forms and the run-time substitution -map (e.g., `horizontal_dimension → horizontal_loop_begin:horizontal_loop_end`). - ---- - -## 5. Feature comparison - -| Feature | prebuild | capgen | Notes | -|---|---|---|---| -| **Input formats** | | | | -| Native `.meta` format | via bridge | yes | | -| Old pipe-delimited format | deprecated warn | not supported | | -| **Parsing and validation** | | | | -| Fortran source cross-validation | no | yes | capgen parses actual .F90 to cross-check | -| Preprocessor directive support | no | yes | `--preproc-directives` | -| **Variable handling** | | | | -| Variable data class | `mkcap.Var` (flat attrs) | `metavar.Var` (validated prop dict) | | -| Scope-chain variable search | no | yes | group→suite→constituent→host | -| Variable promotion group→suite | no | yes | | -| Unit conversion | yes | yes | | -| Optional/active variables | yes (fully) | yes | both: local pointer + conditional ASSOCIATE | -| DDT library (first-class) | no | yes | `VarDDT` recursive chain | -| **Suite and cap generation** | | | | -| Suite definition (SDF XML) | yes | yes | Same XML format | -| Subcycle loops | yes | yes | | -| State machine in generated caps | no | yes | Runtime state enforcement | -| Static API module (dispatch switch) | yes | no | `ccpp_static_api.F90` | -| Host cap generation | no | yes | `_ccpp_cap.F90` | -| `ccpp_kinds.F90` | no | yes | | -| **Constituent/tracer support** | | | | -| Constituent variable management | no | yes | Auto-allocation, `ConstituentVarDict` | -| **Build system output** | | | | -| CMake/Makefile file-list snippets | yes | no | Six snippet files | -| Datatable XML (queryable) | no | yes | `ccpp_datafile.py` | -| Clean via datatable | no | yes | | -| **Documentation** | | | | -| HTML variable table | yes | no (stub, raises error) | `mkdoc.metadata_to_html` | -| LaTeX variable table | yes | no | `mkdoc.metadata_to_latex` | -| **Developer tools** | | | | -| Variable tracking diagnostic | yes | no | `ccpp_track_variables.py` | -| Fortran-to-metadata bootstrap | no | yes | `ccpp_fortran_to_metadata.py` | -| **Runtime API** | | | | -| In-memory database object | no | yes | `CCPPDatabaseObj` | -| **Debug / developer aids** | | | | -| Debug array-size checks in caps | yes (`--debug`) | no | | -| Namespace suffix for API name | yes (`--namespace`) | no | | -| **Configuration** | | | | -| Config mechanism | Python module (flexible) | CLI args only | | - -**Known gaps and corrections:** - -- Capgen's `--generate-docfiles` is declared in the CLI but raises - `CCPPError("not yet supported")` — documentation generation is unimplemented. -- Prebuild handles `TYPEDEFS_NEW_METADATA` for mixed old/new metadata deployments; - capgen has no equivalent because it only accepts the new format. -- Capgen validates Fortran source against metadata; prebuild trusts metadata and never - reads Fortran code. -- Capgen has no `--namespace` equivalent for the generated API module name. -- Capgen's `CCPPDatabaseObj` and datatable XML allow programmatic querying; prebuild - has no equivalent. -- Prebuild's static API pattern (single Fortran module with runtime dispatch) is absent - from capgen, which uses a different host-cap integration model. -- **Capgen cannot pass DDTs to group caps** — it passes everything as flat fields. - Despite considerable effort by multiple developers, this has not been fixed. This is - the primary reason capgen is being abandoned. -- **Capgen does not support multiple model instances in memory** (ensemble approach). - Prebuild's `initialized(200)` array handles this correctly. -- **Capgen does not own or allocate any data.** Wait — this is a prebuild characteristic. - Capgen *does* allocate data for physics-internal variables (variables used only within - the physics, not provided by the host model) at the suite level. Prebuild requires the - host model to provide and own all data, including any physics-internal scratch space. - ---- - -## 6. Build system integration - -### 6.1 How a host model invokes ccpp-prebuild - -Direct call (as in the test suite): -```bash -python ../../scripts/ccpp_prebuild.py \ - --config=ccpp_prebuild_config.py \ - --builddir=build \ - --suites=suite_A,suite_B \ - [--debug] [--namespace mymodel] -``` - -Typical CMake integration: -```cmake -# Run prebuild at configure time -execute_process( - COMMAND ${Python3_EXECUTABLE} - ${CCPP_FRAMEWORK}/scripts/ccpp_prebuild.py - --config=${HOST_CCPP_PREBUILD_CONFIG} - --builddir=${CMAKE_CURRENT_BINARY_DIR} - --suites=${CCPP_SUITES} - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - RESULT_VARIABLE PREBUILD_RESULT -) -if(NOT PREBUILD_RESULT EQUAL 0) - message(FATAL_ERROR "ccpp_prebuild.py failed") -endif() - -# Consume the generated snippet files -include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) # → ${CAPS} -include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) # → ${SCHEMES} -include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} -include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) # → ${API} - -add_library(ccpp_physics OBJECT ${CAPS} ${SCHEMES} ${API}) -``` - -### 6.2 How a host model invokes ccpp-capgen - -Direct call: -```bash -python scripts/ccpp_capgen.py \ - --host-files host_data.meta,host_model.meta \ - --scheme-files scheme1.meta,scheme2.meta \ - --suites suite_A.xml,suite_B.xml \ - --output-root ${BUILD_DIR}/ccpp \ - --host-name my_host \ - --kind-type kind_phys=REAL64 \ - --ccpp-datafile ${BUILD_DIR}/ccpp/datatable.xml -``` - -Typical CMake integration: -```cmake -# Run capgen at configure time -execute_process( - COMMAND ${Python3_EXECUTABLE} - ${CCPP_FRAMEWORK}/scripts/ccpp_capgen.py - --host-files ${HOST_META_FILES} - --scheme-files ${SCHEME_META_FILES} - --suites ${SUITE_SDFS} - --output-root ${CMAKE_CURRENT_BINARY_DIR}/ccpp - --host-name ${HOST_MODEL_NAME} - --ccpp-datafile ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml - RESULT_VARIABLE CAPGEN_RESULT -) - -# Query the datatable for generated file lists -execute_process( - COMMAND ${Python3_EXECUTABLE} - ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py - ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml - --suite-files - OUTPUT_VARIABLE SUITE_CAPS OUTPUT_STRIP_TRAILING_WHITESPACE -) -execute_process( - COMMAND ${Python3_EXECUTABLE} - ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py - ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml - --host-files - OUTPUT_VARIABLE HOST_CAP OUTPUT_STRIP_TRAILING_WHITESPACE -) - -add_library(ccpp_physics OBJECT ${SUITE_CAPS} ${HOST_CAP}) -``` - -### 6.3 Available datatable query flags - -``` ---host-files → generated host cap .F90 files ---suite-files → generated suite cap .F90 files ---utility-files → generated utility .F90 files (e.g. ccpp_kinds.F90) ---ccpp-files → all generated .F90 files ---process-list → physics process types in the suite ---module-list → Fortran module names needed ---dependencies → scheme dependency files ---suite-list → configured suite names ---required-variables → variables required by all suites ---input-variables → input-only variables for a suite ---output-variables → output variables for a suite ---host-variables → variables provided by the host model -``` - ---- - -## 7. Key architectural differences - -### 7.1 Data model - -| Dimension | ccpp-prebuild | ccpp-capgen | -|---|---|---| -| Variable representation | `mkcap.Var` with plain Python attributes | `metavar.Var` with validated `__prop_dict` | -| Variable storage | Two flat `OrderedDict`s | Scope-chain `VarDictionary` tree | -| Container encoding | Encoded string: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | Explicit class hierarchy | -| DDT handling | Encoded as string in `local_name`; helper regexes to extract | First-class `VarDDT` recursive chain | -| Variable matching | One batch `compare_metadata()` call | Incremental during suite analysis | -| Matching result | `bool` + side effects on `.target` / `.actions` | Rich `VarCompatObj` with transformation info | -| **Cap argument style** | **DDTs passed to group caps** | **Flat fields passed to group caps** | -| Subsetting location | At the scheme call site inside the group cap | Done at a higher level, before group cap | -| Data ownership | Host model owns all data including physics-internal | Capgen allocates physics-internal suite-level data | -| Multiple model instances | Yes — `initialized(200)` array, one flag per instance | No | -| Optional variable handling | Local pointer, conditionally associated | Same mechanism, but blocked by flat-field issue | - -### 7.2 Error handling - -| Aspect | ccpp-prebuild | ccpp-capgen | -|---|---|---| -| Style | `(success, result)` tuples + `logging.error()` | `CCPPError` / `ParseInternalError` exceptions | -| Collection | Errors accumulate via `logging`; `main()` checks success | Raised immediately at point of detection | -| Location info | Filename from context; line numbers sometimes | `ParseContext` objects with file + line number | -| User errors vs bugs | Not distinguished | `CCPPError` (user) vs `ParseInternalError` (programmer) | - -### 7.3 Extensibility - -| Aspect | ccpp-prebuild | ccpp-capgen | -|---|---|---| -| New metadata property | Add to `VALID_ITEMS` dict + `mkcap.Var` attribute | Add one `VariableProperty` entry + checker fn | -| New CCPP phase | Update `CCPP_STAGES` + regenerate static API template | Add one transition tuple to `CCPP_STATE_MACH` | -| New compatibility rule | Modify `var.compatible()` in `mkcap.py` | Extend `VarCompatObj` in `var_props.py` | -| New host model | Write a new Python config file | New `.meta` files + CLI invocation | - -### 7.4 Performance - -Prebuild generates caps for multiple suites in seconds. Capgen, on the same suite set -with the same physics, takes more than 10 minutes. Two independent causes: - -**Cause 1 — Repeated scope-chain traversal.** Every variable lookup in capgen traverses -a five-level `VarDictionary` parent chain (group → suite → constituent dict → host model -→ DDT dict) for every scheme argument in every group in every suite. Prebuild's -`compare_metadata()` does one flat dict lookup per standard name, once, and caches the -result in `var.container` and `var.target`. All subsequent use during Fortran generation -reads these cached attributes directly. - -**Cause 2 — Flat-field cap arguments.** This is likely the dominant cost. Capgen resolves -every scheme argument down to its individual flat field, generates a `use` statement and -an explicit argument for each one, and emits them in the generated Fortran. A DDT with -200 fields becomes 200 individual argument declarations, 200 `use` statements, and 200 -argument positions in the scheme call. Prebuild passes the DDT itself — one argument, -one `use` statement — and then subsets at the call site. - -**Consequence for correctness.** Passing flat fields in capgen also breaks optional -variable handling under Fortran compiler debugging flags. When a field inside a DDT is -conditionally allocated (optional), passing it as a flat field requires dereferencing the -DDT to extract the field — which the compiler will flag as an error if debugging is on -and the field happens to be unallocated. Prebuild avoids this entirely by passing the -DDT and using a local pointer at the scheme call site. - -### 7.5 Team comprehension and maintainability - -This is the critical real-world difference. `ccpp-prebuild` is understood by the whole -team because it is procedural Python: you can read `ccpp_prebuild.py` top-to-bottom and -follow what happens. The data structures are flat dicts; the control flow is linear. - -`ccpp-capgen` has a five-level class hierarchy, scope-chain dictionary lookups, -`VarCompatObj` carrying transformation state, `ConstituentVarDict` as a pluggable -scope-chain node, and a `StateMachine` with regex-based dispatch. No remaining team -member fully understands all of it. Development is extremely slow and risky. - -The failed effort to make capgen pass DDTs instead of flat fields is the concrete proof -point: three developers spent considerable time and could not fix it without fully -understanding the interplay between `VarDDT`, `DDTLibrary`, `VarDictionary` scope chains, -and the Fortran writer. This is the proximate reason for the redesign. - ---- - -## 8. Design considerations for the redesign - -The following observations from this analysis should inform the redesign: - -### 8.1 What to keep from prebuild -- Procedural, top-down control flow — easy to read and debug -- Config file as a Python module — extremely flexible without adding CLI arguments -- The static API pattern (`ccpp_static_api.F90` with runtime suite/group dispatch) — - proven, simple integration for the host model -- **DDT arguments in group caps** — pass DDTs, not flat fields; this is the core correctness - and performance requirement -- **Subsetting at the scheme call site** — group caps always receive full data; loop-bound - application and fixed-index extraction happen in the individual scheme call expressions - or via a local variable/pointer declared just before the call -- **Optional variable pattern** — local pointer declared in the group cap, conditionally - associated based on the `active` expression, then passed to the scheme; this is safe - under all compiler debugging modes -- The `initialized(N)` per-instance tracking — handles multiple simultaneous model - instances in memory (ensemble approach); `N` is the max number of instances -- **Framework-owned data needs a simpler design** — capgen's variable promotion and - `ConstituentVarDict` scope-chain approach is too complex; a cleaner mechanism for - framework-allocated physics-internal data is needed (to be designed) -- HTML and LaTeX documentation generation -- The six CMake/Makefile/shell snippet output files — simple and direct (can be revisited) - -### 8.2 What to keep from capgen -- Native `.meta` file parsing (eliminate the `metadata_parser.py` bridge entirely) -- Fortran source cross-validation (`check_fortran_against_metadata()`) — catches real bugs -- Rich compatibility reporting (`VarCompatObj`-style) — better error messages -- `ccpp_kinds.F90` generation — important for portability -- Datatable XML as output accounting (strictly better than six include files) -- `--preproc-directives` support -- Constituent variable support (needed for CAM-SIMA) -- State machine enforcement (optional feature, but architecturally clean) - -### 8.3 What to eliminate -- The `mkcap.Var` / `metavar.Var` duality — one variable class, natively reading `.meta` -- The `metadata_parser.py` bridge module — it exists only because of the old format -- The scope-chain `VarDictionary` hierarchy — replace with flat, explicit lookup: - one host dict, one scheme dict; no parent-chain traversal -- The five-level class inheritance (Suite → VarDictionary → ParseSource → ...) -- `ConstituentVarDict` as a scope-chain node — a simple explicit constituent registry suffices -- Capgen's variable promotion (group → suite level) — this complexity exists only because - capgen allocates physics-internal data; if the host always owns all data, promotion - is unnecessary -- Capgen's flat-field cap generation — DDT arguments must be the foundation - -### 8.4 Framework-owned data — open design question - -Capgen's variable promotion mechanism (promoting a variable from group scope to suite scope -when a later group needs it) and the `ConstituentVarDict` complexity exist because capgen -allocates and manages physics-internal data — variables used only within the physics, -not visible to the host model. This capability is **wanted** in the redesign: the host -model should not have to declare and own scratch variables that are purely internal to the -physics. - -The problem is not the concept but the implementation. Capgen's approach — weaving -framework-allocated variables into the `VarDictionary` scope chain and promoting them -upward — produces the complexity that made capgen unmaintainable. - -**Open question for the redesign:** What is a simpler mechanism for the framework to -allocate, own, and pass physics-internal variables? Candidate approaches (to be evaluated -with real-world examples): - -- A completely separate, flat "framework data" dictionary, distinct from the host variable - lookup, populated during analysis and passed explicitly to the caps as a dedicated - argument (e.g., a framework-managed DDT or allocatable array container). -- A simplified promotion concept: variables are statically promoted to the widest scope - that needs them during the analysis phase, but stored in a simple flat dict rather than - via a scope-chain lookup. -- Constituent variables (tracers) as a special sub-case with their own well-defined - allocation interface, separate from generic physics-internal data. - -This question will be revisited once real-world examples clarify how many and what kind of -physics-internal variables actually need to be managed. - -### 8.5 Critical design decisions for the redesign prompt - -1. **DDT cap arguments are non-negotiable.** Group caps must receive DDTs. The entire - subsetting, optional-variable, and performance story depends on this. - -2. **Data ownership**: host-owns-all (prebuild model) vs. generator-allocates-internals - (capgen model). This single decision determines whether variable promotion and - suite-level allocation are needed. - -3. **Integration pattern**: static API (prebuild style, `suite_name` + `group_name` dispatch) - vs. host cap (capgen style, separate host-side Fortran glue). Models currently using - each pattern depend on it. - -4. **Config mechanism**: Python module (prebuild style, flexible) vs. pure CLI + file lists - (capgen style, scriptable). The Python module config is very powerful for complex models. - -5. **DDT member access parsing**: `extract_parents_and_indices_from_local_name()` and - `extract_dimensions_from_local_name()` in `mkstatic.py` handle expressions like - `Atm(blk_no)%q(:,:,:,graupel_index)`. The redesign needs a clean, explicit design for - parsing and emitting these — not an afterthought regex patch. - -6. **Output accounting**: datatable XML (capgen) is the right answer. The six CMake snippet - files (prebuild) are redundant and harder to extend. - -7. **Multiple model instances**: the redesign must preserve the `initialized(N)` pattern - or an equivalent. The value of `N` may need to be configurable. - -8. **Backward compatibility of generated Fortran interfaces**: real-world model examples - will define exactly which naming conventions, argument orders, and module structures the - host models depend on. - -### 8.6 Implementation decisions made during redesign - -The following decisions were made during implementation of `capgen-ng` and are recorded -here as amendments to the analysis above. - -**State machine parameters are local to each generated group cap module.** -The original redesign prompt described the integer state constants as coming from a -shared framework library module. In practice they are generated as `private` named -parameters directly inside each group cap module: - -```fortran -integer, parameter, private :: CCPP_GROUP_UNINITIALIZED = 0 -integer, parameter, private :: CCPP_GROUP_INITIALIZED = 1 -integer, parameter, private :: CCPP_GROUP_IN_TIMESTEP = 2 -``` - -This keeps generated files self-contained — no implicit dependency on a framework -runtime library at the caps level. The values are replicated across all generated group -cap files, but the names are the contract. - -**`source_path` is used by the validator, not the generator.** -The generator trusts metadata and never opens Fortran source files. `source_path` is -meaningful only to the standalone validator tool, which uses it to auto-discover the -`.F90` file paired with each `.meta` file (same base name, different directory). - -**`dependencies` paths are written to `datatable.xml`.** -The resolved absolute paths from each scheme's `dependencies` table-level property are -collected and written to the `` section of `datatable.xml`, sorted and -deduplicated. The CMake build system reads these via `ccpp_datafile.py` to add external -dependency files to the build graph. - -**Optional variable (pointer wrapper) implementation decisions.** -Optional arguments (Case 2 and Case 4) use per-suite Fortran derived types for pointer -wrappers. All unique `(type, kind, rank)` combinations needed by any optional arg across -all groups in a suite are collected and written to `ccpp__types.F90`. Each type -name is generated as `{type}_{kind}_rank{N}_ptr_type` (e.g. `real_kind_phys_rank1_ptr_type`). -Group cap modules `USE` this file. The types file is omitted entirely when no optional -args exist in the suite. The active condition for a pointer assignment is inherited from -the **host variable's** `active` attribute when the scheme itself specifies no `active`. - -**Character length (`len=N` / `len=*`) rules.** -Character kind declarations follow specific compatibility rules enforced by the resolver: - -- `len=*` in a **scheme** is always compatible with any host `len=` — assumed-length - dummy arguments accept any host-declared length. No transform is generated. -- Matching specific `len=N` in both host and scheme requires no transform (naturally equal). -- Mismatched specific lengths (`len=512` host vs `len=128` scheme) are a **metadata error**; - the scheme must declare `len=*` or match the defining metadata exactly. -- `len=*` in the **host** with a specific `len=N` in the scheme is also an error. - -The resolver raises `CCPPError` for the illegal cases. No kind transform is ever generated -for character variables — lengths are a Fortran compatibility constraint, not a unit conversion. - -**`source_path` is used by the validator, not the generator.** -The group cap's `state_alloc` subroutine always takes `number_of_instances` as an -explicit `intent(in)` integer argument — it never USEs any host module to obtain it. -The call chain is: `ccpp_init` → `_init` → each group's `state_alloc`. At each -level the argument is conditional: - -- **Multi-instance host** (`number_of_instances` declared in host metadata with local - name e.g. `ninstances`): - - `ccpp_init(suite_name, ninstances, errmsg, errflg)` — static API receives it - - `_init(ninstances, errmsg, errflg)` — suite cap threads it through - - `state_alloc(ninstances, errmsg, errflg)` — group cap allocates array of that size -- **Single-instance host** (no `number_of_instances` in host metadata): - - All three signatures omit the argument - - `state_alloc(1, errmsg, errflg)` — the literal `1` is passed - -State array **indexing** uses `instance_number`'s local name (e.g. `inst_num`) from -the control metadata. For single-instance hosts the literal `1` is used. `instance_number` -is injected into the group cap's `_init` and `_final` subroutine signatures even when no -scheme in those phases uses it — the state guard and state transition require it: - -```fortran -subroutine ccpp___init(inst_num, ...) - if (ccpp_group_state(inst_num) >= CCPP_GROUP_INITIALIZED) return - ! ... scheme _init calls ... - ccpp_group_state(inst_num) = CCPP_GROUP_INITIALIZED -``` - -This injection does **not** happen for `_run`, `_timestep_init`, or `_timestep_final` -unless a scheme in those phases explicitly requests `instance_number`. The suite cap's -`_physics_init` and `_physics_final` dispatch subroutines similarly pass -`instance_number` to the group cap calls when the host provides it. - -**Control variable validation — flat unconditional required set.** -All required control variables (`suite_name`, `horizontal_loop_begin`, `horizontal_loop_end`, -`thread_number`, `number_of_threads`, `number_of_physics_threads`, `ccpp_error_code`, -`ccpp_error_message`, `instance_number`) are unconditional — every host must declare all -of them. Single-threaded or single-instance models pass `1` or `''` for any they don't -actively use. The generator validates the complete set after parsing host metadata, -collects all missing-variable errors together, and halts before emitting any code. -`instance_number` in particular is NOT conditional on `instance_dimension` usage — -it is always required. - -**`group_name` is conditionally included, not in the required set.** -`group_name` is included in the static API signature only if the host declares it in -their `type=control` table. When absent, the cap calls all groups unconditionally. The -generator warns (not errors) if `group_name` is absent and any suite has multiple groups. -When present: a required (non-optional) character argument; `''` or `'all'` calls all -groups in order; any other value dispatches to the named group only. - -**`horizontal_loop_extent` eliminated; schemes always use `horizontal_dimension`.** -Scheme metadata always declares `horizontal_dimension` as the horizontal extent -dimension, regardless of phase. There is no `horizontal_loop_extent` standard name in -the new design. The distinction between run-phase chunked processing and full-domain -init/final processing is handled entirely at the host level — the host passes actual -chunk bounds to `ccpp_physics_run` and `1`/`ncols` to all other phases. The cap always -generates `(horizontal_loop_begin:horizontal_loop_end)` for scheme call-site array -slices. For suite-owned array allocation sizing, `horizontal_dimension` from the host -`type=host` table (module USE) is used directly. This separation means allocation -correctness does not depend on the host passing any particular control variable values. - -**Uniform signature across all `ccpp_physics_*` entry points.** -All five physics entry points (`ccpp_physics_init`, `ccpp_physics_timestep_init`, -`ccpp_physics_run`, `ccpp_physics_timestep_final`, `ccpp_physics_final`) share the -same control argument set. No per-phase signature variations. `horizontal_loop_begin` -and `horizontal_loop_end` are in scope for all phases — a `_init` scheme that declares -`horizontal_dimension` correctly receives `(lb:ub)` slicing just as a `_run` scheme -would, with the host responsible for passing the right values. - ---- - -## 9. Real-world example: CCPP Single Column Model (SCM) - -*Source:* `EXT/ccpp-scm/` — uses `ccpp-prebuild`. - -The SCM is a horizontally degenerate model (always `im = 1`, no OpenMP threading) but -it compiles the largest set of suites in the CCPP ecosystem, making it the most complete -real-world picture of what prebuild must handle. - -**Scale:** 63 suites, 257 scheme files (137 scheme entries in config, many containing -multiple modules), 300 generated cap files, ~1,200+ host model variables, ~550 optional -(conditionally active) variables. - ---- - -### 9.1 The `TYPEDEFS_NEW_METADATA` bridge — the DDT accessor map - -This is the most important SCM-specific configuration. It maps each DDT type name to the -Fortran expression used to access an instance of that type from the host model's top-level -scope. It is what allows the code generator to convert a `local_name` like `tgrs` (declared -inside `GFS_statein_type`) into the cap argument expression -`physics%Statein%tgrs(...)`. - -```python -TYPEDEFS_NEW_METADATA = { - 'GFS_typedefs': { - 'GFS_diag_type' : 'physics%Diag', - 'GFS_control_type' : 'physics%Model', - 'GFS_cldprop_type' : 'physics%Cldprop', - 'GFS_tbd_type' : 'physics%Tbd', - 'GFS_sfcprop_type' : 'physics%Sfcprop', - 'GFS_coupling_type': 'physics%Coupling', - 'GFS_statein_type' : 'physics%Statein', - 'GFS_radtend_type' : 'physics%Radtend', - 'GFS_grid_type' : 'physics%Grid', - 'GFS_stateout_type': 'physics%Stateout', - 'GFS_typedefs' : '', - }, - 'CCPP_typedefs': { - 'GFS_interstitial_type': 'physics%Interstitial(cdata%thrd_no)', - 'CCPP_typedefs' : '', - }, - 'scm_type_defs': { - 'physics_type': 'physics', - 'scm_type_defs': '', - }, - 'ccpp_types': { - 'ccpp_t' : 'cdata', - 'ccpp_types': '', - 'MPI_Comm': '', - }, - # ... plus 8 more entries for physics-side modules (machine, radsw_param, etc.) -} -``` - -**How it works:** For a variable with `local_name = tgrs` declared in `GFS_statein_type`, -the generator looks up `'GFS_statein_type'` in the map, finds `'physics%Statein'`, and -constructs the target as `physics%Statein%tgrs`. For the thread-indexed interstitial DDT, -`physics%Interstitial(cdata%thrd_no)%` is produced automatically. - -This dictionary is the **entire** mechanism by which the prebuild bridge converts flat -metadata into correct DDT-member accessor expressions. It is a hand-maintained workaround -that the redesigned generator must **eliminate**: all information needed to derive these -accessor expressions is already present in the CCPP metadata, provided the metadata storage -model is designed correctly to capture the DDT hierarchy and instance/thread indexing. - ---- - -### 9.2 Host model DDT structure - -``` -! Module-level variables accessible globally: -physics (type physics_type, from module scm_type_defs) -cdata (type ccpp_t, from module ccpp_types) -one (integer parameter = 1, from module ccpp_types) - -! physics_type contains: -physics%Model → GFS_control_type (control parameters: integers, logicals, 1D arrays) -physics%Statein → GFS_statein_type (input atmospheric state: 2D/3D real arrays) -physics%Stateout → GFS_stateout_type (output tendencies) -physics%Sfcprop → GFS_sfcprop_type (surface properties: 2D real arrays) -physics%Coupling → GFS_coupling_type (coupling fields) -physics%Grid → GFS_grid_type (grid geometry) -physics%Tbd → GFS_tbd_type (to-be-determined / miscellaneous) -physics%Cldprop → GFS_cldprop_type (cloud microphysics properties) -physics%Radtend → GFS_radtend_type (radiation tendencies) -physics%Diag → GFS_diag_type (diagnostic output arrays) -physics%Interstitial(1:thrd_cnt) → GFS_interstitial_type (per-thread scratch space) -``` - -The interstitial DDT is an array indexed by thread number. Even though the SCM is -single-threaded, all caps use `physics%Interstitial(cdata%thrd_no)` (i.e., index 1). -This is the pattern that enables OpenMP parallelism in the full UFS models. - ---- - -### 9.3 The horizontal dimension in the SCM - -The SCM uses a **chunked** horizontal loop even though `im = 1`. The chunk mechanism is: - -```fortran -chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) -chunk_end = physics%Model%chunk_end(cdata%chunk_no) -``` - -All 2D and 3D array slice expressions in caps use this pattern: -```fortran -physics%Statein%tgrs(chunk_begin:chunk_end, one:levs) -``` - -In the SCM, `chunk_begin = chunk_end = 1` always, but the pattern is general enough for -multi-column models. The `one` lower bound (a named integer constant = 1) is a framework -convention used consistently throughout all caps. - ---- - -### 9.4 Four categories of local variables in group caps - -Every group cap generates four categories of local variable declarations before its scheme -calls: - -**Category 1 — Loop bounds and scalars (always present):** -```fortran -integer :: chunk_begin, chunk_end -integer :: levs -chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) -chunk_end = physics%Model%chunk_end(cdata%chunk_no) -levs = physics%Model%levs -``` - -**Category 2 — Fixed-index extractions (tracer indices, surface-level slices):** - -For a tracer `qgrs(:,:,ntqv)`: -```fortran -! No local variable declared — the expression is used inline at the call site: -call scheme_run(qv = physics%Statein%qgrs(chunk_begin:chunk_end, one:levs, physics%Model%ntqv), ...) -``` - -For a surface-level slice `prsi(:,1)`: -```fortran -call scheme_run(prsi_sfc = physics%Statein%prsi(chunk_begin:chunk_end, 1), ...) -``` - -The fixed index may be a literal integer (`1`) or a runtime scalar variable from a DDT -field (`physics%Model%ntqv`). Both are inlined at the call site. - -**Category 3 — Optional variable pointer arrays:** - -One pointer-array type and one pointer-array variable are declared for each optional -variable. They are dimensioned by thread count: -```fortran -type :: real_kind_phys_rank2_ptr_arr_type - real(kind_phys), dimension(:,:), pointer :: p => null() -end type real_kind_phys_rank2_ptr_arr_type -type(real_kind_phys_rank2_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array -``` - -Before each scheme call that uses the variable, the condition is evaluated and the pointer -either associated or left null: -```fortran -if (physics%Model%lndp_type /= 0) then - sfc_wts_1_ptr_array(cdata%thrd_no)%p => & - physics%Coupling%sfc_wts(chunk_begin:chunk_end, one:physics%Model%n_var_lndp) -end if -``` - -Passed to the scheme as a keyword argument: -```fortran -call gfs_surface_generic_pre_run(..., sfc_wts=sfc_wts_1_ptr_array(cdata%thrd_no)%p, ...) -``` - -After the call, the pointer is nullified: -```fortran -if (physics%Model%lndp_type /= 0) then - nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) -end if -``` - -**Category 4 — Unit conversion local variables:** - -Not present in the SCM (GFS uses consistent SI units throughout). When present in other -models, a local array is declared, populated before the call, and passed as the argument: -```fortran -real(kind_phys) :: converted_var(chunk_begin:chunk_end) -converted_var(:) = physics%Statein%source_field(chunk_begin:chunk_end) * conversion_factor -call scheme_run(..., target_arg=converted_var, ...) -``` - ---- - -### 9.5 Array size checks - -Every array argument — mandatory or optional — has a size check immediately before the -scheme call. The check uses `size()` and computes the expected size from dimension variables: - -```fortran -! Mandatory variable — outer condition is always .true. -if (.true.) then - if (size(physics%Statein%tgrs(chunk_begin:chunk_end, one:levs)) /= & - (chunk_end-chunk_begin+1)*(levs-one+1)) then - write(cdata%errmsg, '(a,i8,a,i8)') & - 'Detected size mismatch for variable tgrs: expected ', expected, ' but got ', actual - ierr = 1 - return - end if -end if - -! Optional variable — outer condition mirrors the active= expression -if (physics%Model%lndp_type /= 0) then - if (associated(sfc_wts_1_ptr_array(cdata%thrd_no)%p)) then - if (size(sfc_wts_1_ptr_array(cdata%thrd_no)%p) /= expected_size) then - ...error... - end if - end if -end if -``` - ---- - -### 9.6 The `initialized(200)` array and instance management - -```fortran -logical, dimension(200), save :: initialized = .false. -``` - -`cdata%ccpp_instance` is a 1-based integer assigned to each independent CCPP state object. -In an ensemble, each ensemble member gets a different instance number (1–200). The `init_cap` -sets `initialized(cdata%ccpp_instance) = .true.` at the end of successful init. The -`run_cap` checks `if (.not. initialized(cdata%ccpp_instance))` and aborts with an error -if init was never called for that instance. The `final_cap` resets the flag to `.false.`. - -The value 200 is hardcoded — it is the maximum supported number of simultaneous model -instances. This could be made configurable. - ---- - -### 9.7 Suite and group cap hierarchy - -Three-level cap hierarchy: - -``` -ccpp_static_api.F90 (module ccpp_static_api) - → dispatches by suite_name + optional group_name - → owns physics, cdata, constants via module use - → calls suite-level caps: - -ccpp_scm_gfs_v16_cap.F90 (module ccpp_scm_gfs_v16_cap) - → aggregates arguments from all groups - → calls group caps in order per phase: - -ccpp_scm_gfs_v16_time_vary_cap.F90 (module ccpp_scm_gfs_v16_time_vary_cap) -ccpp_scm_gfs_v16_radiation_cap.F90 (module ccpp_scm_gfs_v16_radiation_cap) -ccpp_scm_gfs_v16_phys_ps_cap.F90 (module ccpp_scm_gfs_v16_phys_ps_cap) -ccpp_scm_gfs_v16_phys_ts_cap.F90 (module ccpp_scm_gfs_v16_phys_ts_cap) -``` - -Each level is a pure Fortran module. Argument passing is explicit keyword-argument style -at every level; no implicit global data (except in the static API, which uses `use`). - ---- - -### 9.8 Static API: module-level variable ownership - -The static API module uses all host-model modules and accesses their variables at module -scope. It does **not** take host data as subroutine arguments — instead it fills the -group cap arguments from its own module-use-associated variables: - -```fortran -module ccpp_static_api - use scm_type_defs, only: physics - use ccpp_types, only: cdata, one - use scm_physical_constants, only: con_g, con_pi, con_t0c, ... - use gfs_typedefs, only: ltp - use ccpp_scm_gfs_v16_cap, only: scm_gfs_v16_run_cap, ... - ... -contains - subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) - ! cdata passed in, others accessed from module scope - select case (to_lower(trim(suite_name))) - case ('scm_gfs_v16') - if (present(group_name)) then - select case (to_lower(trim(group_name))) - case ('phys_ps') - ierr = scm_gfs_v16_phys_ps_run_cap(one=one, physics=physics, cdata=cdata, ...) - ... - end select - else - ierr = scm_gfs_v16_run_cap(one=one, physics=physics, cdata=cdata, ...) - end if - case ('scm_gfs_v17_p8') - ... - end select - end subroutine -end module -``` - -This design means the static API file must be recompiled whenever any host-model module -changes (because it `use`s them), and it must be regenerated whenever suites change. -Its location in the **source tree** (not build tree) is a deliberate SCM design choice: -the file is committed to the repository as a generated artifact. - ---- - -### 9.9 Build system - -Prebuild runs at **cmake configure time** via `execute_process()`, before any compilation -starts. This is unusual but simplifies the cmake dependency graph. - -```cmake -execute_process( - COMMAND ccpp/framework/scripts/ccpp_prebuild.py - --config=ccpp/config/ccpp_prebuild_config.py - --suites=${CCPP_SUITES} - --builddir=${CMAKE_CURRENT_BINARY_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../.. - OUTPUT_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.out - ERROR_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.err -) -include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_CAPS.cmake) # → ${CAPS} -include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_SCHEMES.cmake) # → ${SCHEMES} -include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} -include(scm/src/CCPP_STATIC_API.cmake) # → ${API} -``` - -**Suite selection:** If `CCPP_SUITES` is not set by the user, a helper script -`suite_info.py` selects a compiler-appropriate subset. The full set of 63 suites is -used for production; subsets speed up development builds. - ---- - -### 9.10 Observations relevant to the redesign - -1. **`TYPEDEFS_NEW_METADATA` is a workaround that the redesign must eliminate.** The - DDT accessor information (which type lives at which accessor path) can be fully derived - from the CCPP metadata itself, given a well-designed metadata storage model. The - redesign must derive DDT accessor expressions automatically from the metadata rather - than requiring a separate hand-maintained dictionary. This is one of the primary - motivations for the new metadata storage design. - -2. **Three-level cap hierarchy (group → suite → static API) should be preserved.** - It provides clean separation: group caps are independently testable, suite caps - aggregate phases, the static API is the single host-callable entry point. - -3. **The static API's module-level `use` of host data is model-specific.** In models - where host data is not module-level (e.g., passed as subroutine arguments), the - static API pattern changes. The SCM is the simplest case because `physics` and `cdata` - are global module variables. - -4. **Instance and thread indexing are two orthogonal dimensions of host data access.** - Host model data uses two distinct indexing patterns that must be handled correctly: - - - **Regular state data** (Statein, Stateout, Sfcprop, etc.): dimensioned by instance - number — `physics%Statein(ccpp_instance_number)%array(1:horizontal_dimension, 1:vertical_dimension, ...)`. - In models supporting multiple in-memory model instances (ensemble), the top-level - DDT is an array indexed by `cdata%ccpp_instance`. - - - **Interstitial (per-thread scratch) data**: dimensioned by both instance and thread — - `physics%Interstitial(ccpp_instance_number, ccpp_thread_number)%array(1:horizontal_loop_extent, ...)`. - Critically, the horizontal dimension of interstitial arrays is sized to - `horizontal_loop_extent` (one OpenMP thread's chunk), not `horizontal_dimension` - (the full column count). `max_number_of_threads` instances are allocated per model - instance. Interstitial data can only be used during the **run phase** — this is a - known limitation of ccpp-prebuild that the redesign should address or at minimum - preserve explicitly. - -5. **Optional variable pointer arrays dimensioned by thread count** are the current - solution to thread-safe optional variable handling. This pattern is verbose (one - derived type + one array per optional variable per cap function) but correct. - The redesign could simplify this. - -6. **~550 optional variables in this model.** Optional/conditional variables are not - a corner case — they are a first-class feature. The redesign must handle them - efficiently and correctly. - -7. **Array size checks are debug-only and should not appear in the redesign by default.** - In prebuild they are only generated when the `--debug` flag is passed. The redesigned - generator should not produce them in normal mode — out-of-bounds access is caught at - runtime by compiler flags (e.g., `-fcheck=bounds` with gfortran, `-check bounds` with - ifort). The 12,991-line group cap is partly a consequence of generating these checks - unconditionally in the debug mode artifact examined here. - -8. **No unit conversions appear in GFS/SCM.** Unit conversion infrastructure must be - present in the redesign but the GFS physics package is self-consistent in units. - Unit conversions are more relevant for other host models. - -9. **The `one` constant** (integer parameter = 1) is passed as an explicit argument - everywhere and used as the lower bound in all array slices. This is a framework - convention. The redesign should decide whether this convention is preserved or - whether array lower bounds are handled differently. - -10. **Subcycles produce actual Fortran `do` loops inside the generated group cap.** - The loop from `1` to `cdata%loop_max` is generated directly in the cap function, - not left to the host model: - ```fortran - cdata%loop_max = 2 - do cdata%loop_cnt = 1, cdata%loop_max - call scheme_A_run(...) - if (ierr /= 0) return - call scheme_B_run(...) - if (ierr /= 0) return - end do - ``` - `cdata%loop_max` is set at the start of the subcycle block (from the `loop=` attribute - in the SDF XML) and `cdata%loop_cnt` is the current iteration counter, both visible - to schemes via the `ccpp_t` DDT. - ---- - -## 10. Real-world example: CAM-SIMA (capgen) - -*Source:* `EXT/cam-sima/` — uses `ccpp-capgen`. - -CAM-SIMA is the only model currently using capgen. It is still primarily a research model. -Unlike the SCM it uses a full 3D grid with OpenMP parallelism, but exposes host model -data as flat module variables rather than DDTs in the metadata layer. This example reveals -both what capgen can do and where it fundamentally fails. - -**Scale:** 1 suite (`cam7`), 2 run groups (`physics_before_coupler`, `physics_after_coupler`), -~75 scheme calls, 18 host `.meta` files, 893-line host cap, 2865-line suite cap. - ---- - -### 10.1 Suite structure - -`suite_cam7.xml` has two groups and no subcycles: - -| Group | Schemes (approx.) | Purpose | -|---|---|---| -| `physics_before_coupler` | 52 scheme calls | Cloud fraction, energy checks, dry adiabatic adjustment, Zhang-McFarlane deep convection full cycle, constituent tendency application | -| `physics_after_coupler` | ~20 scheme calls | Tropopause diagnostics, gravity wave drag (7 parameterizations + diagnostics), tendency application, energy consistency | - -CCPP phases in use: register, initialize, timestep_initial, run (per group), timestep_final, finalize. - ---- - -### 10.2 Host model variable structure - -**18 host `.meta` files, all of type `module`.** There are no `host` or `ddt` table types -anywhere. All host variables are flat scalars or arrays in Fortran modules. - -CAM-SIMA does **not** expose its physics DDTs (`phys_state`, `phys_tend`, `cam_in`, etc.) -through metadata. These types exist in `physics_types.F90` but have no `.meta` file. -The generated host cap accesses them directly via `use physics_types, only: phys_state, ...` -and passes individual DDT members as flat keyword arguments: -```fortran -! In cam_ccpp_cap.F90 — direct access to non-metadataized DDT members: -call cam7_physics_before_coupler(..., pint=phys_state%pint, t=phys_state%t, & - dtdt_total=phys_tend%dtdt_total, landfrac=cam_in%landfrac, ...) -``` - -This means capgen has no knowledge of how host data is structured. The host cap is -partly machine-generated and partly depends on manually wiring non-metadataized sources. -**This is a fundamental architectural gap** — changes to `physics_types` are invisible -to the framework. - -**Key host variables by module:** - -| Module | Key variables | -|---|---| -| `physics_grid` | `columns_on_task` (horizontal_dimension), `col_start`, `col_end`, lat, lon, area | -| `vert_coord` | `pver` (vertical_layer_dimension), `pverp` (vertical_interface_dimension) | -| `physconst` | ~35 physical constants, all `protected = True` | -| `cam_constituents` | `num_advected` (count of advected tracers) | -| `spmd_utils` | `mpicom`, `masterproc`, `npes`, `iam` | - -No instance indexing (`physics(1)`) and no thread-indexed DDTs appear — CAM-SIMA uses -a fundamentally different data model from the GFS/SCM stack. - ---- - -### 10.3 The two-cap architecture - -Capgen generates two distinct Fortran files: - -**`cam_ccpp_cap.F90` — the host cap (893 lines)** -- Module `cam_ccpp_cap` -- Imports non-metadataized host variables directly via `use physics_types`, `use physconst`, etc. -- Manages the constituent object (`ccpp_model_constituents_t`) — registration, initialization, gather/scatter, index lookup -- Public subroutines: `cam_ccpp_physics_run`, `cam_ccpp_physics_initialize`, etc. — the entry points the host model calls -- Dispatches to the suite cap, passing ~61–76 flat keyword arguments - -**`ccpp_cam7_cap.F90` — the suite cap (2865 lines)** -- Module `ccpp_cam7_cap` -- No host-specific imports — knows nothing about `physics_types`, `phys_state`, etc. -- All arguments are flat scalars and arrays, fully matched to metadata standard_names -- Contains all scheme calls, suite-level persistent variables, local temporaries, state machine -- The suite cap could in principle be used with any host model that provides the same standard names - -This two-cap split is **architecturally correct**: it separates host-specific binding -from physics-neutral dispatch. The redesign should preserve this separation. - ---- - -### 10.4 The flat-field argument problem — concrete evidence - -The run-phase subroutines expose the core problem with capgen's approach directly: - -```fortran -subroutine cam7_physics_before_coupler(errflg, errmsg, col_start, col_end, pver, dtime, & - gravit, pint, te_ini_dyn, teout, amiroot, iulog, ptend_s, temp, dtdt_total, cpair, & - lagrang, layer_surf, layer_toa, interface_surf, interface_toa, ncnst, piln, pmid, pdel, & - rpdel, qv, carr, cprops, rair, zvir, zi, zm, cp_or_cv_dycore, u, v, pintdry, phis, & - te_cur_phys, te_cur_dyn, tw_cur, latice, latvap, energy_formula_physics, & - energy_formula_dycore, cappa, q_tend, const_tend, qmin, pverp, cpwv, cpliq, rh2o, lat, & - long, pblh, mcon, tpert, dlf, rprd, ql, rliq, landfrac, cpair3, ttend_dp, tmelt, & - top_lev, ke, ke_lnd, cldfrc, domomtran, momcu, momcd, il1g, nstep, & - dudt_total, dvdt_total, fracis, dpdry, ps) -``` - -**61 dummy arguments for one group cap.** `physics_after_coupler` has 76. These are -individual flat arrays and scalars — no DDT in sight. This is exactly the problem that -three developers failed to fix: in the GFS/UFS context, this would be 1,200+ arguments. -The GFS physics stack simply cannot be connected to capgen in its current form. - -In contrast, the prebuild equivalent for the same data would pass `physics` (one DDT argument) -and `cdata` — two arguments covering hundreds of variables. - ---- - -### 10.5 Suite-level persistent variables — the framework-owned data pattern - -The suite cap allocates and owns arrays that persist across group calls within a timestep. -These are allocated in `cam7_initialize` and deallocated in `cam7_finalize`: - -```fortran -! Suite-level persistent (allocated in initialize, freed in finalize): -real(kind_phys), allocatable :: windu_tend(:,:) ! GW drag u-tendency accumulator -real(kind_phys), allocatable :: windv_tend(:,:) ! GW drag v-tendency accumulator -real(kind_phys), allocatable :: scaling_dycore(:,:) ! energy scaling factor -real(kind_phys), allocatable :: tend_te_tnd(:) ! energy tendency accumulator -real(kind_phys), allocatable :: tend_tw_tnd(:) ! water tendency accumulator -real(kind_phys), allocatable :: temp_ini(:,:) ! temperature saved at timestep start -real(kind_phys), allocatable :: z_ini(:,:) ! height saved at timestep start -real(kind_phys), allocatable :: flx_vap(:), flx_cnd(:), flx_ice(:), flx_sen(:) -logical, allocatable :: doconvtran(:) ! per-constituent convection flag -type(coords1d) :: p ! pressure coordinate DDT for GW drag -``` - -These are physics-internal variables — the host model does not know about them, does not -own them, and does not need to. This is the capgen "data ownership" model: the suite cap -is the data owner for variables that only matter within the physics. - -**This pattern is correct and desirable.** The complexity in capgen comes not from the -concept but from how these variables are discovered during analysis (scope-chain promotion) -and passed around (via VarDictionary). The redesign needs a simpler mechanism to achieve -the same result: statically enumerate physics-internal variables during analysis and have -the suite cap own them as named allocatables. - -During the run phase, suite-level persistent arrays are subsetted when passed to schemes: -```fortran -call gw_common_run(..., windu_tend=windu_tend(col_start:col_end, 1:pver), ...) -``` - -Run-phase local temporaries (e.g., `cape`, `cme`, `mu`, `md`) are allocated at function -entry and deallocated at exit: -```fortran -allocate(cape(col_start:col_end)) -... -call zm_convr_run(..., cape=cape, ...) -... -deallocate(cape) -``` - -These temporaries use `col_start` as the lower bound so that assumed-shape dummy arguments -in schemes see a 1-based array — a subtle but important detail. - ---- - -### 10.6 Horizontal chunking model - -CAM-SIMA uses `col_start`/`col_end` (passed as arguments to every run subroutine) to -define the current horizontal chunk: - -```fortran -ncol = col_end - col_start + 1 -``` - -Schemes declare `horizontal_loop_extent` and receive `ncol`. The horizontal dimension -in the host (storage dimension) is `columns_on_task`. The subsetting from storage to -loop extent happens at the boundary between host cap and suite cap — the host cap -passes the right subsections: - -```fortran -! In cam_ccpp_cap.F90: -call cam7_physics_before_coupler(..., col_start=col_start, col_end=col_end, & - pmid=phys_state%pmid, ...) ! full arrays passed; suite cap subsets internally -``` - -Inside the suite cap, persistent arrays are subsetted explicitly when passed to schemes: -```fortran -windu_tend(col_start:col_end, 1:pver) -``` -Local temporaries allocated as `allocate(cape(col_start:col_end))` are already -correctly sized and passed as assumed-shape `(:)`. - ---- - -### 10.7 State machine - -The suite cap has a character module variable tracking lifecycle state: - -```fortran -character(len=16) :: ccpp_suite_state = 'uninitialized' -``` - -Transitions: `uninitialized` → register → `uninitialized` → initialize → `initialized` -→ timestep_initial → `in_time_step` → (run, no state change) → timestep_final → -`initialized` → finalize → `uninitialized`. - -Each phase entry point checks the expected prior state: -```fortran -if (trim(ccpp_suite_state) /= 'in_time_step') then - errflg = 1 - write(errmsg, '(3a)') "Invalid initial CCPP state, '", trim(ccpp_suite_state), & - "' in cam7_physics_before_coupler" - return -end if -``` - -Non-run phases also include an OpenMP thread guard: -```fortran -#ifdef _OPENMP - if (omp_get_thread_num() > 1) then - errflg = 1 - errmsg = "Cannot call initialize routine from a threaded region" - return - end if -#endif -``` - -The state machine is simple, complete, and useful. The redesign should preserve it. - ---- - -### 10.8 Constituent variable handling - -CAM-SIMA demonstrates the full constituent lifecycle: - -```fortran -! In cam_ccpp_cap.F90: -type(ccpp_model_constituents_t), target :: cam_constituents_obj - -! Registration (scheme-declared constituents): -call suite_cam7_constituents_num_consts(num_consts) -call suite_cam7_constituents_const_name(iconst, const_name) -call cam_constituents_obj%new_field(const_name, ...) - -! Initialization (host-declared constituents like water vapor): -cam_model_const_stdnames(1) = "water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water" -call cam_constituents_obj%new_field(cam_model_const_stdnames(1), ...) - -! Per-timestep gather from host: -call cam_ccpp_gather_constituents(phys_state%q, ...) - -! Passing to suite cap: -call cam7_physics_before_coupler(..., - qv = cam_constituents_obj%vars_layer(:, :, cam_model_const_indices(1)), - carr = cam_constituents_obj%vars_layer, - cprops = cam_constituents_obj%const_metadata, ...) - -! Per-timestep scatter back to host: -call cam_ccpp_update_constituents(phys_state%q, ...) -``` - -The suite cap sees constituents as: -- `carr(:,:,:)` — the full rank-3 constituent array (ncol, nlev, ncnst) -- `qv(:,:)` — water vapor slice extracted in the host cap: `cam_constituents_obj%vars_layer(:,:,cam_model_const_indices(1))` -- `cprops(:)` — array of `ccpp_constituent_prop_ptr_t` metadata objects -- `doconvtran(1:ncnst)` — suite-level logical array set by scheme init indicating which constituents are convected - -This constituent API is sophisticated and worth preserving or improving in the redesign. - ---- - -### 10.9 Known defects in the capgen output - -**Repeated scheme init/final calls.** Capgen generates one init call per occurrence of -a scheme name in the XML, without deduplication: -- `qneg_init` called 5 times (once per `qneg` entry in the suite XML) -- `qneg_timestep_final` called 5 times -- `check_energy_chng_init` called twice -- `save_ttend_from_convect_deep_timestep_init` called 3 times - -If these routines have internal state, allocations, or side effects, this is a correctness -defect. The redesign must deduplicate init/final calls by unique scheme name. - -**Unit conversion embedded silently in the cap.** Before `zm_conv_convtran_run`: -```fortran -dpdry_local(:,1:pver) = 1.0E-2_kind_phys * dpdry(:,1:pver) ! Pa → hPa -``` -This is generated from the metadata units mismatch but appears as an opaque transform -in the cap. The redesign should make this visible (e.g., a comment naming the standard -name, the source units, and the target units). - ---- - -### 10.10 Build system — capgen invocation - -Capgen is invoked from Python (`cam_autogen.py`), not from cmake: - -```python -from ccpp_capgen import capgen -capgen_db = capgen(run_env, return_db=True) -``` - -This is a programmatic API call, not a subprocess. The `CCPPDatabaseObj` returned -(`capgen_db`) is then used directly in Python to query scheme lists, constituent names, -and file paths — avoiding the datatable XML query step that cmake-based invocations need. - -Output files consumed by the build: -- `cam_ccpp_cap.F90` — compiled into the atmosphere component -- `ccpp_cam7_cap.F90` — compiled into the atmosphere component -- `ccpp_kinds.F90` — compiled into the atmosphere component -- Utility files from `ccpp_framework/src/` (copied to build dir) -- `ccpp_datatable.xml` — queried by the build system for file lists - ---- - -### 10.11 Observations relevant to the redesign - -1. **The two-cap split (host cap + suite cap) is the right architecture.** It cleanly - separates host-specific binding from physics-neutral dispatch. The redesign must - preserve this. - -2. **Flat-field arguments in the suite cap are the critical failure.** 61–76 dummy - arguments per run subroutine is already large for a research model; for UFS/GFS - with 1,200+ variables it is completely infeasible. The redesign must pass DDTs. - -3. **The CAM-SIMA host does not use DDTs in metadata.** All host variables are flat - module variables. This is a fundamentally different host model architecture from - GFS/SCM. The redesign must support both styles: flat-module hosts (CAM-SIMA) and - deep-DDT hosts (GFS/SCM). - -4. **Non-metadataized variables hardwired into the host cap is a serious gap.** - `phys_state`, `phys_tend`, `cam_in` from `physics_types` have no `.meta` files. - The host cap accesses them directly. This means the framework cannot verify or - track these variables. The redesign should either require full metadata coverage - or have an explicit mechanism for declaring non-metadataized pass-through variables. - -5. **Suite-level persistent variables (framework-owned data) work well in practice.** - `windu_tend`, `scaling_dycore`, `temp_ini`, etc. are owned by the suite cap, invisible - to the host, and persist across group calls. This is the right pattern for - physics-internal state. The redesign needs this but with a simpler discovery mechanism - than capgen's scope-chain promotion. - -6. **Deduplicate init/final calls.** The redesign must deduplicate `_init`, `_finalize`, - `_timestep_init`, and `_timestep_final` calls by unique scheme name (not by occurrence - in the XML). - -7. **The constituent API in the host cap is comprehensive.** The `ccpp_model_constituents_t` - object with its register/init/gather/scatter/index API is sophisticated and should be - preserved or improved. - -8. **The suite-variables introspection subroutine** (`ccpp_physics_suite_variables`, - enumerating 83 standard names as inputs/outputs) is a useful capability for build - system integration and should be in the redesign. - -9. **The programmatic Python API** (`capgen(run_env, return_db=True)`) is valuable - for hosts like CAM-SIMA that invoke the generator from Python. The redesign should - support both CLI and programmatic invocation. - -10. **Unit conversions must be annotated in the generated cap**, not silently embedded - as magic-number multiplications. A comment with source units, target units, and the - standard name involved is the minimum. - -11. **The horizontal chunking model** (`col_start`/`col_end` as explicit arguments, - `ncol = col_end - col_start + 1` computed at entry) works and is clean. Suite-level - persistent arrays are allocated full-size and subsetted at call sites. - -12. **No optional variables in this model.** CAM-SIMA does not exercise optional/active - variable handling. This feature must be in the redesign but is not demonstrated here. - ---- - -## 11. Real-world example: UFS Weather Model (prebuild) - -The UFS Weather Model is the most complex and production-critical of the three examples. -It is a fully-coupled, 3-D operational NWP model. The CCPP physics is used in the -atmospheric component (`UFSATM`). Unlike the SCM (column model, process-split only) and -CAM-SIMA (capgen, flat-field arguments), UFS uses prebuild in a 3-D blocked/threaded -configuration that is architecturally distinct from both prior examples. - -The two suites analyzed here are: -- `FV3_GFS_v17_coupled_p8` — the primary operational GFS suite -- `FV3_GFS_v17_coupled_p8_ugwpv1` — a variant replacing `unified_ugwp` with `ugwpv1` - -The ugwpv1 suite is structurally identical to the base suite except for the `phys_ps` -group (4 extra scheme calls), so all observations below apply to both. - ---- - -### 11.1 Suite structure - -The primary suite has 5 groups: - -| Group | Subcycles | Scheme calls | Phase called | -|-------|-----------|-------------|--------------| -| `time_vary` | 1 | 4 | timestep_init (domain-level, no blocking) | -| `radiation` | 1 | 8 | run (block/thread loop) | -| `phys_ps` | 3 (loop=1, loop=2, loop=1) | 21 | run (block/thread loop) | -| `phys_ts` | 3 (loop=1, loop=1, loop=1) | 12 | run (block/thread loop) | -| `stochastics` | 1 | 2 | run (block/thread loop) | - -The `time_vary` group is the only one called at timestep_init/finalize. All other groups -are called from the run phase via the OpenMP blocked loop. This is a fundamentally -different usage pattern from SCM (which runs everything sequentially) and CAM-SIMA -(which has no run phase at all for the groups analyzed). - -The `phys_ps` group has a surface iteration subcycle with `loop="2"`, which generates an -actual Fortran `do` loop in the cap body: -```fortran -! Start of next subcycle -cdata%loop_max = 2 -do cdata%loop_cnt = 1, cdata%loop_max - ! ... sfc_diff, sfc_nst, noahmpdrv, sfc_land, sfc_cice, sfc_sice ... -end do -``` - ---- - -### 11.2 Cap hierarchy and scale - -The three-level hierarchy is preserved from prebuild: - -``` -ccpp_static_api.F90 (627 lines) ← suite+group name dispatch - ↓ -ccpp_fv3_gfs_v17_coupled_p8_cap.F90 (363 lines) ← calls all group caps in order - ↓ -ccpp_fv3_gfs_v17_coupled_p8_time_vary_cap.F90 (1404 lines) -ccpp_fv3_gfs_v17_coupled_p8_radiation_cap.F90 (967 lines) -ccpp_fv3_gfs_v17_coupled_p8_phys_ps_cap.F90 (4226 lines) ← 200 optional ptr arrays -ccpp_fv3_gfs_v17_coupled_p8_phys_ts_cap.F90 (1953 lines) -ccpp_fv3_gfs_v17_coupled_p8_stochastics_cap.F90 (443 lines) -``` - -The ugwpv1 variant generates another 10,220 lines of largely redundant code (identical -caps with one suite-name prefix change and minor scheme-list differences). Total for both -suites: 18,333 lines of generated Fortran. - -This redundancy is a key motivation for the redesign: suite variants that share groups -should not regenerate identical cap code. The redesign should support group-level cap -sharing across suite variants. - ---- - -### 11.3 Host model DDT structure - -All host data lives in `CCPP_data.F90` as module-level `save, target` variables: - -```fortran -type(GFS_control_type) :: GFS_control ! config/control -type(GFS_statein_type) :: GFS_statein ! atmospheric state in -type(GFS_stateout_type) :: GFS_stateout ! atmospheric state out -type(GFS_grid_type) :: GFS_grid ! grid geometry -type(GFS_tbd_type) :: GFS_tbd ! temporal interp data -type(GFS_cldprop_type) :: GFS_cldprop ! cloud properties -type(GFS_sfcprop_type) :: GFS_sfcprop ! surface properties -type(GFS_radtend_type) :: GFS_radtend ! radiation tendencies -type(GFS_coupling_type) :: GFS_coupling ! coupling fields -type(GFS_diag_type) :: GFS_intdiag ! diagnostics -type(GFS_interstitial_type), allocatable (:) :: GFS_interstitial ! scratch, per thread -``` - -Plus three `ccpp_t` instances for different levels of parallelism (see §11.5). - -This is structurally similar to the SCM's `physics` DDT hierarchy, but with one key -difference: all DDTs are at the same flat level rather than nested (no `physics%Statein`, -only `GFS_statein`). Each DDT maps to a distinct functional role. - -The `GFS_typedefs.F90` file (not auto-generated) defines all DDT types along with ~30 -physical constants (`con_pi`, `con_g`, `con_rd`, etc.) that also appear in the metadata. - ---- - -### 11.4 DDT arguments in the cap chain - -The static API imports all DDTs and physical constants from `CCPP_data` and `GFS_typedefs` -via `use` statements, then passes them as named arguments to group cap functions. This is -the full DDT-argument pattern that prebuild implements: - -```fortran -! In ccpp_static_api.F90: -use ccpp_data, only: gfs_control, gfs_statein, gfs_sfcprop, ... -use gfs_typedefs, only: con_pi, con_g, con_rd, ... - -ierr = fv3_gfs_v17_coupled_p8_phys_ps_run_cap( & - one=one, gfs_control=gfs_control, cdata=cdata, & - gfs_statein=gfs_statein, gfs_sfcprop=gfs_sfcprop, & - con_g=con_g, con_pi=con_pi, ... & - gfs_interstitial=gfs_interstitial) -``` - -The group cap receives these as typed `intent(*), target` dummy arguments and uses them -directly to construct call-site subsections. This means **the group cap is fully portable -— it does not use any host module directly**, only what it receives as arguments. - -The `target` attribute is required because the cap creates pointer sections of these DDTs -(array subsections via pointer assignment) when handling optional variables. - ---- - -### 11.5 The dual cdata architecture - -UFS uses two distinct sets of `ccpp_t` handles with different scopes: - -**Domain-level (`cdata_domain`)**: Used for non-run phases (init, finalize, time_vary -timestep_init/finalize). Called once per step, no blocking: -```fortran -cdata_domain%blk_no = 1; cdata_domain%chunk_no = 1 -cdata_domain%thrd_no = 1; cdata_domain%thrd_cnt = 1 -``` - -**Block/thread-level (`cdata_block(nb, nt)`)**: Used for run phase (radiation, phys_ps, -phys_ts, stochastics). Allocated as a 2-D array `(1:nblks, 1:nthrdsX)` where `nthrdsX` -accounts for non-uniform last-block sizing: -```fortran -cdata_block(nb,nt)%blk_no = nb -cdata_block(nb,nt)%chunk_no = nb ! block number = chunk number -cdata_block(nb,nt)%thrd_no = nt -cdata_block(nb,nt)%thrd_cnt = nthrdsX -``` - -The redesign must support this dual cdata usage: a single `cdata` handle for domain-level -phases and a 2-D array of handles for blocked run phases. - ---- - -### 11.6 OpenMP threading model - -Non-run phases allow internal threading in physics schemes: -```fortran -GFS_control%nthreads = nthrds ! all N threads available to physics -call ccpp_physics_timestep_init(cdata_domain, ...) -``` - -Run phase uses all threads for blocking, so physics must not spawn additional threads: -```fortran -GFS_control%nthreads = 1 ! no internal threading allowed -!$OMP parallel num_threads(nthrds) ... -!$OMP do schedule(dynamic,1) -do nb = 1, nblks - call GFS_Interstitial(nt)%create(ixs=chunk_begin(nb), ixe=chunk_end(nb), model=GFS_control) - call ccpp_physics_run(cdata_block(nb,nt), group_name="phys_ps", ...) - call GFS_Interstitial(nt)%destroy(GFS_control) -end do -!$OMP end do -!$OMP end parallel -``` - -The `nt = omp_get_thread_num()+1` pattern (1-based thread index) is used throughout. -Each thread owns one `GFS_Interstitial(nt)` and one `cdata_block(nb,nt)` per block -iteration. The dynamic schedule means different threads process different blocks at -different times, which is why the interstitial must be created/destroyed per-iteration -rather than pre-allocated per-thread. - ---- - -### 11.7 Horizontal dimension: the chunk_begin/chunk_end pattern - -For non-run phases, the full horizontal dimension is used at every call site: -```fortran -tgrs(one:gfs_control%ncols, one:gfs_control%levs) -``` - -For run phases, the chunk range is looked up from the control DDT using the block number: -```fortran -tgrs(gfs_control%chunk_begin(cdata%chunk_no) : gfs_control%chunk_end(cdata%chunk_no), & - one:gfs_control%levs) -``` - -The chunk size (horizontal extent `im`) is retrieved as: -```fortran -im = gfs_control%blksz(cdata%blk_no) -``` - -`blksz(nb)` handles **non-uniform block sizes**: the last block may be smaller than the -others if the domain size is not divisible by the number of blocks. The `chunk_begin`/ -`chunk_end` arrays (indexed by chunk number = block number) give the global offset range. - -This is a cleaner pattern than SCM's `chunk_begin`/`chunk_end` as explicit dummy -arguments, because UFS looks them up from the already-passed `gfs_control` DDT. - -**Critical implication for the redesign**: The subsetting pattern `(chunk_begin:chunk_end)` -appears at every single array call site in the run phase — literally hundreds of times in -the phys_ps cap alone. This boilerplate is generated by prebuild from the metadata. In -the redesign, this subsetting must remain at the call site (not higher up) to allow each -thread to process its own chunk independently. - ---- - -### 11.8 The GFS_interstitial — pointer-based scratch DDT - -`GFS_interstitial_type` (defined in `CCPP_typedefs.F90`) is a DDT where **every field is -a pointer**, initialized to null: -```fortran -type GFS_interstitial_type - real(kind_phys), pointer :: adjsfculw_land(:) => null() - real(kind_phys), pointer :: del(:,:) => null() - ! ... ~200+ pointer fields -end type -``` - -This is dramatically different from the SCM's interstitial (which is a regular allocatable -DDT allocated once per thread at startup). The UFS interstitial is: -1. **Created** (`GFS_Interstitial(nt)%create(ixs, ixe, model)`) before each block — this - allocates all required fields to the chunk size `ixe-ixs+1` -2. **Reset** (`GFS_Interstitial(nt)%reset(model)`) to zero before radiation and phys_ps -3. **Destroyed** (`GFS_Interstitial(nt)%destroy(model)`) after each block — deallocates - -This design exists because different blocks (especially the last block) can have different -sizes. Pre-allocating to the maximum size wastes memory at scale; per-block allocation -ensures exact sizing. The pointer-based design also allows the `create()` method to -selectively allocate only the fields needed for the current physics configuration. - -In the caps, the interstitial is accessed as: -```fortran -gfs_interstitial(cdata%thrd_no)%del(chunk_begin:chunk_end, one:levs) -``` - -The interstitial array is 1-D (indexed by thread, not by `(instance, thread)` as in SCM). -This works because UFS has only one model instance at runtime — no ensemble-in-memory. - ---- - -### 11.9 Optional variables — the pointer array pattern at scale - -The phys_ps run cap has **200 optional pointer arrays** in its local variable section. -Each looks like: -```fortran -type :: real_kind_phys_rank1_ptr_arr_type - real(kind_phys), dimension(:), pointer :: p => null() -end type real_kind_phys_rank1_ptr_arr_type -type(real_kind_phys_rank1_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array -``` - -Usage pattern (consistent with SCM but with threading dimension): -```fortran -if (gfs_control%lndp_type /= 0) then - sfc_wts_1_ptr_array(cdata%thrd_no)%p => & - gfs_coupling%sfc_wts(chunk_begin:chunk_end, one:gfs_control%n_var_lndp) -end if -! ... scheme call ... -if (gfs_control%lndp_type /= 0) then - nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) -end if -``` - -The array is dimensioned by `cdata%thrd_cnt` (total thread count) and indexed by -`cdata%thrd_no` (current thread number). This handles the threaded run phase where -multiple threads are simultaneously executing the same run cap function with different -chunk ranges. Each thread independently associates and nullifies its own pointer slot. - -200 optional variables in `phys_ps` alone. This is the regime for which the SCM had ~550 -total optional vars — confirming that operational 3-D GFS physics is heavily optional-var -driven. The design is sound but generates enormous boilerplate. - -A key observation: the type definition for each pointer wrapper (`integer_..._ptr_arr_type`, -`real_kind_phys_rank1_ptr_arr_type`, etc.) is **re-declared inside every single function -that needs it**. This results in duplicate type definitions across all group caps. The -redesign should define these wrapper types once in a shared module. - ---- - -### 11.10 Physical constants as metadata variables - -The UFS static API has an extensive USE list of physical constants from `gfs_typedefs`: -``` -con_pi, con_g, con_t0c, con_hfus, con_solr_2008, con_solr_2002, con_c, con_plnk, -con_boltz, con_rd, ltp, con_zero, con_rerth, con_p0, con_rv, con_cp, con_rgas, -con_amd, con_amw, con_avgd, con_hvap, con_eps, con_omega, con_fvirt, con_ttp, -con_thgni, con_epsm1, con_rog, con_rocp, con_tice, con_sbc, con_jcal, con_rhw0, -rlapse, rhowater, karman, con_1ovg, con_cliq, con_cvap, rainmin, con_epsm1 (30+ total) -``` - -These travel through the full chain: static API USE → suite cap argument → group cap -argument → scheme call argument. Each constant is declared as a separate scalar dummy -argument (`real(kind_phys), intent(in), target :: con_pi`) in every group cap that needs -it. - -This is correct but verbose. The redesign should consider whether constants should be -gathered into a dedicated DDT (e.g., `gfs_constants_type`) so the cap chain carries one -argument instead of 30. This would also eliminate the need to explicitly enumerate which -constants each group needs — they could all come along in the constants DDT. - ---- - -### 11.11 The `one` lower-bound anchor - -The integer constant `one = 1` (from `ccpp_types`) is passed as an explicit argument -throughout the UFS cap chain for the same reason as in SCM: it anchors lower array bounds -without triggering association-status issues: -```fortran -type(gfs_interstitial_type), intent(inout), target :: gfs_interstitial(one:) -tgrs(one:gfs_control%ncols, one:gfs_control%levs) -``` - -This pattern is ubiquitous and is a known prebuild idiom. - ---- - -### 11.12 No framework-owned persistent variables - -Unlike CAM-SIMA (which allocates scheme-persistent variables in the suite cap), the UFS -has no framework-owned persistent state in any cap. All persistent state lives in the host -DDTs (`GFS_tbd`, `GFS_sfcprop`, etc.). The interstitial DDT (`GFS_interstitial`) is purely -transient — created and destroyed each block. - -This is consistent with UFS's prebuild-based architecture. Whether framework-owned -persistent variables would be beneficial for UFS is an open question for the redesign. - ---- - -### 11.13 Build system and driver - -Prebuild is invoked from CMake (not programmatically) and generates: -- Group cap files (one per group × suites) -- Suite cap files (one per suite) -- `ccpp_static_api.F90` -- `CCPP_CAPS.cmake`, `CCPP_SCHEMES.cmake`, `CCPP_TYPEDEFS.cmake` — consumed by CMake to - enumerate files to compile - -The host driver (`CCPP_driver.F90`) is **hand-written**, not auto-generated. It owns the -OpenMP loop, the cdata allocation/setup, the interstitial create/destroy, and the -diagnostic bucket zeroing. This is a significant difference from CAM-SIMA where the -equivalent driver code is partially generated. In the redesign, this host driver code -should remain hand-written — it encodes model-specific threading and blocking decisions -that cannot be derived from metadata alone. - ---- - -### 11.14 Observations relevant to the redesign - -1. **The DDT-argument cap chain is fully validated at UFS scale.** Passing 10+ DDTs plus - 30+ scalar constants as named arguments through three cap levels works correctly in - production. The redesign must replicate this exactly. - -2. **The chunk_begin/chunk_end subsetting at call sites is non-negotiable.** Hundreds of - array sections per group cap. The generator must produce this from the metadata - `horizontal_dimension` standard name and the `active` flag for optional variables. - This is prebuild's core value at 3-D scale. - - *Design direction*: Rather than carrying `chunk_no` in cdata and having the cap look - up `gfs_control%chunk_begin(chunk_no)`, the redesign should pass - `horizontal_loop_begin` and `horizontal_loop_end` as explicit arguments directly to - `ccpp_physics_run()` (and analogous calls). This decouples the cap from knowing about - the host's internal chunk-lookup arrays. The host driver sets these for each block - iteration and passes them in; the cap uses them directly. - -3. **The domain-vs-block execution contexts must be supported, but the cdata object is - not necessarily the right mechanism.** The key information is: instance number, thread - number, horizontal_loop_begin, horizontal_loop_end, error flag/message. If all of - these are explicit named arguments to `ccpp_physics_*`, the cdata object becomes - redundant scaffolding. This is an open design question to be discussed separately, but - the UFS analysis shows that cdata carries exactly these values — the object is a - transport container, not a framework abstraction. - -4. **The `blksz` non-uniform block size is a first-class concern.** The generator must - produce `im = gfs_control%blksz(cdata%blk_no)` (or an equivalent `horizontal_loop_extent` - computed from the explicit begin/end) for the horizontal extent argument in run phases. - -5. **GFS_interstitial as a pointer-DDT is the correct design for 3-D models.** Creating - and destroying per block avoids memory waste from over-allocation to the maximum chunk - size. The pointer-based field design enables selective allocation. The redesign should - document this pattern and support it. (Whether the generator should emit the - `type(X_interstitial_type)` DDT definition itself or only the caps is TBD.) - -6. **200 optional pointer arrays in one group cap is manageable but the wrapper type - proliferation is not.** The 4 wrapper types (`integer_r1_ptr_arr_type`, - `real_r1_ptr_arr_type`, `real_r2_ptr_arr_type`, `character_len3_r1_ptr_arr_type`) - should be defined once in a shared module (e.g., `ccpp_types.F90`) and reused across - all caps, eliminating thousands of duplicate lines. - -7. **Physical constants as metadata variables must be gathered into a constants DDT.** - The redesign will collect all physics constants into a single `constants_type` DDT - (or equivalent), reducing 30+ individual scalar arguments in the cap chain to one - argument. This requires a metadata declaration mechanism for compound read-only - objects (i.e., constants do not need intent tracking the way state variables do). - -8. **No framework-owned persistent variables in UFS** confirms that this feature is - optional and model-specific. The redesign needs to support it (for CAM-SIMA-like - models) but should not force it on models that do not need it. - -9. **The host driver is correctly hand-written.** The OpenMP blocking, interstitial - lifecycle, diagnostic bucket management — these are model-specific decisions that - belong in the host driver, not in generated code. The redesign should not try to - generate the driver. - -10. **Suite variant cap redundancy is not a concern.** For research/development, multiple - suites are active simultaneously and generated code size doesn't matter. For - production, only one suite is compiled and used at a time. The redesign need not - prioritize eliminating redundant group cap code across suite variants. - ---- - -## 12. Real-world example: Navy NEPTUNE (prebuild, restricted) - -The NEPTUNE source code cannot be shared. The following is based on architectural -description provided by the lead developer. - -NEPTUNE uses `ccpp-prebuild` with the same GFS physics as UFS and nearly identical suites. -Its unique distinguishing feature is **multiple coexisting CCPP physics instances** — it -is the only model among the four examples that exercises this capability at runtime. - ---- - -### 12.1 Multiple instances — the N-dimensioned DDT array mechanism - -In NEPTUNE, the host model allocates N copies of all GFS DDTs as 1-D arrays indexed by -instance number: - -```fortran -type(GFS_sfcprop_type), allocatable :: gfs_sfcprop(1:N) -type(GFS_statein_type), allocatable :: gfs_statein(1:N) -type(GFS_stateout_type), allocatable :: gfs_stateout(1:N) -! ... all GFS DDTs dimensioned 1:N -type(GFS_control_type), allocatable :: gfs_control(1:N) -``` - -The static API imports these module-level arrays via `use` statements (same as UFS). -The instance selection happens at the call site inside the group cap, using -`cdata%ccpp_instance` as the array index: - -```fortran -call foo_run( & - tair = gfs_statein(cdata%ccpp_instance)%tair( & - gfs_control(cdata%ccpp_instance)%chunk_begin(cdata%chunk_no) : & - gfs_control(cdata%ccpp_instance)%chunk_end(cdata%chunk_no), & - 1:nvertical), & - ...) -``` - -Three things are happening simultaneously at each call-site array section: -1. **Instance selection**: `gfs_statein(cdata%ccpp_instance)` picks the correct DDT from - the N-element array -2. **Chunk subsetting**: `chunk_begin(chunk_no):chunk_end(chunk_no)` applies the run-phase - horizontal slice -3. **Vertical bound**: explicit `1:nvertical` - -This is the same pattern as UFS except the DDTs are 1-D arrays rather than scalars. -The generator must produce this instance-indexed subsetting when the host declares its -DDTs as arrays. - ---- - -### 12.2 What NEPTUNE tells us about `cdata%ccpp_instance` - -The `initialized(200)` array in every group cap (confirmed in both SCM and UFS caps) now -has its full motivation: it handles up to 200 simultaneous instances without requiring -per-instance cap code. The `cdata%ccpp_instance` value (1-based) is the runtime selector -into both the host DDT arrays and the `initialized` guard array. - -NEPTUNE is the reason `200` is not `1`. In single-instance models (UFS, SCM, CAM-SIMA) -`cdata%ccpp_instance` is always 1 and the N-dimensioned DDT arrays have `N=1`. - ---- - -### 12.3 Observations relevant to the redesign - -1. **Multiple instances require only one change at the call site**: inserting the instance - index at the correct dimension position. Everything else (chunking, optional variables, - threading) composes with this unchanged. - -2. **The instance dimension can appear anywhere in any host variable — not just as an - index into an array of DDTs.** A flat array `flat_field(1:ninstance, 1:nhoriz, 1:nvert)` - is equally valid; its call site becomes: - ```fortran - flat_field(instance_number, horiz_begin:horiz_end, 1:nvert) - ``` - The generator handles this by classifying each dimension by its declared standard name. - `instance_dimension` is a registered standard name (like `horizontal_dimension` and - `vertical_dimension`) — the generator knows its semantics regardless of where it - appears in the dimension list or whether the variable is a DDT array element or a - plain array. See §13.4 for the full dimension classification model. - -3. **No new cap-level mechanism is needed for multi-instance.** The instance number - (from the control layer, see §13) is sufficient. The cap code shape is the same; - only the call-site indexing expression differs based on the declared dimension roles. - ---- - -## 13. Cross-cutting design decision: how host data enters the cap chain - -Across all four models, two mechanisms are used for getting host model data into the -generated caps: - -| Mechanism | Models using it | Description | -|-----------|----------------|-------------| -| **Module USE** | UFS, SCM, CAM-SIMA, NEPTUNE | Static API has `use ccpp_data, only: gfs_statein, ...`. Data module name is known at generation time. | -| **Command-line arguments** | capgen (optional) | Generator accepts host variable access paths as CLI flags; generated caps receive data as explicit dummy arguments. | - -### 13.1 The capgen dual-mechanism problem - -Capgen supports both mechanisms, and this is a direct source of its complexity. The -variable-matching logic, VarDictionary scope chains, and `CCPPDatabaseObj` all exist -partly to handle the routing of variables that may arrive via either path. Maintaining -two entry points to the data layer doubles the surface area that must be tested and -reasoned about. - -### 13.2 The proposed single-mechanism approach - -The redesign will use **module USE exclusively** for all host data. The reasoning: - -- All four production models already use module USE, including CAM-SIMA (the capgen - model), which does not use capgen's CLI-argument path in practice. -- Module names are stable, known at generation time, and make the generated code - self-documenting (`use ccpp_data, only: gfs_statein` is unambiguous). -- Eliminating the CLI-argument entry path eliminates an entire class of generator - complexity. - -### 13.3 Runtime control variables — the thin explicit layer - -While all *data* enters via module USE, a set of *control* variables must be passed at -runtime because they change from call to call. These are not physics data; they tell the -cap *how* to index into the data it already has access to: - -| Variable | Purpose | When it matters | -|----------|---------|----------------| -| `ccpp_instance` | Select the instance dimension in host variables | NEPTUNE (N>1); others use 1 | -| `ccpp_thread_no` | Index optional pointer arrays per thread | Run phase with OpenMP | -| `horizontal_loop_begin` | Start of horizontal chunk to process | Run phase | -| `horizontal_loop_end` | End of horizontal chunk to process | Run phase | -| `ccpp_nthreads` | Max threads available for internal physics use | Non-run phases (currently `gfs_control%nthreads`) | -| `errmsg` / `errflg` | Error reporting return path | All phases | - -These are exactly the values that `cdata` carries in the current implementation. -Whether they are packaged as a `ccpp_t` struct or passed as individual named arguments to -`ccpp_physics_*` is an open design question for implementation. Either way, the generator -only needs to know about these variables and their standard names — it does not need to -accept host data paths on the command line. - -### 13.4 The dimension classification model - -A host variable's metadata declares the **standard name of each of its dimensions** in -order. The generator classifies every dimension into one of three categories and -constructs the call-site expression accordingly. - -**Category 1 — Registered dimensions.** The generator knows the semantics of these -standard names and generates special call-site expressions for them: - -| Standard name | Call-site expression | Notes | -|--------------|---------------------|-------| -| `instance_dimension` | `instance_number` (scalar index) | Omitted if variable has no instance dimension | -| `horizontal_dimension` | `1:horizontal_dimension` (non-run) or `horiz_begin:horiz_end` (run) | Phase-dependent | -| `vertical_dimension` | `1:vertical_dimension` | Fixed range | - -`instance_dimension` has the same registered status as `horizontal_dimension` and -`vertical_dimension`. Single-instance models simply do not declare any variables with -an `instance_dimension`, and the generator omits that index entirely. - -**Category 2 — Arbitrary host-declared dimensions.** Any dimension whose standard name -is not in the registered set. These are declared in host metadata pointing to a Fortran -expression accessible via module USE — either a flat module variable or a DDT member -(e.g. `gfs_control%ntrac`, `gfs_control%kice`). The generator emits `1:expression` -at the call site, resolved at generation time from the metadata. Fixed-index extractions -(e.g. `gfs_statein%qgrs(..., gfs_control%ntqv)`) are a special case: the dimension -value is a scalar index rather than a range upper bound, and the metadata must declare -which case applies. - -**Category 3 — Optional selector.** Not a dimension per se, but a boolean `active` -condition declared in variable metadata. Generates a pointer-association guard around -the call site (the pattern described in §9 and §11). - -This three-category model works uniformly regardless of host layout: -- `gfs_statein(instance)%tair(horiz, vert)` — registered instance + registered horizontal + registered vertical -- `flat_field(instance, horiz, vert, ntrac)` — registered + registered + registered + arbitrary -- `flat_field(horiz, vert)` — no instance dimension, single-instance model - -No special-casing per host model is needed in the generator. - -### 13.5 `type = control` — metadata declaration for runtime control variables - -The registered dimensions (§13.4 Category 1) are *dimension names* that appear in a -variable's `dimensions = (...)` list. Their actual *runtime values* are supplied by a -separate set of variables declared with `type = control` in host metadata. - -| `type = control` standard name | Fills in registered dimension / purpose | -|-------------------------------|----------------------------------------| -| `ccpp_instance` | `instance_dimension` — scalar index selecting the active instance | -| `ccpp_thread_no` | Not a dimension; indexes optional pointer arrays per thread | -| `horizontal_loop_begin` | Lower bound of `horizontal_dimension` in run phase | -| `horizontal_loop_end` | Upper bound of `horizontal_dimension` in run phase | -| `ccpp_nthreads` | Not a dimension; max threads available for internal physics use | -| `errmsg` / `errflg` | Error reporting return path | - -Variables declared `type = control` are: -- **Passed explicitly as runtime arguments** to `ccpp_physics_*` by the host driver - (not accessed via module USE, because their values change per call) -- **Used by the generator** to construct call-site indexing expressions for registered - dimensions, and to generate the `ccpp_nthreads` assignment before non-run scheme calls -- **Available to physics schemes** by standard name like any other variable — if a scheme - declares a variable with a matching standard name (e.g. `ccpp_nthreads`, - `horizontal_loop_begin`), the framework passes it as a scheme argument in the normal way - -This is similar in concept to capgen's `type = host` annotation but with a narrower, -well-defined scope. The name `control` is intentional: these variables *control* how -the cap indexes into the data, not what the data is. - -The set of recognized standard names for `type = control` variables is fixed and small. -Declaring them explicitly in metadata — rather than having the generator recognize magic -names — keeps the mechanism open and self-documenting. - -### 13.6 Consequences for the generator - -1. The generator reads host metadata to learn: - - Module names for all host data variables (emitted as `use` statements in the static API) - - The dimension standard names of each variable (for call-site expression construction) - - Which variables are `type = control` (for the runtime argument layer) -2. At cap generation time, the static API's `use` statements are emitted from the module - names — no runtime flexibility, no CLI data routing. -3. Call-site subsetting for every variable is constructed purely from its declared - dimension standard names: registered dimensions use the Category 1 rules; arbitrary - dimensions are resolved to Fortran expressions via the host metadata. -4. The only runtime inputs to the cap are the `type = control` variables. Their values - are supplied by the host driver for each `ccpp_physics_*` call. diff --git a/doc/redesign_prompt_20260513T0733.md b/doc/redesign_prompt_20260513T0733.md deleted file mode 100644 index 0ac76cde..00000000 --- a/doc/redesign_prompt_20260513T0733.md +++ /dev/null @@ -1,1230 +0,0 @@ -# CCPP Framework Code Generator — Redesign Specification - -*Last revised: 2026-05-13.* - -## Purpose - -This document is a complete implementation specification for a new CCPP Framework code -generator (`ccpp-capgen-ng`). It supersedes both `ccpp-prebuild` and `ccpp-capgen`. An -implementer should be able to build the new generator from scratch using this document -alone, supplemented by the real-world examples in `redesign_analysis.md`. - -The spec is essentially as-implemented as of the date above. User-facing -deltas relative to ccpp-prebuild and the original ccpp-capgen are -collected in `doc/migration.md`; section 18 of this document is a rolling -"outstanding work" tracker. - ---- - -## 1. Background and Motivation - -The CCPP Framework couples host NWP models (UFS Weather Model, NEPTUNE, CCPP-SCM, -CAM-SIMA) to physics parameterization schemes by auto-generating Fortran interface -("cap") code. Two generators exist today: - -- **`ccpp-prebuild`** — simple, procedural Python; fast; DDT arguments; used in - production by UFS, NEPTUNE, SCM. Does not support framework-owned variables. -- **`ccpp-capgen`** — complex OO Python; flat-field arguments; used in CAM-SIMA. The - deep class hierarchy (`VarDictionary`, `VarCompatObj`, `CCPPDatabaseObj`, etc.) makes - it unmaintainable. Three developers spent considerable time trying to add DDT argument - passing and could not succeed. - -The redesign starts fresh, drawing lessons from both. The guiding principle is: -**simplicity of prebuild, feature set of capgen**. - -The primary failures that triggered the redesign: -1. capgen passes flat fields to group caps — infeasible at UFS/NEPTUNE scale (1200+ - variables), breaks under compiler debug flags for optional variables. -2. capgen's scope-chain variable promotion is the source of most complexity. -3. Nobody on the team fully understands capgen. - ---- - -## 2. Toolchain Structure - -The redesign produces **two separate tools** that share the same metadata parsing library: - -### 2.1 Validator (`ccpp_validator.py`) - -Parses both Fortran source files and metadata files, compares them, and reports -discrepancies. Run by developers before invoking the generator — e.g., during scheme -development or in CI. Does **not** generate any Fortran output. - -For each scheme phase declared in a `.meta` file, the validator checks that the -corresponding Fortran subroutine: (1) exists in the source tree, (2) has the same number -of dummy arguments, and (3) the argument names match the `local_name` values in the -metadata (order-insensitive). - -Fortran source files can be supplied explicitly on the CLI (`--source-files`). When -omitted, the validator auto-discovers the Fortran source for each scheme table using the -`source_path` table-level property (Section 3.5): it looks for a `.F90` file with the -same base name as the `.meta` file, in the directory given by `source_path`. - -### 2.2 Code Generator (`ccpp_capgen_ng.py`) - -Parses metadata only. Assumes metadata correctly describes the Fortran source — performs -no Fortran parsing. Generates all cap files and supporting modules. - -**Both tools import the same metadata parsing module.** No duplication of metadata -parsing logic between the two tools. - ---- - -## 3. Metadata Format - -### 3.1 File format - -The existing ini-file format is preserved unchanged. Every metadata file consists of -`[ccpp-table-properties]` header blocks followed by `[ccpp-arg-table]` variable listing -blocks, exactly as in the current framework. - -The `[ccpp-table-properties]` + `[ccpp-arg-table]` pair is redundant for non-scheme -tables (the distinction is vestigial for host/DDT/suite tables) but is preserved for -symmetry with scheme metadata tables. - -### 3.2 Table types (`type =` in `[ccpp-table-properties]`) - -Five table types are supported: - -| `type =` | Ownership | Import mechanism | -|---|---|---| -| `scheme` | Physics scheme | Intent args on scheme subroutines | -| `host` | Host model | Module USE (direct or via DDT member) | -| `control` | Framework runtime layer | Explicit args to `ccpp_physics_*` entry points | -| `suite` | Generated suite cap | Module USE of generated suite data module | -| `ddt` | Type definition | Structural — describes DDT fields, no instance info | - -Notes: -- `type = module` from capgen is renamed to `type = host`. Breaking change, intentional. -- `type = suite` tables are **written by the generator** (never hand-authored). They - appear on disk for inspection and debugging only. -- `type = ddt` describes the structure of a Fortran derived type. It contains no - instance information — only field definitions. - -### 3.3 Per-variable attributes - -All existing per-variable attributes are preserved: `standard_name`, `long_name`, -`units`, `dimensions`, `type`, `kind`, `intent`, `optional`, `active`, `protected`. - -`protected = True` means: any scheme that declares `intent` other than `in` for this -variable is a metadata error, caught at generation time. This is how constants are -handled — a constants DDT is declared `type = host` with all fields `protected = True`. -No separate `type = constants` is needed. - -### 3.4 DDT type definitions - -A DDT type definition uses `type = ddt`: - -```ini -[ccpp-table-properties] - name = gfs_statein_type - type = ddt - -[ccpp-arg-table] - name = gfs_statein_type - type = ddt - -[phii] - standard_name = geopotential_at_interface - long_name = geopotential at model layer interfaces - units = m2 s-2 - dimensions = (horizontal_dimension, vertical_interface_dimension) - type = real - kind = kind_phys -``` - -### 3.5 Table-level properties - -The `[ccpp-table-properties]` block supports the following table-level keys beyond `name` -and `type`: - -| Key | Applies to | Purpose | -|---|---|---| -| `source_path` | `scheme` | Relative path from the `.meta` file directory to the directory containing the corresponding Fortran `.F90` source file. Defaults to the `.meta` file's own directory if absent. Used by the validator for auto-discovery of Fortran source. | -| `dependencies` | `scheme`, `host` | Comma-separated list of dependency file names or relative paths. Resolved to absolute paths using `dependencies_path` as a base directory (or the `.meta` file's directory if `dependencies_path` is absent). | -| `dependencies_path` | `scheme`, `host` | Optional subdirectory (relative to the `.meta` file's directory) used as the base when resolving entries in `dependencies`. Has no effect if `dependencies` is absent or `none`. | - -Example: - -```ini -[ccpp-table-properties] - name = my_scheme - type = scheme - source_path = ../src - dependencies_path = ../deps - dependencies = utility_module.F90, shared_constants.F90 -``` - -The resolved `dependencies` paths are collected across all scheme tables and written to the -`` section of `datatable.xml`. The validator uses `source_path` (not -`dependencies`) for locating the Fortran `.F90` corresponding to each `.meta` file. - -**Parser implementation note:** The INI parser applies these table-level properties to -the `MetadataTable` object before transitioning to any `[ccpp-arg-table]` section. A -`flush_table_props()` call must happen at every parser-state transition (new table -header, first arg-table header, end-of-file) to avoid silently discarding the properties. - -### 3.6 DDT instances - -A DDT instance is declared as a regular variable entry inside a `type = host` table. -The enclosing `[ccpp-table-properties]` block's `name` attribute identifies the Fortran -module from which the instance is imported via `use`. No separate `module` attribute is -needed on the variable entry. - -```ini -[ccpp-table-properties] - name = CCPP_data - type = host - dependencies = CCPP_typedefs.F90,GFS_typedefs.F90 - -[ccpp-arg-table] - name = CCPP_data - type = host - -[gfs_statein] - standard_name = gfs_statein - long_name = GFS state input for all instances - units = mixed - dimensions = (number_of_instances) - type = gfs_statein_type -``` - -The generator looks up the `gfs_statein_type` DDT table, traverses its fields, and -constructs access paths of the form -`gfs_statein(instance_number)%fieldname(loop_begin:loop_end, 1:nlevs)`. - -For scalar DDT instances (no instance dimension), dimensions is `()`. -For nested DDTs, the same mechanism applies recursively. - -### 3.7 Control variable declarations - -Control variables are declared in a `type = control` table. The generator resolves -each control variable by its standard name and uses whatever local Fortran name the -host declared: - -```ini -[ccpp-table-properties] - name = my_host_control_module - type = control - -[ccpp-arg-table] - name = my_host_control_module - type = control - -[loop_begin] - standard_name = horizontal_loop_begin - long_name = start of horizontal loop - units = index - dimensions = () - type = integer -``` - ---- - -## 4. Control Variables - -The generator recognizes the following standard names for control variables. Local -Fortran names are host-defined (resolved from the `type = control` metadata table). - -### 4.1 Entry point arguments (non-register phases) - -All required control variables are unconditional — every host must declare all of them. -Models that don't use a variable pass the neutral value: `1` for integers, `''` for -character arguments. - -| Standard name | Expected type | Role | -|---|---|---| -| `suite_name` | `character` | Suite name for runtime dispatch | -| `horizontal_loop_begin` | `integer` | Start of horizontal slice (chunk bounds for `ccpp_physics_run`; `1` for all other phases) | -| `horizontal_loop_end` | `integer` | End of horizontal slice (chunk bounds for `ccpp_physics_run`; `ncols` for all other phases) | -| `thread_number` | `integer` | Current thread index (1..number_of_threads); pass `1` if single-threaded | -| `number_of_threads` | `integer` | Host blocking loop thread count; pass `1` if single-threaded | -| `number_of_physics_threads` | `integer` | Thread budget for physics-internal OpenMP; pass `1` if none | -| `ccpp_error_message` | `character` | Error message string | -| `ccpp_error_code` | `integer` | Integer error return code | - -`instance_number` is **paired-optional** with `number_of_instances` (host -table, §3.6): declare both for a multi-instance API, declare neither for a -single-instance API. Declaring exactly one is a hard error. When the pair -is absent, the static API signatures drop the `instance_number` argument -entirely and per-instance state arrays size to 1. - -`group_name` is **not** in the required set. It is included in the static API signature -only if the host declares it in their `type=control` table. When absent: the static API -calls all groups in declared order; no dispatch argument is generated; the generator -warns (not errors) if any loaded suite has more than one group. When present: it is a -required (non-optional) `character` argument; the value `''` (empty string) or `'all'` -calls all groups in order; any other value dispatches to the named group only. - -The generator validates the required set at startup (after host metadata is parsed): -every required standard name must be present with the expected Fortran type (rank-0 -scalar). All failures are collected and reported together before halting. - -### 4.2 Loop-generated control variables (subcycles only) - -| Standard name | Role | -|---|---| -| `ccpp_loop_counter` | Current subcycle iteration (1..ccpp_loop_extent) | -| `ccpp_loop_extent` | Total subcycle iterations; value comes from the `loop=N` attribute on the `` element in the suite XML definition file | - -These are **not** passed as `ccpp_physics_*` arguments from the host. They are set by -the generated `do` loop inside the group cap and are available to any scheme called -within that loop. Outside a subcycle loop, these variables are not in scope. - -### 4.3 Registered dimension standard names - -The generator has built-in semantic knowledge of these dimension standard names: - -| Standard name | Indexing semantic | -|---|---| -| `instance_dimension` | Scalar extraction: `var(instance_number)` — `instance_number` is the control variable | -| `horizontal_dimension` | **At scheme call sites**: always `horizontal_loop_begin:horizontal_loop_end` (using control variable local names), for all phases. **For suite-owned array allocation sizing**: local name of `horizontal_dimension` from the host `type=host` table (accessed via module USE, not the control variable). | -| `vertical_*` | Slice: `1:` | - -`horizontal_dimension` and `vertical_*` are "registered" because the generator knows -their slicing semantics, but they are resolved to local names the same way as arbitrary -dimensions — by looking up the variable with that standard name in the host metadata. -**Scheme metadata always uses `horizontal_dimension`** for the horizontal extent, regardless of which phase the entrypoint belongs to. The standard name `horizontal_loop_extent` does not exist in the new design. The distinction between a chunk call and a full-domain call is handled entirely by what the host passes for `horizontal_loop_begin` and `horizontal_loop_end` — invisible to scheme developers and to the cap generator's slicing logic. - -All other dimension standard names are resolved identically: look up the variable with -that standard name, get its local Fortran name, emit `1:local_name`. - -The timing of `instance_dimension` substitution — whether at parse time (when building -the flat dict access path) or at call-string generation time (like other registered -dimensions) — is an implementation decision left to the developer. Either is correct; -choose whichever is easier to implement, understand, and maintain. - ---- - -## 5. Entry Points - -Eight entry points are generated in the static API. Two tiers: - -### 5.1 Framework lifecycle (no group_name dispatch) - -These operate on the entire suite at once. They take `suite_name`, -`ccpp_error_code`, and `ccpp_error_message` (plus `instance_number` when the -host opts into the multi-instance pair, §4.1). No scheme `_run/_init/_final` -calls. - -| Entry point | Purpose | -|---|---| -| `ccpp_register(suite_name, errcode, errmsg, [instance_number])` | Calls each scheme's `_register` entrypoint; transitions suite state to `REGISTERED`. Auto-provisions `ccpp_model_constituents_obj(:)` and friends in `ccpp_host_constituents.F90` when any register-phase scheme declares `ccpp_constituent_properties_t(:)` (constituents are not a formal arg). | -| `ccpp_init(suite_name, errcode, errmsg, [instance_number])` | Allocates integer state arrays in all group caps; allocates suite-owned interstitial data; calls the suite-level `` scheme if declared (§5.5); no per-group scheme calls. | -| `ccpp_final(suite_name, errcode, errmsg, [instance_number])` | Calls the suite-level `` scheme if declared (§5.5); deallocates integer state arrays and suite-owned data; no per-group scheme calls. | - -Constituents are **opt-in**: a separate generated module -`ccpp_host_constituents.F90` declares `ccpp_model_constituents_obj(:)` and a -host-facing API (`ccpp_register_constituents`, `ccpp_initialize_constituents`, -`ccpp_const_get_index`, `ccpp_constituents_array(instance_number)`, -`ccpp_advected_constituents_array`, `ccpp_model_const_properties`, -`ccpp_number_constituents`, `ccpp_gather_constituents`, -`ccpp_update_constituents`, `ccpp_is_scheme_constituent`). The host calls -these directly — they are not formal arguments of `ccpp_register` / `ccpp_init`. -See `doc/constituents.md`. - -`instance_number` appears in every framework-lifecycle signature only when -the host declares the `instance_number` / `number_of_instances` pair (§4.1). -When present it propagates: `ccpp_init` → `_init` → each group's -`state_alloc(number_of_instances, ...)`. - -### 5.2 Physics group invocation (dispatched by suite_name + group_name) - -| Entry point | Calls scheme phase | -|---|---| -| `ccpp_physics_init(...)` | `_init` | -| `ccpp_physics_timestep_init(...)` | `_timestep_init` | -| `ccpp_physics_run(...)` | `_run` | -| `ccpp_physics_timestep_final(...)` | `_timestep_final` | -| `ccpp_physics_final(...)` | `_final` | - -All five take the full required control variable argument list (Section 4.1) — a uniform -signature across all phases. If `group_name` is declared in the host's `type=control` -table, it is also included; `''` or `'all'` calls all groups in order, any other value -dispatches to the named group only. If `group_name` is absent from the control table, -no dispatch argument is generated and all groups are called in order. - -The host is responsible for passing appropriate horizontal bounds: actual chunk bounds -for `ccpp_physics_run`; `1` and `ncols` (full domain) for all other phases. The cap -always uses `(horizontal_loop_begin:horizontal_loop_end)` for array slices — no -phase-specific special-casing. - -### 5.3 Naming note - -`finalize` is renamed to `final` throughout (e.g., `ccpp_physics_final`, not -`ccpp_physics_finalize`). Breaking change, intentional for symmetry: -`ccpp_init`/`ccpp_final`, `ccpp_physics_init`/`ccpp_physics_final`, -`ccpp_physics_timestep_init`/`ccpp_physics_timestep_final`. - -The SDF likewise accepts only the canonical short element names: -`` and `` (§5.5). The legacy long spellings — `` -(old typo), `` (correct long form), `` — are -rejected at parse time with a clear error pointing at the short form. - -### 5.5 Suite-level lifecycle hooks (`` / ``) - -The SDF root may declare a **single** scheme that runs at suite-init -and/or suite-final time: - -```xml - - my_init_scheme - … - my_final_scheme - -``` - -The named scheme's `init` (resp. `final`) phase is resolved from the -scheme metadata and called from inside `_init` (resp. -`_final`). Ordering: - -- `_init`: after all group `state_alloc` and - `suite_data_init_fields`, **before** the `CCPP_SUITE_FRAMEWORK_INITIALIZED` - state transition. An errflg from the init scheme prevents the - state transition. -- `_final`: before the `CCPP_SUITE_UNREGISTERED` transition. - -Constraints: - -- One scheme per `` / ``. Multiple `` children - inside (the "group" shape) is a schema violation. -- The named scheme must have the matching phase in its metadata. - Missing-phase metadata is a generator error. - -### 5.4 Suite introspection routines - -In addition to the eight entry points above, the static API exposes **five** -suite-introspection subroutines that let a host query, at runtime, what is -compiled into the API. These mirror the equivalent routines in the original -capgen (`scripts/ccpp_suite.py` — `write_inspection_routines`) and are -used by CMake integration and host-side build glue. - -| Entry point | Purpose | -|---|---| -| `ccpp_physics_suite_list(suites)` | Return all suite names compiled into the API | -| `ccpp_physics_suite_part_list(suite_name, part_list, errmsg, errflg)` | Return the list of group ("part") names for a given suite | -| `ccpp_physics_suite_schemes(suite_name, scheme_list, errmsg, errflg)` | Return the list of scheme module names that compose a suite | -| `ccpp_physics_suite_variables(suite_name, variable_list, errmsg, errflg, [input_vars], [output_vars], [struct_elements])` | Standard-name list a suite consumes/produces; optional flags filter by intent and whether DDT sub-fields are flattened | -| `ccpp_physics_suite_host_data(suite_name, variable_list, errmsg, errflg)` | Standard-name list of host data the suite reads — DDT-collapsed view, excludes generated control variables | - -These routines do not advance the state machine and do not call any scheme -entrypoints. All inputs derive from generator-time data already held in -`SuiteResolution` plus the host/scheme metadata; no new metadata is required. -The `_variables` vs `_host_data` split distinguishes the flat-leaf view -(every DDT field that is actually consumed) from the DDT-collapsed view -(parent DDT instances), and excludes capgen-ng-generated control -variables from `_host_data` since the host owns those. - ---- - -## 6. Cap Hierarchy - -All three levels are fully auto-generated. No hand-written components in the cap layer. - -### 6.1 Static API (`ccpp_static_api.F90`) - -- Imports all host DDTs and flat fields via `module use` (resolved from host metadata) -- Does not USE `ccpp_kinds` directly: the static API has no kind-typed declarations of - its own (it dispatches by `suite_name` and forwards control args). `ccpp_kinds` is - USEd only by files that declare kind-typed variables: group caps, the suite types - module, and the suite data module. -- Dispatches all eight entry points by `suite_name` to the appropriate suite cap -- Does not own constituent state; constituents are accessed via the separate - `ccpp_host_constituents.F90` module by both the host and group caps -- Holds no physics state - -### 6.2 Suite cap (`ccpp__cap.F90`) - -- Imports the generated suite data module (`ccpp__data.F90`) -- Contains the suite-level integer state array: `integer, allocatable :: ccpp_suite_state(:)` indexed by instance -- Implements the suite-level state machine (see Section 7) -- On `ccpp_register`: calls all scheme `_register` entrypoints across the suite -- On `_init(number_of_instances, errmsg, errflg)`: calls `state_alloc` for every - group, passing `number_of_instances` (or literal `1` for single-instance hosts); - also allocates suite-owned interstitial data. The `number_of_instances` argument is - conditional on the host declaring it (Section 7.2.1). -- On `ccpp_final`: deallocates all of the above -- Routes `ccpp_physics_*` calls by `group_name` to the appropriate group cap function; - passes `instance_number` (if present) through to group cap `_init` and `_final` subs - -### 6.3 Group cap (`ccpp___cap.F90`) - -- Imports `ccpp__data` (suite-owned interstitial data) -- Imports `ccpp__types` (shared wrapper types) -- Contains the group-level integer state array: `integer, allocatable :: ccpp_group_state(:)` indexed by instance -- Implements the group-level state machine (see Section 7) -- Contains the actual scheme call sites for each phase: - - Loop bound locals - - Optional variable pointer arrays (thread-dimensioned) - - Fixed-index extraction locals - - Unit/kind conversion locals - - Subcycle `do` loops - - Scheme calls with full argument lists - ---- - -## 7. State Machine - -Integer state parameters are defined as **private named parameters directly inside each -generated group cap module** — they are NOT imported from a shared framework library -module. Each group cap file declares: - -```fortran -integer, parameter, private :: CCPP_GROUP_UNINITIALIZED = 0 -integer, parameter, private :: CCPP_GROUP_INITIALIZED = 1 -integer, parameter, private :: CCPP_GROUP_IN_TIMESTEP = 2 -``` - -This means the integer values are replicated across generated files (acceptable — the -names are the contract, not the values). No generated file USEs a framework state module. - -Two levels, both indexed by `instance_number`. - -### 7.1 Suite-level state (in suite cap) - -```fortran -integer, parameter :: CCPP_SUITE_UNREGISTERED = 0 -integer, parameter :: CCPP_SUITE_REGISTERED = 1 -integer, parameter :: CCPP_SUITE_FRAMEWORK_INITIALIZED = 2 -``` - -| Entry point | Required state | State after | -|---|---|---| -| `ccpp_register` | `== UNREGISTERED` | `REGISTERED` | -| `ccpp_init` | `== REGISTERED` | `FRAMEWORK_INITIALIZED` | -| `ccpp_physics_*` | `== FRAMEWORK_INITIALIZED` | (unchanged) | -| `ccpp_final` | `== FRAMEWORK_INITIALIZED` | `REGISTERED` | - -### 7.2 Group-level state (in each group cap) - -```fortran -integer, parameter :: CCPP_GROUP_UNINITIALIZED = 0 -integer, parameter :: CCPP_GROUP_INITIALIZED = 1 -integer, parameter :: CCPP_GROUP_IN_TIMESTEP = 2 -``` - -| Entry point | Required state | State after | -|---|---|---| -| `ccpp_physics_init` | `< INITIALIZED` (idempotent if `== INITIALIZED`) | `INITIALIZED` | -| `ccpp_physics_timestep_init` | `== INITIALIZED` | `IN_TIMESTEP` | -| `ccpp_physics_run` | `== IN_TIMESTEP` | `IN_TIMESTEP` | -| `ccpp_physics_timestep_final` | `== IN_TIMESTEP` | `INITIALIZED` | -| `ccpp_physics_final` | `>= INITIALIZED` | `UNINITIALIZED` | - -The idempotency rule for `ccpp_physics_init`: if the group is already in state -`INITIALIZED`, return immediately without calling any scheme `_init` routines. This -allows the host to call `ccpp_physics_init` multiple times safely. Any further call -after the first must result in no change (idempotency is a scheme contract). - -#### 7.2.1 State array allocation and instance indexing - -Each group cap declares an allocatable module-level array: - -```fortran -integer, private, allocatable :: ccpp_group_state(:) -``` - -Two generated subroutines manage it: - -```fortran -! Always takes number_of_instances as an explicit arg — never USEs a host module. -subroutine ccpp___state_alloc(number_of_instances, errmsg, errflg) - integer, intent(in) :: number_of_instances - ... - allocate(ccpp_group_state(number_of_instances)) - ccpp_group_state(:) = CCPP_GROUP_UNINITIALIZED - -subroutine ccpp___state_dealloc(errmsg, errflg) - ... - if (allocated(ccpp_group_state)) deallocate(ccpp_group_state) -``` - -`state_alloc` is called from the suite cap's `_init` subroutine. The count is -passed as an explicit argument: the local name of `number_of_instances` from host -metadata (multi-instance), or the integer literal `1` (single-instance): - -```fortran -! Multi-instance (host provides number_of_instances with local name ninstances): -subroutine test_suite_init(ninstances, errmsg, errflg) - integer, intent(in) :: ninstances - ... - call ccpp_test_suite_physics_state_alloc(ninstances, errmsg, errflg) - -! Single-instance (no number_of_instances in host metadata): -subroutine test_suite_init(errmsg, errflg) - ... - call ccpp_test_suite_physics_state_alloc(1, errmsg, errflg) -``` - -State array **indexing** in the phase subroutines uses the local name of -`instance_number` (e.g. `inst_num`) when the host provides it, otherwise the literal -`1`: - -```fortran -subroutine ccpp___init(inst_num, ...) - if (ccpp_group_state(inst_num) >= CCPP_GROUP_INITIALIZED) return - ... - ccpp_group_state(inst_num) = CCPP_GROUP_INITIALIZED -``` - -`instance_number` is injected into the `_init` and `_final` phase subroutine signatures -even when no scheme in those phases uses it directly — the state guard and state -transition require it. It does **not** appear in `_run`, `_timestep_init`, or -`_timestep_final` unless a scheme in those phases explicitly requests it. - -These two integer arrays replace both the boolean `initialized(:)` array from prebuild -and the string-based `ccpp_suite_state` from CAM-SIMA. - ---- - -## 8. Scheme Metadata and Variable Matching - -### 8.1 Scheme metadata structure - -Each scheme source file has a companion `.meta` file with `type = scheme` tables — one -table per public phase subroutine (`scheme_name_init`, `scheme_name_run`, -`scheme_name_timestep_init`, etc.). The section header for each variable entry is the -**local variable name** as it appears in the scheme's Fortran subroutine argument list. - -The internal metadata store is keyed as: - -``` -metadata[scheme_name][phase][standard_name] → {local_name, units, kind, dimensions, - intent, optional, active, ...} -``` - -Keying by `scheme_name` then `phase` enables cross-phase queries ("does this scheme -have a register phase?", "what are all variables of scheme X?") and matches the -conceptual model of the suite XML. - -### 8.2 Reading order - -All metadata files (host + scheme + DDT) are read in one pass without resolving DDT -type references. After the full read, the generator builds the known DDT list, then -resolves all type references. This avoids ordering dependencies between metadata files. - -### 8.3 Known DDT list - -After reading all metadata, the generator assembles the set of known DDT types from -`type = ddt` tables. Two categories: - -**Framework-defined DDTs**: declared in `type = ddt` metadata tables. The generator -knows their fields, dimensions, and access paths. - -**External DDTs**: types from external libraries (MPI, ESMF, etc.) that the generator -cannot introspect. These are declared in variable entries using an extended `type` -syntax: - -```ini -[mycomm] - standard_name = mpi_communicator - type = external:mpi_f08:mpi_comm - ... -``` - -The format is `external::`. The generator emits -`use mpi_f08, only: mpi_comm` and treats the variable as an opaque type — no field -traversal, no dimension indexing beyond what the metadata declares. - -### 8.4 Variable matching: scheme vs. host - -For each argument in a scheme's phase function (looked up by standard name): - -1. **Found in host+control flat dict** → use the resolved access path. If `units` or - `kind` differ from what the scheme declares, generate a transformation (Section 9). -2. **Not found, first use is `intent(out)`** → suite-owned variable. Add to suite data, - generate declaration in `ccpp__data.F90`. -3. **Not found, first use is `intent(in)` or `intent(inout)`** → **error**: variable - used before it is provided by any scheme or host. -4. **Found in suite data (from a prior scheme)** → use the suite data access path. - Apply transformation if needed. - -### 8.5 Cap call argument construction - -The generator builds the argument list for each scheme call from the scheme's metadata -argument order. For each argument: - -- **Direct pass-through** (no transformation, not optional): inline host access - expression — no local variable declared -- **Transformation**: cap-local temporary named after the scheme's local variable name - (from scheme metadata section header); see Section 10.2 for naming rules -- **Optional**: cap-local pointer array named `_p`; see Section 10.3 -- **Optional + transformation**: combined in the `if (active) then` block - -The generator does not parse Fortran source. All local names, types, kinds, dimensions, -and intents come exclusively from metadata. - ---- - -## 9. Variable Resolution and Access Path Construction - -### 9.1 Flat storage model (host+control+suite) - -The generator flattens the DDT hierarchy at parse time. After parsing, all host, -control, and suite variables are stored in a flat dictionary keyed by standard name. -Each entry contains: -- The Fortran local name -- The fully-qualified access path (e.g., `gfs_statein(instance_number)%phii`) -- The module to USE -- Dimension information with registered/arbitrary classification - -The DDT hierarchy is discarded after the flat dict is built. No live DDT object tree -is maintained during code generation. - -### 9.2 Access path construction - -For each variable, the generator constructs the call-site expression by applying -dimension rules to each dimension in order: - -1. **`instance_dimension`** → substitute `instance_number` (scalar extraction) -2. **`horizontal_dimension`** → always substitute `horizontal_loop_begin:horizontal_loop_end` - (using control variable local names) at scheme call sites. For suite-owned array - allocation sizing, `horizontal_dimension` from the host `type=host` table is used directly. -3. **`vertical_*`** → substitute `1:local_vertical_dimension` -4. **Arbitrary dimension** → resolve to local name via its own metadata entry, emit - `1:local_name` -5. **`active` condition** → generate optional pointer-association guard (see Section 10.3) - -### 9.3 Module USE - -For each variable used in a group cap, the generator emits a `use module, only: varname` -statement. The module name comes from the enclosing `[ccpp-table-properties]` block name. -For suite-owned variables, the module is the generated `ccpp__data`. - -### 9.4 Eliminating TYPEDEFS_NEW_METADATA - -The manually-maintained `TYPEDEFS_NEW_METADATA` Python dict from prebuild is eliminated. -All information previously in that dict is now in metadata: -- The DDT type structure → `type = ddt` table -- The module-level instance → variable entry in the `type = host` table, module - implied by enclosing table name - ---- - -## 10. Variable Transformations and Optional Variables - -Variable transformations (unit/kind conversions) and optional variable handling are -combined — both occur within the same `if (active) then` block when a variable is -optional. - -### 10.1 Supported transformations - -- **Unit conversions** (e.g., Pa → hPa): formula from built-in conversion table (shared - with validator), keyed on source/target unit pair from metadata `units` attribute -- **Kind conversions** (e.g., r8 → kind_phys): from `kind` metadata attribute comparison - -The transformation framework is generic and pluggable — additional transformation types -(e.g., vertical flipping) can be added without restructuring the generator. - -### 10.2 Local variable naming - -The local variable name for a transformation temporary or optional pointer is derived -from the **scheme's local variable name** as declared in the scheme's metadata section -header (e.g., `[phii]` → local name is `phii`). The generator has this name without -parsing any Fortran. - -- Transformation temporary: scheme's local name + `_l` (e.g., `phii_l`) -- Optional pointer: scheme's local name + `_p` (e.g., `phii_p`) -- Conflict resolution: if two schemes in the same group cap use the same local name for - different standard names, append a numeric suffix before the suffix (e.g., `phii_2_l`) - -The generator validates that all generated local variable names and generated subroutine -names stay within Fortran's 63-character identifier limit. Violations are code-generation -errors — the developer must use a shorter local name in their metadata. - -The `active` expression in metadata is a Fortran logical expression written using -**CCPP standard names** (not local names). The generator translates all standard names -in the expression to their local Fortran names before emitting. - -Transformations **always** use a local temporary variable. The host variable is never -modified in-place — required for bit-for-bit reproducibility and to leave host data -uncorrupted if an exception occurs. Every conversion line carries an inline Fortran -comment (e.g., `! unit conversion: Pa to hPa`). Transformation mismatches (unknown -unit pair, unknown kind pair) are code-generation errors — no stdout/stderr from -generated code. - -### 10.3 The four cases - -The generator handles exactly four combinations per variable: - -**Case 1: No pointer, no transformation** (not optional, no unit/kind mismatch) - -No local variable is declared. The host access expression is used inline at the call -site: -```fortran -call scheme_run(..., gfs_statein(instance_number)%phii(lb:ub,1:nlevs), ...) -``` - -**Case 2: Pointer only** (optional, no transformation) - -Pointer array declared at function top; conditional association in `if (active)` block: -```fortran -! declaration: -type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) - -! before call: -if () then - phii_p(thread_number)%ptr => gfs_statein(instance_number)%phii(lb:ub,1:nlevs) -else - nullify(phii_p(thread_number)%ptr) -end if - -call scheme_run(..., phii_p(thread_number)%ptr, ...) - -! after call: -nullify(phii_p(thread_number)%ptr) -``` - -**Case 3: Transformation only** (not optional, unit/kind mismatch) - -Local temporary `phii_l` (scheme's local name + `_l`); intent-driven emission: - -| `intent` | Pre-call | Post-call | -|---|---|---| -| `in` | `phii_l = host_phii(...) * factor ! unit conversion: X to Y` | nothing | -| `out` | nothing | `host_phii(...) = phii_l / factor ! unit conversion: Y to X` | -| `inout` | pre-call as above | post-call as above | - -```fortran -! declaration: -real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) ! or appropriate rank/kind - -! before call (intent in/inout): -phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa - -call scheme_run(..., phii_l, ...) - -! after call (intent inout/out): -gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa -``` - -**Case 4: Pointer and transformation** (optional + unit/kind mismatch) - -Two local variables: `phii_l` (transformation temporary) and `phii_p` (pointer array). -Sequence: (1) apply transformation to `phii_l` depending on intent, (2) assign pointer -to `phii_l`, (3) call scheme, (4) nullify pointer, (5) apply back-transformation from -`phii_l` to host depending on intent. All within the `if (active)` block: - -```fortran -! declarations: -real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) -type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) - -! before call: -if () then - ! step 1: apply forward transformation (intent in/inout) - phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa - ! step 2: assign pointer to transformed local - phii_p(thread_number)%ptr => phii_l -else - nullify(phii_p(thread_number)%ptr) -end if - -call scheme_run(..., phii_p(thread_number)%ptr, ...) - -! after call: -if () then - ! step 4: nullify pointer - nullify(phii_p(thread_number)%ptr) - ! step 5: apply back-transformation (intent inout/out) - gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa -end if -``` - -The wrapper types (`real_kind_phys_rank1_ptr_type` etc.) are defined once in the -generated shared types module (`ccpp__types.F90`), not re-declared inside every -group cap function. Passing an unassociated pointer is safe under all compiler modes. - ---- - -## 11. Subcycle Loops - -When a group in the suite XML contains ``, the generator emits a -Fortran `do` loop in the group cap: - -```fortran -do = 1, N ! subcycle: N iterations from suite XML - ! ... scheme calls ... -end do -``` - -`ccpp_loop_counter` and `ccpp_loop_extent` are **loop-context variables** — a special -class that does not fit any of the five table types: -- NOT `type = control`: not passed as `ccpp_physics_*` arguments from the host -- NOT `type = host`: not from host module USE -- NOT `type = suite`: not persistent allocated data - -The generator has built-in knowledge of these two standard names. `ccpp_loop_extent` -value comes from the `loop=N` attribute in the suite XML definition file. -`ccpp_loop_counter` is the do loop induction variable. Both exist only within the -generated loop scope — they are not in scope outside a subcycle block. - -Any scheme that requests `ccpp_loop_counter` or `ccpp_loop_extent` by standard name -receives the loop variables at the call site. Their local names in the cap are derived -from the scheme's metadata section headers as for any other variable (Section 10.2). - ---- - -## 12. Init/Finalize Deduplication - -If the same scheme appears more than once within a single group (e.g., via subcycles), -having its `_init` called multiple times is a **code generator bug** — the generator -**errors out** rather than silently deduplicating. The suite XML must not list the same -scheme multiple times in the same group for non-run phases. - -If the same scheme appears in multiple groups, its `_init` is called once per group. -This is acceptable — idempotency is a contract all scheme `_init` routines must satisfy. - -The same rule applies to `_timestep_init`, `_timestep_final`, and `_final`. - ---- - -## 13. Suite-Owned Data - -### 13.1 Discovery - -The generator identifies suite-owned variables during variable resolution: variables -requested by schemes that are not satisfied by host metadata (`type = host` or -`type = control`). - -**Error condition**: if a variable is determined to be suite-owned (not provided by the -host) and the first scheme that uses it does not have it as `intent(out)`, the generator -errors out. A suite-owned variable that is first read before it is written would be -used uninitialized. - -### 13.2 Generated files and allocation - -Suite-owned variables are declared in a generated suite data module -(`ccpp__data.F90`) as fields of a Fortran DDT. The suite cap and all group caps -`use` this module. - -Allocation happens in `ccpp_init` (suite cap). Deallocation happens in `ccpp_final`. -Suite-owned arrays are allocated for the **full `horizontal_dimension`** — threads -access their respective horizontal chunk (`horizontal_loop_begin:horizontal_loop_end`) -at scheme call sites. This avoids per-call allocation overhead. If this proves to -consume too much memory at scale, a future revision may move allocation to per-phase -with chunk-sized arrays; start with the full-dimension approach. - -Subsetting (applying horizontal loop bounds, instance index) happens at scheme call -sites in the group cap, not in the suite cap. - -### 13.3 Metadata - -The generator also writes a `type = suite` metadata table (`ccpp_.meta`) as a -byproduct. This file is for inspection and debugging — it is not consumed by the -generator on subsequent runs. - ---- - -## 14. Constituent API - -> **Status (2026-05-12).** The constituent API in capgen-ng has evolved past -> the sketch below. The current implementation is: -> -> - One `ccpp_model_constituents_obj(:)` array (sized to -> `number_of_instances`), declared and owned by the **generator** in -> `ccpp_host_constituents.F90` — not by the host. -> - Host-facing API: `ccpp_register_constituents(host_constituents, -> instance_number, ...)`, `ccpp_initialize_constituents`, -> `ccpp_number_constituents`, `ccpp_const_get_index`, -> `ccpp_constituents_array(instance_number)`, etc. All per-instance. -> - Schemes follow four rules: register-phase -> `ccpp_constituent_properties_t(:), intent=out, allocatable`; -> physics-phase consume via `advected=true intent=in/inout`; tendency -> produce via `constituent=true intent=out` + `tendency_of_` -> std_name; mismatched combos are codegen errors. -> - **Authoritative reference**: `doc/constituents.md` (full lifecycle + -> API + examples). -> - **Architecture review and proposed reforms**: `doc/constituents_overhaul.md` -> (2026-05-12, meeting-quality discussion of original capgen vs -> capgen-ng vs cam-sima needs, bugs/flaws, class-A/B property -> classification, three proposals A/B/C). -> -> The historic text below is retained for context but does not describe -> the live system. - -### 14.1 Type definition (historic) - -`ccpp_model_constituents_t` is unchanged from CAM-SIMA. The type definition lives in -the framework library (`ccpp_constituent_prop_mod`), not in generated code. - -### 14.2 Ownership and lifecycle (historic — superseded) - -The **host model** declares and owns the constituent object: - -```fortran -use ccpp_constituent_prop_mod, only: ccpp_model_constituents_t -type(ccpp_model_constituents_t) :: constituents ! unallocated initially -``` - -The host passes it to `ccpp_register`, which allocates and populates it. After -`ccpp_register` returns, the host holds a fully allocated object ready for `lock_table`, -`const_index`, `copy_in`, `copy_out`. - -The `constituents` argument to `ccpp_register` is mandatory. - -### 14.3 Register phase mechanics (historic — superseded) - -The suite cap's register routine: -1. Iterates over all constituent-providing schemes in the suite -2. Calls each scheme's `_register` entrypoint, which returns a - `ccpp_constituent_properties_t` array -3. Collects these arrays and populates the constituent object - -No `group_name` dispatch is needed for register — it operates on the whole suite. - ---- - -## 15. Generated Output Files - -All files are written to `--output-root`. - -| File | Contents | -|---|---| -| `ccpp_kinds.F90` | Kind parameter definitions. **Always generated.** Re-exports specs from `iso_fortran_env` (default) or host-supplied modules as `integer, parameter, public :: = `. If no `--kind-type` is supplied, `kind_phys=iso_fortran_env:REAL64` is injected automatically (logged at INFO). | -| `ccpp_static_api.F90` | Static API — host imports, suite_name dispatch | -| `ccpp__cap.F90` | Suite cap — suite data import, state machine, group dispatch | -| `ccpp___cap.F90` | Group cap — scheme call sites, state array, optionals, transformations. USEs `ccpp_kinds` for any kind referenced in transformation temporaries. | -| `ccpp__data.F90` | Suite data module — framework-owned interstitial DDT. USEs `ccpp_kinds` for any kind referenced in suite-var declarations. | -| `ccpp__types.F90` | Shared cap types — optional pointer wrapper types, transformation locals. USEs `ccpp_kinds` for any kind referenced in pointer wrappers. | -| `ccpp_.meta` | Generated `type = suite` metadata table (output-only, for inspection) | -| `datatable.xml` | Generator database for `ccpp_datafile.py` queries. `ccpp_kinds.F90` and `ccpp_static_api.F90` appear in `...` (matches original capgen). | - -`ccpp_kinds.F90` is a dependency of all generated Fortran files that reference any kind parameter (group cap, suite types, suite data). The static API and suite cap have no kind references and do not USE it. - ---- - -## 16. CLI Invocation - -``` -ccpp_capgen_ng.py - --host-name - --host-files - --scheme-files - --suites - --output-root - --kind-type NAME=[MODULE:]SPEC # repeatable, see § 16.1 - --verbose # once = info; twice = debug -``` - -The generator also supports programmatic Python invocation (import and call directly), -using the same internal code paths as the CLI. - -### 16.1 Kind specifications - -Each `--kind-type` maps a CCPP-visible kind name to a Fortran precision constant. Syntax: - -``` ---kind-type =[:] -``` - -* `` — kind name as published in `ccpp_kinds` and referenced in scheme metadata - (e.g. `kind_phys`). -* `` — name of a precision constant (kind parameter) defined in some Fortran - module. -* `` — Fortran module that defines ``. **Optional**: when omitted, - `` must be a standard `ISO_FORTRAN_ENV` constant (`REAL32`, `REAL64`, `INT32`, - ...) and the module defaults to `iso_fortran_env`. If `` is not a known ISO - constant, omitting `` is an error. - -Examples: - -* `--kind-type kind_phys=REAL64` → - `use iso_fortran_env, only: REAL64; integer, parameter, public :: kind_phys = REAL64` -* `--kind-type kind_phys=my_host_kinds:kind_r8` → - `use my_host_kinds, only: kind_r8; integer, parameter, public :: kind_phys = kind_r8` - -The flag may be specified multiple times. `ccpp_kinds.F90` is **always generated**. If -no `--kind-type` is supplied (or `kind_phys` is omitted from a non-empty list), the -generator injects `kind_phys=iso_fortran_env:REAL64` and logs an INFO message. - -### 16.2 datatable.xml and ccpp_datafile.py - -The generator emits `datatable.xml` encoding the full relationships between suites, -groups, schemes, and variables. A separate query utility (`ccpp_datafile.py`) provides -a rich query interface used by CMake and other build systems. The full query surface of -the existing `ccpp_datafile.py` is preserved — the simplified `--ccpp-files` query is -one of many. - -The XML structure is: - -```xml - - - - /abs/path/ccpp_kinds.F90 - /abs/path/ccpp_static_api.F90 - - - /abs/path/ccpp__cap.F90 - ... - - - - - - - - ... - - - - - ... - - - - - - - ... - - - - - - /abs/path/to/dep.F90 - ... - - -``` - -The `` section is populated from the `dependencies` table-level property -of all scheme metadata files (Section 3.5). Paths are resolved to absolute paths at -generation time, then sorted and deduplicated before writing. - -### 16.3 CMake integration pattern - -The generator runs at CMake configure time via `execute_process`. Generated sources are -discovered by querying `ccpp_datafile.py --ccpp-files`. Host and scheme Fortran sources -are found by replacing `.meta` with `.F90` (same base name convention). - ---- - -## 17. Design Decisions Not Carried Forward - -The following patterns from prebuild or capgen are explicitly **not** carried forward: - -| Pattern | Reason | -|---|---| -| `ccpp_t` / `cdata` struct | Replaced by explicit named control variable arguments | -| `TYPEDEFS_NEW_METADATA` Python dict | Replaced by DDT instance declarations in metadata | -| String-based `ccpp_suite_state` | Replaced by integer state arrays with named parameters | -| Boolean `initialized(:)` array | Replaced by integer state arrays | -| Flat-field arguments to group caps | DDT arguments are used instead (as in prebuild) | -| Scope-chain variable promotion | Suite-owned variables explicitly discovered and declared | -| Fortran-vs-metadata validation in generator | Moved to standalone validator tool | -| Re-declaration of pointer wrapper types per function | Declared once in shared types module | -| `ccpp_physics_suite_init/finalize` | Replaced by `ccpp_init`/`ccpp_final` | -| `type = module` (capgen) | Renamed to `type = host` | -| `finalize` phase name | Renamed to `final` | -| Array size checks in caps | Not generated by default; rely on compiler bounds checking | -| Auto-clone of `is_constituent` scheme args into framework `%instantiate` calls | Replaced by explicit registration (host_constituents arg + register-phase `ccpp_constituent_properties_t`) | -| `ConstituentVarDict` synthetic scope between suite and host | Removed; constituents are a `source='constituent'` classification on `ResolvedArg` | - ---- - -## 18. Outstanding Work - -See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` -(deferred items) for the canonical list. Snapshot as of 2026-05-13: - -### Landed in the 2026-05-12 session - -- **`instance_number` / `number_of_instances` paired opt-in** — hosts - may omit both for a single-instance API. -- **Module-name override** — `module_name = ` on - `[ccpp-table-properties]` for scheme/host/ddt. -- **Vertical-flip transform** — `top_at_one` per-var attribute; - composes with unit/kind transforms. -- **Multiple `dependencies = …` lines** per `[ccpp-table-properties]`. -- **Sliced local names** with long subscript-token CCPP standard names - no longer trip the 63-char Fortran-id limit. -- **Unit normalisation** — `m2` ≡ `m+2` (and friends). -- **Subcycle bound = CCPP std name** — including DDT-component access - paths (`phys_state%num_subcycles`). -- **Nested ``** — preserved end-to-end as nested `do` loops. -- **Active-expression + subcycle bounds** included in introspection - inputs. -- **TARGET on `ccpp_suite_data(:)`** module-level array. -- **Group-state alloc idempotency** (matches suite-state alloc). -- **Framework PR**: `ccpt_deallocate` ownership tracking via - `framework_owns_me` flag. Backward-compatible. Landed in - capgen-ng's vendored framework copy; still needs upstream merge - to ccpp-framework + original ccpp-capgen. -- **Identity unit conversions** no longer emit misleading "unit - conversion: kind_phys to kind_phys" comment. -- **Improved duplicate-standard-name error** lists both colliding - access paths. -- **Suite-level `` / ``** SDF elements consumed: named - scheme's init/final phase emitted inside `_init` / - `_final`. Single scheme only; long-form spellings - (``, ``, ``) rejected. -- **Constituent resolver — host metadata wins**: hosts that declare - framework-named std_names (`ccpp_constituents`, `index_of_`, ...) - short-circuit capgen-ng's auto-provisioning so legacy hosts (GFS, - SCM) keep using their own short local names (e.g. `ntcw`) without - blowing Fortran's 63-char identifier limit. - -### Landed 2026-05-13 - -- **`--legacy-mode` shim** — transient parse-time rewrite of legacy - CCPP standard names (`horizontal_loop_extent` → - `horizontal_dimension`). Available on `ccpp_capgen_ng.py` and - `ccpp_validator.py`; loud banner at startup. Isolated in - `metadata/legacy_compat.py` and tagged `# legacy-compat:` for clean - removal once scheme metadata has been migrated. -- **`_FRAMEWORK_CONST_DIM_INPUTS` cleanup** — the hand-curated - frozenset in `generator/static_api.py` was removed; framework- - constituent dimension references now ride on a dedicated - `used_const_dim_std_names` field on `ResolvedArg`. -- **`active` expression case-folding** — mixed-case standard names - in `active = (...)` are now lowercased at parse time so they match - the canonical lowercase host_dict keys (Fortran is case-insensitive, - so embedded logical operators are unaffected). - -### Test status - -- **Unit tests**: 1127 passing (`python -m pytest unit-tests/`). -- **End-to-end tests**: `advection`, `unit_conv`, `nested_suite`, - `variable_transform` covered. Tree is off-limits for in-session - edits — user-driven. - -### Still deferred - -- **Constituents overhaul** — discussion doc at - `doc/constituents_overhaul.md` (2026-05-12). Three proposals on the - table (A bugfix-only / B class-A/B split + setters / C host-only - registration). Pending decision in upcoming meeting. -- **Framework setter additions** — `set_advected`, `set_diagnostic_name`, - `set_default_value`, possibly `set_mixing_ratio_type`. Coordinated with - the overhaul. -- **Validator host-metadata check** — `ccpp_validator.py` is currently - scheme-only; revisit after the e2e test suite settles. See - `project_validator_host_check_deferred.md` (memory). -- **Codegen-time scheme-registration cross-check** — new metadata attr - `registers_std_names = a, b, c` on register-phase tables; replaces - current runtime `int_unassigned` check with codegen-time error. -- **Suppress `ccpp_host_constituents.F90` when unused** — currently - emitted for every build; now *correct* (empty) for SCM-style hosts - thanks to the host-wins rule, but still dead code. -- **`--legacy-mode` shim removal** — transient; remove - `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and - every `# legacy-compat:` touchpoint when scheme metadata has - migrated. -- **Nested subcycle `ccpp_loop_counter` semantics**: a scheme inside a - nested subcycle requesting `ccpp_loop_counter` would get the - OUTERMOST counter, not the innermost. None of the cam-sima schemes - use this — revisit if a real scheme needs the innermost. -- **Python linter / formatter pass** — pick `ruff` and apply across - `capgen-ng/`. -- **Generated Fortran ↔ Codee formatter idempotency** — emitted `.F90` - must round-trip cleanly through the project's Codee formatter. -- **`fortran_to_metadata` developer utility** — bootstrap a `.meta` - skeleton from an existing `.F90` subroutine. - -### Where to find the migration summary - -`doc/migration.md` — user-facing single-page summary of metadata + SDF -+ host-Fortran requirements after all the above changes. Read it -first when porting a host model. diff --git a/doc/redesign_prompt_original_20260505T2044.md b/doc/redesign_prompt_original_20260505T2044.md deleted file mode 100644 index 8a3c029e..00000000 --- a/doc/redesign_prompt_original_20260505T2044.md +++ /dev/null @@ -1,814 +0,0 @@ -# CCPP Framework Code Generator — Redesign Specification - -## Purpose - -This document is a complete implementation specification for a new CCPP Framework code -generator (`ccpp-capgen-ng`). It supersedes both `ccpp-prebuild` and `ccpp-capgen`. An -implementer should be able to build the new generator from scratch using this document -alone, supplemented by the real-world examples in `redesign_analysis.md`. - ---- - -## 1. Background and Motivation - -The CCPP Framework couples host NWP models (UFS Weather Model, NEPTUNE, CCPP-SCM, -CAM-SIMA) to physics parameterization schemes by auto-generating Fortran interface -("cap") code. Two generators exist today: - -- **`ccpp-prebuild`** — simple, procedural Python; fast; DDT arguments; used in - production by UFS, NEPTUNE, SCM. Does not support framework-owned variables. -- **`ccpp-capgen`** — complex OO Python; flat-field arguments; used in CAM-SIMA. The - deep class hierarchy (`VarDictionary`, `VarCompatObj`, `CCPPDatabaseObj`, etc.) makes - it unmaintainable. Three developers spent considerable time trying to add DDT argument - passing and could not succeed. - -The redesign starts fresh, drawing lessons from both. The guiding principle is: -**simplicity of prebuild, feature set of capgen**. - -The primary failures that triggered the redesign: -1. capgen passes flat fields to group caps — infeasible at UFS/NEPTUNE scale (1200+ - variables), breaks under compiler debug flags for optional variables. -2. capgen's scope-chain variable promotion is the source of most complexity. -3. Nobody on the team fully understands capgen. - ---- - -## 2. Toolchain Structure - -The redesign produces **two separate tools** that share the same metadata parsing library: - -### 2.1 Validator (`ccpp_validator.py`) - -Parses both Fortran source files and metadata files, compares them, and reports -discrepancies. Run by developers before invoking the generator — e.g., during scheme -development or in CI. Does **not** generate any Fortran output. - -### 2.2 Code Generator (`ccpp_capgen_ng.py`) - -Parses metadata only. Assumes metadata correctly describes the Fortran source — performs -no Fortran parsing. Generates all cap files and supporting modules. - -**Both tools import the same metadata parsing module.** No duplication of metadata -parsing logic between the two tools. - ---- - -## 3. Metadata Format - -### 3.1 File format - -The existing ini-file format is preserved unchanged. Every metadata file consists of -`[ccpp-table-properties]` header blocks followed by `[ccpp-arg-table]` variable listing -blocks, exactly as in the current framework. - -The `[ccpp-table-properties]` + `[ccpp-arg-table]` pair is redundant for non-scheme -tables (the distinction is vestigial for host/DDT/suite tables) but is preserved for -symmetry with scheme metadata tables. - -### 3.2 Table types (`type =` in `[ccpp-table-properties]`) - -Five table types are supported: - -| `type =` | Ownership | Import mechanism | -|---|---|---| -| `scheme` | Physics scheme | Intent args on scheme subroutines | -| `host` | Host model | Module USE (direct or via DDT member) | -| `control` | Framework runtime layer | Explicit args to `ccpp_physics_*` entry points | -| `suite` | Generated suite cap | Module USE of generated suite data module | -| `ddt` | Type definition | Structural — describes DDT fields, no instance info | - -Notes: -- `type = module` from capgen is renamed to `type = host`. Breaking change, intentional. -- `type = suite` tables are **written by the generator** (never hand-authored). They - appear on disk for inspection and debugging only. -- `type = ddt` describes the structure of a Fortran derived type. It contains no - instance information — only field definitions. - -### 3.3 Per-variable attributes - -All existing per-variable attributes are preserved: `standard_name`, `long_name`, -`units`, `dimensions`, `type`, `kind`, `intent`, `optional`, `active`, `protected`. - -`protected = True` means: any scheme that declares `intent` other than `in` for this -variable is a metadata error, caught at generation time. This is how constants are -handled — a constants DDT is declared `type = host` with all fields `protected = True`. -No separate `type = constants` is needed. - -### 3.4 DDT type definitions - -A DDT type definition uses `type = ddt`: - -```ini -[ccpp-table-properties] - name = gfs_statein_type - type = ddt - -[ccpp-arg-table] - name = gfs_statein_type - type = ddt - -[phii] - standard_name = geopotential_at_interface - long_name = geopotential at model layer interfaces - units = m2 s-2 - dimensions = (horizontal_dimension, vertical_interface_dimension) - type = real - kind = kind_phys -``` - -### 3.5 DDT instances - -A DDT instance is declared as a regular variable entry inside a `type = host` table. -The enclosing `[ccpp-table-properties]` block's `name` attribute identifies the Fortran -module from which the instance is imported via `use`. No separate `module` attribute is -needed on the variable entry. - -```ini -[ccpp-table-properties] - name = CCPP_data - type = host - dependencies = CCPP_typedefs.F90,GFS_typedefs.F90 - -[ccpp-arg-table] - name = CCPP_data - type = host - -[gfs_statein] - standard_name = gfs_statein - long_name = GFS state input for all instances - units = mixed - dimensions = (number_of_instances) - type = gfs_statein_type -``` - -The generator looks up the `gfs_statein_type` DDT table, traverses its fields, and -constructs access paths of the form -`gfs_statein(instance_number)%fieldname(loop_begin:loop_end, 1:nlevs)`. - -For scalar DDT instances (no instance dimension), dimensions is `()`. -For nested DDTs, the same mechanism applies recursively. - -### 3.6 Control variable declarations - -Control variables are declared in a `type = control` table. The generator resolves -each control variable by its standard name and uses whatever local Fortran name the -host declared: - -```ini -[ccpp-table-properties] - name = my_host_control_module - type = control - -[ccpp-arg-table] - name = my_host_control_module - type = control - -[loop_begin] - standard_name = horizontal_loop_begin - long_name = start of horizontal loop - units = index - dimensions = () - type = integer -``` - ---- - -## 4. Control Variables - -The generator recognizes the following standard names for control variables. Local -Fortran names are host-defined (resolved from the `type = control` metadata table). - -### 4.1 Entry point arguments (non-register phases) - -| Standard name | Role | Conditional? | -|---|---|---| -| `suite_name` | Suite name for runtime dispatch | No | -| `group_name` | Group name for runtime dispatch | No | -| `horizontal_loop_begin` | Start of horizontal chunk/domain | No | -| `horizontal_loop_end` | End of horizontal chunk/domain | No | -| `thread_number` | Current thread index (1..number_of_threads) | No | -| `number_of_threads` | Host blocking loop thread count; allocation bound for thread-dimensioned suite data | No | -| `number_of_physics_threads` | Thread budget for physics-internal OpenMP use | No | -| `ccpp_error_message` | Error message string | No | -| `ccpp_error_code` | Integer error return code | No | -| `instance_number` | Current model instance index | **Conditional** | - -`instance_number` is included only if `instance_dimension` appears anywhere in the -parsed host metadata. Single-instance models omit it from all entry point signatures. -The generator detects this automatically at parse time. - -### 4.2 Loop-generated control variables (subcycles only) - -| Standard name | Role | -|---|---| -| `ccpp_loop_counter` | Current subcycle iteration (1..ccpp_loop_extent) | -| `ccpp_loop_extent` | Total subcycle iterations; value comes from the `loop=N` attribute on the `` element in the suite XML definition file | - -These are **not** passed as `ccpp_physics_*` arguments from the host. They are set by -the generated `do` loop inside the group cap and are available to any scheme called -within that loop. Outside a subcycle loop, these variables are not in scope. - -### 4.3 Registered dimension standard names - -The generator has built-in semantic knowledge of these dimension standard names: - -| Standard name | Indexing semantic | -|---|---| -| `instance_dimension` | Scalar extraction: `var(instance_number)` — `instance_number` is the control variable | -| `horizontal_dimension` | Slice: `horizontal_loop_begin:horizontal_loop_end` (run phase) or `1:` (non-run) | -| `vertical_*` | Slice: `1:` | - -`horizontal_dimension` and `vertical_*` are "registered" because the generator knows -their slicing semantics, but they are resolved to local names the same way as arbitrary -dimensions — by looking up the variable with that standard name in the host metadata. -There is no special-casing in the resolution mechanism, only in the indexing expression -emitted. - -All other dimension standard names are resolved identically: look up the variable with -that standard name, get its local Fortran name, emit `1:local_name`. - -The timing of `instance_dimension` substitution — whether at parse time (when building -the flat dict access path) or at call-string generation time (like other registered -dimensions) — is an implementation decision left to the developer. Either is correct; -choose whichever is easier to implement, understand, and maintain. - ---- - -## 5. Entry Points - -Eight entry points are generated in the static API. Two tiers: - -### 5.1 Framework lifecycle (no group_name dispatch) - -These operate on the entire suite at once. They take `suite_name` plus -`ccpp_error_message` and `ccpp_error_code`. No scheme `_run/_init/_final` calls. - -| Entry point | Purpose | -|---|---| -| `ccpp_register(suite_name, constituents, errmsg, errcode)` | Calls each scheme's `_register` entrypoint; allocates and populates the `ccpp_model_constituents_t` object passed by the host | -| `ccpp_init(suite_name, errmsg, errcode)` | Allocates integer state arrays in all group caps; allocates suite-owned interstitial data; no scheme calls | -| `ccpp_final(suite_name, errmsg, errcode)` | Deallocates integer state arrays and suite-owned data; no scheme calls | - -`constituents` in `ccpp_register` is `intent(inout)`: unallocated on entry, allocated -and populated on exit. The host declares and owns this object (imports -`ccpp_model_constituents_t` from the framework library). The argument is mandatory. - -### 5.2 Physics group invocation (dispatched by suite_name + group_name) - -| Entry point | Calls scheme phase | -|---|---| -| `ccpp_physics_init(...)` | `_init` | -| `ccpp_physics_timestep_init(...)` | `_timestep_init` | -| `ccpp_physics_run(...)` | `_run` | -| `ccpp_physics_timestep_final(...)` | `_timestep_final` | -| `ccpp_physics_final(...)` | `_final` | - -All five take the full control variable argument list (Section 4.1). `group_name` is -optional — if omitted, the suite cap calls all groups in declared order. If specified, -only that group is invoked. - -Non-run phases pass `horizontal_loop_begin=1` and -`horizontal_loop_end=` — the cap code is uniform across -phases, with no special-casing for run vs. non-run. - -### 5.3 Naming note - -`finalize` is renamed to `final` throughout (e.g., `ccpp_physics_final`, not -`ccpp_physics_finalize`). Breaking change, intentional for symmetry: -`ccpp_init`/`ccpp_final`, `ccpp_physics_init`/`ccpp_physics_final`, -`ccpp_physics_timestep_init`/`ccpp_physics_timestep_final`. - ---- - -## 6. Cap Hierarchy - -All three levels are fully auto-generated. No hand-written components in the cap layer. - -### 6.1 Static API (`ccpp_static_api.F90`) - -- Imports all host DDTs and flat fields via `module use` (resolved from host metadata) -- Imports `ccpp_kinds` from `ccpp_kinds.F90` -- Dispatches all eight entry points by `suite_name` to the appropriate suite cap -- Passes `ccpp_model_constituents_t` through as an explicit argument (does not own it) -- Holds no physics state - -### 6.2 Suite cap (`ccpp__cap.F90`) - -- Imports the generated suite data module (`ccpp__data.F90`) -- Contains the suite-level integer state array: `integer, allocatable :: ccpp_suite_state(:)` indexed by instance -- Implements the suite-level state machine (see Section 7) -- On `ccpp_register`: calls all scheme `_register` entrypoints across the suite -- On `ccpp_init`: allocates suite-level state array; allocates `ccpp_group_state(:)` in - all group caps for this suite; allocates suite-owned interstitial data -- On `ccpp_final`: deallocates all of the above -- Routes `ccpp_physics_*` calls by `group_name` to the appropriate group cap function - -### 6.3 Group cap (`ccpp___cap.F90`) - -- Imports `ccpp__data` (suite-owned interstitial data) -- Imports `ccpp__types` (shared wrapper types) -- Contains the group-level integer state array: `integer, allocatable :: ccpp_group_state(:)` indexed by instance -- Implements the group-level state machine (see Section 7) -- Contains the actual scheme call sites for each phase: - - Loop bound locals - - Optional variable pointer arrays (thread-dimensioned) - - Fixed-index extraction locals - - Unit/kind conversion locals - - Subcycle `do` loops - - Scheme calls with full argument lists - ---- - -## 7. State Machine - -Integer state parameters are defined in a shared framework library module (not -generated). Two levels, both indexed by `instance_number`. - -### 7.1 Suite-level state (in suite cap) - -```fortran -integer, parameter :: CCPP_SUITE_UNREGISTERED = 0 -integer, parameter :: CCPP_SUITE_REGISTERED = 1 -integer, parameter :: CCPP_SUITE_FRAMEWORK_INITIALIZED = 2 -``` - -| Entry point | Required state | State after | -|---|---|---| -| `ccpp_register` | `== UNREGISTERED` | `REGISTERED` | -| `ccpp_init` | `== REGISTERED` | `FRAMEWORK_INITIALIZED` | -| `ccpp_physics_*` | `== FRAMEWORK_INITIALIZED` | (unchanged) | -| `ccpp_final` | `== FRAMEWORK_INITIALIZED` | `REGISTERED` | - -### 7.2 Group-level state (in each group cap) - -```fortran -integer, parameter :: CCPP_GROUP_UNINITIALIZED = 0 -integer, parameter :: CCPP_GROUP_INITIALIZED = 1 -integer, parameter :: CCPP_GROUP_IN_TIMESTEP = 2 -``` - -| Entry point | Required state | State after | -|---|---|---| -| `ccpp_physics_init` | `< INITIALIZED` (idempotent if `== INITIALIZED`) | `INITIALIZED` | -| `ccpp_physics_timestep_init` | `== INITIALIZED` | `IN_TIMESTEP` | -| `ccpp_physics_run` | `== IN_TIMESTEP` | `IN_TIMESTEP` | -| `ccpp_physics_timestep_final` | `== IN_TIMESTEP` | `INITIALIZED` | -| `ccpp_physics_final` | `>= INITIALIZED` | `UNINITIALIZED` | - -The idempotency rule for `ccpp_physics_init`: if the group is already in state -`INITIALIZED`, return immediately without calling any scheme `_init` routines. This -allows the host to call `ccpp_physics_init` multiple times safely. Any further call -after the first must result in no change (idempotency is a scheme contract). - -These two integer arrays replace both the boolean `initialized(:)` array from prebuild -and the string-based `ccpp_suite_state` from CAM-SIMA. - ---- - -## 8. Scheme Metadata and Variable Matching - -### 8.1 Scheme metadata structure - -Each scheme source file has a companion `.meta` file with `type = scheme` tables — one -table per public phase subroutine (`scheme_name_init`, `scheme_name_run`, -`scheme_name_timestep_init`, etc.). The section header for each variable entry is the -**local variable name** as it appears in the scheme's Fortran subroutine argument list. - -The internal metadata store is keyed as: - -``` -metadata[scheme_name][phase][standard_name] → {local_name, units, kind, dimensions, - intent, optional, active, ...} -``` - -Keying by `scheme_name` then `phase` enables cross-phase queries ("does this scheme -have a register phase?", "what are all variables of scheme X?") and matches the -conceptual model of the suite XML. - -### 8.2 Reading order - -All metadata files (host + scheme + DDT) are read in one pass without resolving DDT -type references. After the full read, the generator builds the known DDT list, then -resolves all type references. This avoids ordering dependencies between metadata files. - -### 8.3 Known DDT list - -After reading all metadata, the generator assembles the set of known DDT types from -`type = ddt` tables. Two categories: - -**Framework-defined DDTs**: declared in `type = ddt` metadata tables. The generator -knows their fields, dimensions, and access paths. - -**External DDTs**: types from external libraries (MPI, ESMF, etc.) that the generator -cannot introspect. These are declared in variable entries using an extended `type` -syntax: - -```ini -[mycomm] - standard_name = mpi_communicator - type = external:mpi_f08:mpi_comm - ... -``` - -The format is `external::`. The generator emits -`use mpi_f08, only: mpi_comm` and treats the variable as an opaque type — no field -traversal, no dimension indexing beyond what the metadata declares. - -### 8.4 Variable matching: scheme vs. host - -For each argument in a scheme's phase function (looked up by standard name): - -1. **Found in host+control flat dict** → use the resolved access path. If `units` or - `kind` differ from what the scheme declares, generate a transformation (Section 9). -2. **Not found, first use is `intent(out)`** → suite-owned variable. Add to suite data, - generate declaration in `ccpp__data.F90`. -3. **Not found, first use is `intent(in)` or `intent(inout)`** → **error**: variable - used before it is provided by any scheme or host. -4. **Found in suite data (from a prior scheme)** → use the suite data access path. - Apply transformation if needed. - -### 8.5 Cap call argument construction - -The generator builds the argument list for each scheme call from the scheme's metadata -argument order. For each argument: - -- **Direct pass-through** (no transformation, not optional): inline host access - expression — no local variable declared -- **Transformation**: cap-local temporary named after the scheme's local variable name - (from scheme metadata section header); see Section 10.2 for naming rules -- **Optional**: cap-local pointer array named `_p`; see Section 10.3 -- **Optional + transformation**: combined in the `if (active) then` block - -The generator does not parse Fortran source. All local names, types, kinds, dimensions, -and intents come exclusively from metadata. - ---- - -## 9. Variable Resolution and Access Path Construction - -### 9.1 Flat storage model (host+control+suite) - -The generator flattens the DDT hierarchy at parse time. After parsing, all host, -control, and suite variables are stored in a flat dictionary keyed by standard name. -Each entry contains: -- The Fortran local name -- The fully-qualified access path (e.g., `gfs_statein(instance_number)%phii`) -- The module to USE -- Dimension information with registered/arbitrary classification - -The DDT hierarchy is discarded after the flat dict is built. No live DDT object tree -is maintained during code generation. - -### 9.2 Access path construction - -For each variable, the generator constructs the call-site expression by applying -dimension rules to each dimension in order: - -1. **`instance_dimension`** → substitute `instance_number` (scalar extraction) -2. **`horizontal_dimension`** → substitute `horizontal_loop_begin:horizontal_loop_end` - (run phase) or `1:local_horizontal_dimension` (non-run) -3. **`vertical_*`** → substitute `1:local_vertical_dimension` -4. **Arbitrary dimension** → resolve to local name via its own metadata entry, emit - `1:local_name` -5. **`active` condition** → generate optional pointer-association guard (see Section 10.3) - -### 9.3 Module USE - -For each variable used in a group cap, the generator emits a `use module, only: varname` -statement. The module name comes from the enclosing `[ccpp-table-properties]` block name. -For suite-owned variables, the module is the generated `ccpp__data`. - -### 9.4 Eliminating TYPEDEFS_NEW_METADATA - -The manually-maintained `TYPEDEFS_NEW_METADATA` Python dict from prebuild is eliminated. -All information previously in that dict is now in metadata: -- The DDT type structure → `type = ddt` table -- The module-level instance → variable entry in the `type = host` table, module - implied by enclosing table name - ---- - -## 10. Variable Transformations and Optional Variables - -Variable transformations (unit/kind conversions) and optional variable handling are -combined — both occur within the same `if (active) then` block when a variable is -optional. - -### 10.1 Supported transformations - -- **Unit conversions** (e.g., Pa → hPa): formula from built-in conversion table (shared - with validator), keyed on source/target unit pair from metadata `units` attribute -- **Kind conversions** (e.g., r8 → kind_phys): from `kind` metadata attribute comparison - -The transformation framework is generic and pluggable — additional transformation types -(e.g., vertical flipping) can be added without restructuring the generator. - -### 10.2 Local variable naming - -The local variable name for a transformation temporary or optional pointer is derived -from the **scheme's local variable name** as declared in the scheme's metadata section -header (e.g., `[phii]` → local name is `phii`). The generator has this name without -parsing any Fortran. - -- Transformation temporary: scheme's local name + `_l` (e.g., `phii_l`) -- Optional pointer: scheme's local name + `_p` (e.g., `phii_p`) -- Conflict resolution: if two schemes in the same group cap use the same local name for - different standard names, append a numeric suffix before the suffix (e.g., `phii_2_l`) - -The generator validates that all generated local variable names and generated subroutine -names stay within Fortran's 63-character identifier limit. Violations are code-generation -errors — the developer must use a shorter local name in their metadata. - -The `active` expression in metadata is a Fortran logical expression written using -**CCPP standard names** (not local names). The generator translates all standard names -in the expression to their local Fortran names before emitting. - -Transformations **always** use a local temporary variable. The host variable is never -modified in-place — required for bit-for-bit reproducibility and to leave host data -uncorrupted if an exception occurs. Every conversion line carries an inline Fortran -comment (e.g., `! unit conversion: Pa to hPa`). Transformation mismatches (unknown -unit pair, unknown kind pair) are code-generation errors — no stdout/stderr from -generated code. - -### 10.3 The four cases - -The generator handles exactly four combinations per variable: - -**Case 1: No pointer, no transformation** (not optional, no unit/kind mismatch) - -No local variable is declared. The host access expression is used inline at the call -site: -```fortran -call scheme_run(..., gfs_statein(instance_number)%phii(lb:ub,1:nlevs), ...) -``` - -**Case 2: Pointer only** (optional, no transformation) - -Pointer array declared at function top; conditional association in `if (active)` block: -```fortran -! declaration: -type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) - -! before call: -if () then - phii_p(thread_number)%ptr => gfs_statein(instance_number)%phii(lb:ub,1:nlevs) -else - nullify(phii_p(thread_number)%ptr) -end if - -call scheme_run(..., phii_p(thread_number)%ptr, ...) - -! after call: -nullify(phii_p(thread_number)%ptr) -``` - -**Case 3: Transformation only** (not optional, unit/kind mismatch) - -Local temporary `phii_l` (scheme's local name + `_l`); intent-driven emission: - -| `intent` | Pre-call | Post-call | -|---|---|---| -| `in` | `phii_l = host_phii(...) * factor ! unit conversion: X to Y` | nothing | -| `out` | nothing | `host_phii(...) = phii_l / factor ! unit conversion: Y to X` | -| `inout` | pre-call as above | post-call as above | - -```fortran -! declaration: -real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) ! or appropriate rank/kind - -! before call (intent in/inout): -phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa - -call scheme_run(..., phii_l, ...) - -! after call (intent inout/out): -gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa -``` - -**Case 4: Pointer and transformation** (optional + unit/kind mismatch) - -Two local variables: `phii_l` (transformation temporary) and `phii_p` (pointer array). -Sequence: (1) apply transformation to `phii_l` depending on intent, (2) assign pointer -to `phii_l`, (3) call scheme, (4) nullify pointer, (5) apply back-transformation from -`phii_l` to host depending on intent. All within the `if (active)` block: - -```fortran -! declarations: -real(kind=kind_phys) :: phii_l(lb:ub, 1:nlevs) -type(real_kind_phys_rank1_ptr_type), target :: phii_p(number_of_threads) - -! before call: -if () then - ! step 1: apply forward transformation (intent in/inout) - phii_l = gfs_statein(instance_number)%phii(lb:ub,1:nlevs) * 0.01_kind_phys ! unit conversion: Pa to hPa - ! step 2: assign pointer to transformed local - phii_p(thread_number)%ptr => phii_l -else - nullify(phii_p(thread_number)%ptr) -end if - -call scheme_run(..., phii_p(thread_number)%ptr, ...) - -! after call: -if () then - ! step 4: nullify pointer - nullify(phii_p(thread_number)%ptr) - ! step 5: apply back-transformation (intent inout/out) - gfs_statein(instance_number)%phii(lb:ub,1:nlevs) = phii_l * 100.0_kind_phys ! unit conversion: hPa to Pa -end if -``` - -The wrapper types (`real_kind_phys_rank1_ptr_type` etc.) are defined once in the -generated shared types module (`ccpp__types.F90`), not re-declared inside every -group cap function. Passing an unassociated pointer is safe under all compiler modes. - ---- - -## 11. Subcycle Loops - -When a group in the suite XML contains ``, the generator emits a -Fortran `do` loop in the group cap: - -```fortran -do = 1, N ! subcycle: N iterations from suite XML - ! ... scheme calls ... -end do -``` - -`ccpp_loop_counter` and `ccpp_loop_extent` are **loop-context variables** — a special -class that does not fit any of the five table types: -- NOT `type = control`: not passed as `ccpp_physics_*` arguments from the host -- NOT `type = host`: not from host module USE -- NOT `type = suite`: not persistent allocated data - -The generator has built-in knowledge of these two standard names. `ccpp_loop_extent` -value comes from the `loop=N` attribute in the suite XML definition file. -`ccpp_loop_counter` is the do loop induction variable. Both exist only within the -generated loop scope — they are not in scope outside a subcycle block. - -Any scheme that requests `ccpp_loop_counter` or `ccpp_loop_extent` by standard name -receives the loop variables at the call site. Their local names in the cap are derived -from the scheme's metadata section headers as for any other variable (Section 10.2). - ---- - -## 12. Init/Finalize Deduplication - -If the same scheme appears more than once within a single group (e.g., via subcycles), -having its `_init` called multiple times is a **code generator bug** — the generator -**errors out** rather than silently deduplicating. The suite XML must not list the same -scheme multiple times in the same group for non-run phases. - -If the same scheme appears in multiple groups, its `_init` is called once per group. -This is acceptable — idempotency is a contract all scheme `_init` routines must satisfy. - -The same rule applies to `_timestep_init`, `_timestep_final`, and `_final`. - ---- - -## 13. Suite-Owned Data - -### 13.1 Discovery - -The generator identifies suite-owned variables during variable resolution: variables -requested by schemes that are not satisfied by host metadata (`type = host` or -`type = control`). - -**Error condition**: if a variable is determined to be suite-owned (not provided by the -host) and the first scheme that uses it does not have it as `intent(out)`, the generator -errors out. A suite-owned variable that is first read before it is written would be -used uninitialized. - -### 13.2 Generated files and allocation - -Suite-owned variables are declared in a generated suite data module -(`ccpp__data.F90`) as fields of a Fortran DDT. The suite cap and all group caps -`use` this module. - -Allocation happens in `ccpp_init` (suite cap). Deallocation happens in `ccpp_final`. -Suite-owned arrays are allocated for the **full `horizontal_dimension`** — threads -access their respective horizontal chunk (`horizontal_loop_begin:horizontal_loop_end`) -at scheme call sites. This avoids per-call allocation overhead. If this proves to -consume too much memory at scale, a future revision may move allocation to per-phase -with chunk-sized arrays; start with the full-dimension approach. - -Subsetting (applying horizontal loop bounds, instance index) happens at scheme call -sites in the group cap, not in the suite cap. - -### 13.3 Metadata - -The generator also writes a `type = suite` metadata table (`ccpp_.meta`) as a -byproduct. This file is for inspection and debugging — it is not consumed by the -generator on subsequent runs. - ---- - -## 14. Constituent API - -### 14.1 Type definition - -`ccpp_model_constituents_t` is unchanged from CAM-SIMA. The type definition lives in -the framework library (`ccpp_constituent_prop_mod`), not in generated code. - -### 14.2 Ownership and lifecycle - -The **host model** declares and owns the constituent object: - -```fortran -use ccpp_constituent_prop_mod, only: ccpp_model_constituents_t -type(ccpp_model_constituents_t) :: constituents ! unallocated initially -``` - -The host passes it to `ccpp_register`, which allocates and populates it. After -`ccpp_register` returns, the host holds a fully allocated object ready for `lock_table`, -`const_index`, `copy_in`, `copy_out`. - -The `constituents` argument to `ccpp_register` is mandatory. - -### 14.3 Register phase mechanics - -The suite cap's register routine: -1. Iterates over all constituent-providing schemes in the suite -2. Calls each scheme's `_register` entrypoint, which returns a - `ccpp_constituent_properties_t` array -3. Collects these arrays and populates the constituent object - -No `group_name` dispatch is needed for register — it operates on the whole suite. - ---- - -## 15. Generated Output Files - -All files are written to `--output-root`. - -| File | Contents | -|---|---| -| `ccpp_kinds.F90` | Kind parameter definitions from `--kind-type` CLI args | -| `ccpp_static_api.F90` | Static API — host imports, suite_name dispatch | -| `ccpp__cap.F90` | Suite cap — suite data import, state machine, group dispatch | -| `ccpp___cap.F90` | Group cap — scheme call sites, state array, optionals, transformations | -| `ccpp__data.F90` | Suite data module — framework-owned interstitial DDT | -| `ccpp__types.F90` | Shared cap types — optional pointer wrapper types, transformation locals | -| `ccpp_.meta` | Generated `type = suite` metadata table (output-only, for inspection) | -| `datatable.xml` | Generator database for `ccpp_datafile.py` queries | - -`ccpp_kinds.F90` is a dependency of all other generated Fortran files. - ---- - -## 16. CLI Invocation - -``` -ccpp_capgen_ng.py - --host-name - --host-files - --scheme-files - --suites - --output-root - --kind-type KIND=PRECISION # repeatable, e.g. --kind-type kind_phys=REAL64 - --verbose # once = info; twice = debug -``` - -The generator also supports programmatic Python invocation (import and call directly), -using the same internal code paths as the CLI. - -### 16.1 Kind specifications - -Kind mappings are passed at the CLI level, not in metadata. The generator substitutes -kind names in all generated Fortran declarations. Example: -`--kind-type kind_phys=REAL64 --kind-type kind_dyn=REAL32`. - -### 16.2 datatable.xml and ccpp_datafile.py - -The generator emits `datatable.xml` encoding the full relationships between suites, -groups, schemes, and variables. A separate query utility (`ccpp_datafile.py`) provides -a rich query interface used by CMake and other build systems. The full query surface of -the existing `ccpp_datafile.py` is preserved — the simplified `--ccpp-files` query is -one of many. - -### 16.3 CMake integration pattern - -The generator runs at CMake configure time via `execute_process`. Generated sources are -discovered by querying `ccpp_datafile.py --ccpp-files`. Host and scheme Fortran sources -are found by replacing `.meta` with `.F90` (same base name convention). - ---- - -## 17. Design Decisions Not Carried Forward - -The following patterns from prebuild or capgen are explicitly **not** carried forward: - -| Pattern | Reason | -|---|---| -| `ccpp_t` / `cdata` struct | Replaced by explicit named control variable arguments | -| `TYPEDEFS_NEW_METADATA` Python dict | Replaced by DDT instance declarations in metadata | -| String-based `ccpp_suite_state` | Replaced by integer state arrays with named parameters | -| Boolean `initialized(:)` array | Replaced by integer state arrays | -| Flat-field arguments to group caps | DDT arguments are used instead (as in prebuild) | -| Scope-chain variable promotion | Suite-owned variables explicitly discovered and declared | -| Fortran-vs-metadata validation in generator | Moved to standalone validator tool | -| Re-declaration of pointer wrapper types per function | Declared once in shared types module | -| `ccpp_physics_suite_init/finalize` | Replaced by `ccpp_init`/`ccpp_final` | -| `type = module` (capgen) | Renamed to `type = host` | -| `finalize` phase name | Renamed to `final` | -| Array size checks in caps | Not generated by default; rely on compiler bounds checking | From 0ac0deedfda617df28b86f768a055f05c9ba1a7c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 1 Jun 2026 14:35:36 -0600 Subject: [PATCH 45/74] ccpp_validator for host data --- capgen-ng/ccpp_validator.py | 646 ++++++++++++- capgen-ng/generator/suite_resolver.py | 88 +- capgen-ng/metadata/metadata_table.py | 23 +- doc/migration.md | 44 +- doc/redesign_prompt.md | 10 +- end-to-end-tests/advection/CMakeLists.txt | 6 +- .../advection_auto_clone/CMakeLists.txt | 2 + end-to-end-tests/capgen_ng/CMakeLists.txt | 6 +- end-to-end-tests/chunked_data/CMakeLists.txt | 6 +- end-to-end-tests/cmake/ccpp_capgen.cmake | 34 +- end-to-end-tests/ddthost/CMakeLists.txt | 6 +- end-to-end-tests/ddthost/host_ccpp_ddt.F90 | 2 +- end-to-end-tests/instances/CMakeLists.txt | 6 +- .../instances_advection/CMakeLists.txt | 7 +- end-to-end-tests/nested_suite/CMakeLists.txt | 6 +- .../nested_suite/test_host_data.meta | 2 - end-to-end-tests/opt_arg/CMakeLists.txt | 6 +- end-to-end-tests/var_compat/CMakeLists.txt | 6 +- .../var_compat/test_host_data.meta | 2 - unit-tests/test_metadata_table.py | 108 +++ unit-tests/test_suite_resolver.py | 150 ++- unit-tests/test_validator.py | 907 +++++++++++++++++- 22 files changed, 1977 insertions(+), 96 deletions(-) diff --git a/capgen-ng/ccpp_validator.py b/capgen-ng/ccpp_validator.py index cec30881..53758520 100755 --- a/capgen-ng/ccpp_validator.py +++ b/capgen-ng/ccpp_validator.py @@ -327,7 +327,15 @@ def _parse_decl_line(line: str) -> Dict[str, _ArgAttrs]: var_tok = var_tok.strip() if not var_tok: continue - name_match = re.match(r'(\w+)\s*(\((.*)\))?\s*(=.*)?$', var_tok) + # Strip any ``= `` (or ``=> ``) clause before the + # name regex. A Fortran array initialiser may carry a full + # ``(/ ..., ..., ... /)`` constructor whose inner commas would + # otherwise be gobbled by the regex's greedy parens-matcher and + # miscounted as rank-N entries. ``=`` characters inside other + # parenthesised sub-expressions (e.g. ``::x = (a==b)``) live at + # depth > 0 and are skipped. + var_tok = _strip_initialiser(var_tok).rstrip() + name_match = re.match(r'(\w+)\s*(\((.*)\))?\s*$', var_tok) if name_match is None: continue name = name_match.group(1).lower() @@ -343,6 +351,40 @@ def _parse_decl_line(line: str) -> Dict[str, _ArgAttrs]: return result +def _strip_initialiser(var_tok: str) -> str: + """Return *var_tok* with any trailing ``= `` removed. + + Paren-aware: only an ``=`` (or the first ``=`` of ``=>``) at nesting + depth zero is treated as the start of an initialiser. ``=`` + characters inside parenthesised sub-expressions (such as array + initialisers ``(/ ... /)`` or default expressions ``(a == b)``) + are skipped. + + Examples + -------- + >>> _strip_initialiser('foo') + 'foo' + >>> _strip_initialiser('foo(:)') + 'foo(:)' + >>> _strip_initialiser('p => null()') + 'p ' + >>> _strip_initialiser('arr(n) = (/ 1, 2, 3 /)') + 'arr(n) ' + >>> _strip_initialiser('std_name_array(num_consts) = (/ ' + ... "'a', 'b', 'c' /)") + 'std_name_array(num_consts) ' + """ + depth = 0 + for i, ch in enumerate(var_tok): + if ch == '(': + depth += 1 + elif ch == ')': + depth -= 1 + elif ch == '=' and depth == 0: + return var_tok[:i] + return var_tok + + def _join_continuation( lines: List[str], filename: Optional[str] = None, @@ -610,6 +652,234 @@ def _load_source_tree(source_files: List[str]) -> Dict[str, _SubSig]: return merged +# --------------------------------------------------------------------------- +# Module-level and derived-type parsing (for host / DDT validation) +# --------------------------------------------------------------------------- + +# Matches the start of a module definition (case-insensitive). Excludes +# ``module procedure`` so we don't mistake interface-block lines for +# module headers. +_MODULE_RE = re.compile(r'(?i)^\s*module\s+(?!procedure\b)(\w+)\s*$') +_END_MODULE_RE = re.compile(r'(?i)^\s*end\s*module\b') + +# Matches the start of a derived-type definition. Accepts the modern +# ``type :: name`` form, the older ``type, :: name`` form, and +# the bare ``type name`` form. Excludes ``type(x) ::`` declarations +# (those are variable decls of a type) by requiring no opening paren +# before the name on the type-defining form. +_TYPE_DEF_RE = re.compile( + r'(?i)^\s*type(?:\s*,\s*[^:]+)?\s*::\s*(\w+)\s*$' +) +_TYPE_DEF_BARE_RE = re.compile(r'(?i)^\s*type\s+(\w+)\s*$') +_END_TYPE_RE = re.compile(r'(?i)^\s*end\s*type\b') + +# Matches ``contains`` at module / type-block scope, used to recognise the +# boundary between module-level decls and the module's subroutines (we +# stop collecting module-level vars at the first ``contains``). Also +# applies inside a derived type with type-bound procedures. +_CONTAINS_RE = re.compile(r'(?i)^\s*contains\s*$') + + +class _ModuleSig(NamedTuple): + """Parsed module-level declarations from one Fortran module. + + Attributes + ---------- + vars : dict + ``{lower_name: _ArgAttrs}`` for every module-level variable + declaration above the ``contains`` line (or end of module). + Used to validate ``type = host`` table entries. + ddts : dict + ``{lower_type_name: {lower_component_name: _ArgAttrs}}`` for + every ``type :: X ... end type X`` block at module scope. + Used to validate ``type = ddt`` table entries. + """ + vars: Dict[str, '_ArgAttrs'] + ddts: Dict[str, Dict[str, '_ArgAttrs']] + + +def _parse_modules( + source: str, + filename: Optional[str] = None, +) -> Dict[str, _ModuleSig]: + """Extract module-level variable decls and derived-type definitions. + + Walks *source* line by line tracking three nested contexts: module, + derived-type block, and subroutine body. Module-level variable + declarations are collected only at module scope above ``contains``; + derived-type components are collected only inside a + ``type :: X ... end type X`` block; subroutine local decls are + ignored. + + The first definition of each module / type wins (Fortran does not + permit redefinition; this only matters for malformed input). + + Examples + -------- + >>> src = ('module my_mod\\n' + ... ' use kinds, only: kind_phys\\n' + ... ' integer :: nlev\\n' + ... ' real(kind=kind_phys) :: cp\\n' + ... ' type :: phys_t\\n' + ... ' real(kind=kind_phys) :: tk(:,:)\\n' + ... ' integer :: nlay\\n' + ... ' end type phys_t\\n' + ... 'contains\\n' + ... ' subroutine helper(x)\\n' + ... ' integer, intent(in) :: x\\n' + ... ' real :: local\\n' + ... ' end subroutine helper\\n' + ... 'end module my_mod\\n') + >>> mods = _parse_modules(src) + >>> sorted(mods.keys()) + ['my_mod'] + >>> sorted(mods['my_mod'].vars.keys()) + ['cp', 'nlev'] + >>> mods['my_mod'].vars['cp'].type_ + 'real' + >>> mods['my_mod'].vars['cp'].kind_ + 'kind_phys' + >>> sorted(mods['my_mod'].ddts.keys()) + ['phys_t'] + >>> sorted(mods['my_mod'].ddts['phys_t'].keys()) + ['nlay', 'tk'] + >>> mods['my_mod'].ddts['phys_t']['tk'].rank + 2 + >>> 'local' in mods['my_mod'].vars + False + """ + logical = _join_continuation( + source.splitlines(keepends=True), filename=filename, + ) + + modules: Dict[str, _ModuleSig] = {} + # Active module context (None outside of any module). + cur_mod_name: Optional[str] = None + cur_mod_vars: Dict[str, _ArgAttrs] = {} + cur_mod_ddts: Dict[str, Dict[str, _ArgAttrs]] = {} + # Active derived-type block context inside the current module. + cur_type_name: Optional[str] = None + cur_type_comps: Dict[str, _ArgAttrs] = {} + # Depth of subroutine / function nesting inside the current module. + # Decls inside a subroutine are local variables, not module-level + # state, and must be skipped. + sub_depth: int = 0 + # Once a ``contains`` is seen at module scope, module-level decls + # are done — the rest of the module is type-bound and subroutine + # bodies. We still need to track sub_depth to find the matching + # ``end module``. + past_module_contains: bool = False + + for line in logical: + # Module header / footer. + m = _MODULE_RE.match(line) + if m and cur_mod_name is None: + cur_mod_name = m.group(1).lower() + cur_mod_vars = {} + cur_mod_ddts = {} + cur_type_name = None + cur_type_comps = {} + sub_depth = 0 + past_module_contains = False + continue + if _END_MODULE_RE.match(line) and cur_mod_name is not None: + if cur_mod_name not in modules: + modules[cur_mod_name] = _ModuleSig( + vars=cur_mod_vars, ddts=cur_mod_ddts, + ) + cur_mod_name = None + continue + if cur_mod_name is None: + # Free-floating decls outside any module are not part of the + # host-validation surface. CCPP host code conventionally + # lives inside modules. + continue + + # Track subroutine / function nesting so we can skip local + # declarations. Use _SUB_RE for subroutines; functions are + # less common in CCPP host code but handled symmetrically. + if _SUB_RE.match(line) or re.match(r'(?i)\s*(?:(?:recursive|pure|elemental|impure)\s+)*(?:(?:real|integer|logical|complex|character|double\s*precision|type\s*\([^)]+\))\s+)?function\s+\w+', line): + sub_depth += 1 + continue + if _END_SUB_RE.match(line) or re.match(r'(?i)^\s*end\s*function\b', line): + if sub_depth > 0: + sub_depth -= 1 + continue + if sub_depth > 0: + continue + + # Derived-type block boundaries (only honoured at module scope, + # never inside a subroutine body). + if cur_type_name is None: + m = _TYPE_DEF_RE.match(line) or _TYPE_DEF_BARE_RE.match(line) + if m: + cur_type_name = m.group(1).lower() + cur_type_comps = {} + continue + else: + if _END_TYPE_RE.match(line): + if cur_type_name not in cur_mod_ddts: + cur_mod_ddts[cur_type_name] = cur_type_comps + cur_type_name = None + cur_type_comps = {} + continue + if _CONTAINS_RE.match(line): + # Type-bound procedures follow; no more components. + continue + # Inside a type block: every parsed decl is a component. + for name, attrs in _parse_decl_line(line).items(): + if name not in cur_type_comps: + cur_type_comps[name] = attrs + continue + + # Module scope: check for ``contains`` boundary and otherwise + # collect module-level variable decls. + if _CONTAINS_RE.match(line): + past_module_contains = True + continue + if past_module_contains: + continue + for name, attrs in _parse_decl_line(line).items(): + if name not in cur_mod_vars: + cur_mod_vars[name] = attrs + + return modules + + +def _load_modules_tree( + source_files: List[str], +) -> Tuple[Dict[str, _ModuleSig], Dict[str, Dict[str, _ArgAttrs]]]: + """Read all Fortran source files and return module + global DDT dicts. + + Returns a tuple ``(modules, ddt_index)``: + + * ``modules`` — ``{module_name_lower: _ModuleSig}``. First occurrence + wins if the same module name appears in multiple files. + * ``ddt_index`` — flat ``{type_name_lower: {component_name_lower: + _ArgAttrs}}`` mapping derived-type names to their component dicts, + collected across every parsed module. Used to resolve + ``type(name) :: var`` declarations whose underlying type lives in + a different module than the variable. First occurrence wins. + + Parameters + ---------- + source_files : list of str + Paths to ``.F90`` / ``.f90`` files. + """ + modules: Dict[str, _ModuleSig] = {} + ddt_index: Dict[str, Dict[str, _ArgAttrs]] = {} + for fpath in source_files: + with open(fpath) as fh: + src = fh.read() + for name, sig in _parse_modules(src, filename=fpath).items(): + if name not in modules: + modules[name] = sig + for ddt_name, comps in sig.ddts.items(): + if ddt_name not in ddt_index: + ddt_index[ddt_name] = comps + return modules, ddt_index + + # --------------------------------------------------------------------------- # Validation logic # --------------------------------------------------------------------------- @@ -893,20 +1163,197 @@ def _check_arg_attributes( ) ) - # rank — number of dimensions. Metadata stores them as a list of - # standard-name strings; Fortran rank is counted from the decl. - meta_rank = len(meta_var.dimensions or []) - if meta_rank != fort.rank: + # rank — number of dimensions. When the metadata ``local_name`` + # carries a subscript (sliced array entry such as + # ``q(:,:,index_of_water_vapor)``), the metadata's ``dimensions`` + # list describes the *view* after slicing, not the underlying + # Fortran rank. ``_expected_fort_rank`` resolves this: it returns + # the subscript width when a subscript is present, otherwise + # ``len(meta_var.dimensions)``. + meta_dims = list(meta_var.dimensions or []) + expected_rank = _expected_fort_rank(meta_var.local_name, meta_dims) + if expected_rank != fort.rank: errs.append( - prefix + "rank mismatch (metadata declares {} dimension(s) {}, " - "Fortran declares rank {})".format( - meta_rank, list(meta_var.dimensions or []), fort.rank, + prefix + "rank mismatch (metadata implies Fortran rank {} " + "from local_name '{}' and dimensions {}, Fortran declares " + "rank {})".format( + expected_rank, meta_var.local_name, meta_dims, fort.rank, ) ) return errs +def _base_local_name(local_name: str) -> str: + """Return the bare Fortran identifier from a metadata ``local_name``. + + Host / DDT metadata occasionally carries subscripted ``local_name`` + values (sliced array entries) — the matching Fortran decl carries the + bare identifier, so strip the subscript before lookup. Lowercase for + case-insensitive comparison. + + Examples + -------- + >>> _base_local_name('cp') + 'cp' + >>> _base_local_name('Phys_State') + 'phys_state' + >>> _base_local_name('tk(:,:)') + 'tk' + >>> _base_local_name('dqdt(:,:,index_of_cloud)') + 'dqdt' + """ + name = local_name.strip() + if '(' in name: + name = name.split('(', 1)[0] + return name.lower() + + +def _expected_fort_rank(local_name: str, meta_dims: List[str]) -> int: + """Return the Fortran rank implied by a metadata ``local_name``. + + Sliced metadata local names (``q(:,:,index_of_X)``) express a + reduced-rank *view* of a higher-rank Fortran component: every + subscript entry consumes one dimension of the underlying Fortran + variable, but only ``:`` entries survive into the resulting view's + rank. The metadata's ``dimensions =`` list describes the view, not + the underlying variable — so the expected Fortran rank equals the + total number of subscript entries, not ``len(meta_dims)``. + + For bare local names (no subscript), Fortran rank simply equals the + metadata-declared dimension count. + + Examples + -------- + >>> _expected_fort_rank('cp', []) + 0 + >>> _expected_fort_rank('tk', ['horizontal_dimension', + ... 'vertical_layer_dimension']) + 2 + >>> _expected_fort_rank('q(:,:,index_of_water_vapor)', + ... ['horizontal_dimension', + ... 'vertical_layer_dimension']) + 3 + >>> _expected_fort_rank('q(:)', ['horizontal_dimension']) + 1 + >>> _expected_fort_rank('q(:,:,:)', ['horizontal_dimension', + ... 'vertical_layer_dimension', + ... 'number_of_tracers']) + 3 + """ + name = local_name.strip() + if '(' not in name: + return len(meta_dims) + inner = name.split('(', 1)[1] + inner = inner.rsplit(')', 1)[0] + # Each subscript entry consumes one Fortran dimension; paren-aware + # split protects nested expressions like ``my(:,foo(a,b),:)`` from + # being miscounted (none of the in-tree fixtures use that today, but + # the parser permits it). + return len(_paren_aware_split(inner, ',')) + + +def _validate_host_table( + table, + modules_tree: Dict[str, _ModuleSig], + logger: logging.Logger, +) -> List[str]: + """Validate a ``type = host`` metadata table against its Fortran module. + + The Fortran module name is taken from ``table.module_name`` when set, + otherwise from ``table.table_name`` (the .meta convention). For each + metadata variable, look up the matching module-level decl by base + local-name and reuse :func:`_check_arg_attributes` for the per-attr + checks (intent is silently ignored since host vars carry no intent). + + Returns a list of error message strings (empty on full match). When + the named module is missing entirely, a single "module not found" + error is returned and per-variable checks are skipped. + """ + errors: List[str] = [] + mod_name = (table.module_name or table.table_name).lower() + sig = modules_tree.get(mod_name) + if sig is None: + errors.append( + "Host module '{}' (from table '{}' in '{}') not found in any " + "source file.".format(mod_name, table.table_name, table.file_path) + ) + return errors + + logger.debug( + "Checking host table '%s' against module '%s' (%d module-level vars)", + table.table_name, mod_name, len(sig.vars), + ) + + for mvar in table.variables(): + base = _base_local_name(mvar.local_name) + fattrs = sig.vars.get(base) + if fattrs is None: + errors.append( + "Host variable '{}' (standard_name '{}') declared in " + "metadata table '{}' not found as a module-level " + "declaration in Fortran module '{}'.".format( + mvar.local_name, mvar.standard_name, + table.table_name, mod_name, + ) + ) + continue + errors.extend( + _check_arg_attributes(mod_name, base, mvar, fattrs) + ) + return errors + + +def _validate_ddt_table( + table, + ddt_index: Dict[str, Dict[str, _ArgAttrs]], + logger: logging.Logger, +) -> List[str]: + """Validate a ``type = ddt`` metadata table against its Fortran type. + + The DDT name is ``table.table_name`` (the .meta convention: the table + name is the Fortran type name). The matching ``type :: X ... end + type X`` block may live in any parsed module — looked up via the flat + *ddt_index* built by :func:`_load_modules_tree`. For each metadata + component, match against the type block by base local-name and reuse + :func:`_check_arg_attributes` for per-attr checks. + + Returns a list of error message strings (empty on full match). + """ + errors: List[str] = [] + ddt_name = table.table_name.lower() + comps = ddt_index.get(ddt_name) + if comps is None: + errors.append( + "DDT '{}' (table in '{}') not found as a derived-type definition " + "in any source file.".format(ddt_name, table.file_path) + ) + return errors + + logger.debug( + "Checking DDT table '%s' (%d components in Fortran)", + ddt_name, len(comps), + ) + + for mvar in table.variables(): + base = _base_local_name(mvar.local_name) + fattrs = comps.get(base) + if fattrs is None: + errors.append( + "DDT component '{}' (standard_name '{}') declared in " + "metadata table '{}' not found as a component of Fortran " + "type '{}'.".format( + mvar.local_name, mvar.standard_name, + table.table_name, ddt_name, + ) + ) + continue + errors.extend( + _check_arg_attributes(ddt_name, base, mvar, fattrs) + ) + return errors + + _FORTRAN_EXTENSIONS = ('.F90', '.f90', '.F', '.f') @@ -956,14 +1403,26 @@ def _fortran_file_for_table(table) -> Optional[str]: def validate( scheme_files: List[str], source_files: Optional[List[str]] = None, + host_files: Optional[List[str]] = None, logger: Optional[logging.Logger] = None, ) -> List[str]: - """Validate scheme metadata against Fortran source files. + """Validate scheme + host metadata against Fortran source files. + + Scheme tables in *scheme_files* are validated against the subroutine + signatures in *source_files* (the existing scheme-side check). + ``type = host`` and ``type = ddt`` tables in *host_files* are + additionally validated against module-level decls and derived-type + definitions in those same source files. ``type = control`` tables + are silent-skipped (no Fortran source backs control vars — they are + framework-injected at the cap call sites). A ``type = scheme`` table + appearing in *host_files* is a hard error: schemes must be passed via + *scheme_files* so the validator can find the per-phase subroutines. When *source_files* is ``None`` or empty, the validator resolves the - Fortran source for each scheme automatically using the ``source_path`` - attribute from the metadata (defaulting to the ``.meta`` file's directory - if ``source_path`` is absent). Pass an explicit list to override. + Fortran source for each scheme / host / ddt table automatically using + the ``source_path`` attribute from the metadata (defaulting to the + ``.meta`` file's directory if ``source_path`` is absent). Pass an + explicit list to override. Parameters ---------- @@ -972,6 +1431,10 @@ def validate( source_files : list of str, optional Explicit Fortran source files to scan. If omitted, auto-discovered via ``source_path`` in the metadata. + host_files : list of str, optional + Paths to host ``.meta`` files (``type = host`` / ``type = ddt`` / + ``type = control``). Defaults to an empty list (scheme-only + validation). logger : Logger, optional Returns @@ -986,21 +1449,88 @@ def validate( """ log = logger or _LOGGER + scheme_files = list(scheme_files or []) + host_files = list(host_files or []) + + # At least one of --scheme-files / --host-files must be supplied; + # otherwise the validator has nothing to do and would silently report + # "Validation passed." That was the old scheme-only behaviour with + # host metadata accidentally passed in; the new contract requires + # the caller to opt in to one side or the other (or both). + if not scheme_files and not host_files: + raise CCPPError( + "ccpp_validator requires at least one of --scheme-files or " + "--host-files; neither was supplied." + ) + log.info("Loading scheme metadata from %d file(s)", len(scheme_files)) - all_tables = [] + scheme_tables = [] for fpath in scheme_files: - all_tables.extend(parse_metadata_file(fpath)) - scheme_store = SchemeStore.build_from(all_tables) + scheme_tables.extend(parse_metadata_file(fpath)) + + # Reject host / control / suite tables passed via --scheme-files. + # ``type = ddt`` IS allowed alongside scheme tables — schemes + # routinely co-locate their own derived-type definitions in the + # same .meta file (e.g. radiation schemes defining their internal + # ty_rad_lw / ty_rad_sw DDTs). Such DDTs go through the same + # ``_validate_ddt_table`` pass as host-side DDTs. Symmetric to + # the rejection of scheme tables in --host-files (see below): each + # CLI flag has a single, narrow responsibility so misclassified + # host / control / suite .meta files fail fast with a clear pointer. + scheme_nonscheme_violations = [ + t for t in scheme_tables + if t.table_type in ('host', 'control', 'suite') + ] + if scheme_nonscheme_violations: + details = sorted({ + "{} (type = {})".format(t.table_name, t.table_type) + for t in scheme_nonscheme_violations + }) + raise CCPPError( + "Only type = scheme and type = ddt tables may appear in " + "--scheme-files; host / control / suite tables must be " + "passed via --host-files instead. Offending tables: {}".format( + details, + ) + ) + + scheme_store = SchemeStore.build_from(scheme_tables) log.info("Found %d scheme(s): %s", len(scheme_store.scheme_names()), scheme_store.scheme_names()) + # Collect DDT tables that travelled in via --scheme-files; they get + # the same per-component validation as host-side DDTs. + scheme_ddt_tables = [t for t in scheme_tables if t.table_type == 'ddt'] + + host_tables = [] + if host_files: + log.info("Loading host metadata from %d file(s)", len(host_files)) + for fpath in host_files: + host_tables.extend(parse_metadata_file(fpath)) + + # Reject scheme tables passed via --host-files: they wouldn't get + # phase-aware validation, and silent acceptance would hide a real + # mistake. Fail fast before any per-table check runs. + host_scheme_violations = [ + t for t in host_tables if t.is_scheme + ] + if host_scheme_violations: + names = sorted({t.table_name for t in host_scheme_violations}) + raise CCPPError( + "type = scheme tables may not appear in --host-files; pass " + "them via --scheme-files instead. Offending tables: {}".format( + names, + ) + ) + if source_files: log.info("Scanning %d explicit Fortran source file(s)", len(source_files)) resolved_sources = list(source_files) else: - # Auto-discover Fortran files via source_path in each scheme table. + # Auto-discover Fortran files via source_path in each scheme + + # host / ddt table (control tables have no Fortran source). resolved_sources = [] - for tbl in all_tables: - if not tbl.is_scheme: + for tbl in scheme_tables: + if tbl.table_type == 'control': continue fort = _fortran_file_for_table(tbl) if fort: @@ -1008,16 +1538,59 @@ def validate( log.debug("Resolved Fortran source for '%s': %s", tbl.table_name, fort) else: log.warning( - "No Fortran source found for scheme '%s' (source_path='%s')", - tbl.table_name, tbl.source_path, + "No Fortran source found for %s '%s' (source_path='%s')", + tbl.table_type, tbl.table_name, tbl.source_path, + ) + for tbl in host_tables: + if tbl.table_type == 'control': + continue + fort = _fortran_file_for_table(tbl) + if fort: + resolved_sources.append(fort) + log.debug( + "Resolved Fortran source for host/ddt table '%s': %s", + tbl.table_name, fort, + ) + else: + log.warning( + "No Fortran source found for %s table '%s' " + "(source_path='%s')", + tbl.table_type, tbl.table_name, tbl.source_path, ) subroutine_tree = _load_source_tree(resolved_sources) log.info("Found %d subroutine definitions", len(subroutine_tree)) + modules_tree: Dict[str, _ModuleSig] = {} + ddt_index: Dict[str, Dict[str, _ArgAttrs]] = {} + # DDT validation needs the module/type parse whenever any DDT table is + # in play — scheme-co-located DDTs count too. + if host_tables or scheme_ddt_tables: + modules_tree, ddt_index = _load_modules_tree(resolved_sources) + log.info( + "Found %d module(s) and %d derived-type definition(s)", + len(modules_tree), len(ddt_index), + ) + all_errors: List[str] = [] for sname in scheme_store.scheme_names(): - errs = _validate_scheme(sname, scheme_store, subroutine_tree, log) - all_errors.extend(errs) + all_errors.extend( + _validate_scheme(sname, scheme_store, subroutine_tree, log) + ) + # Scheme-co-located DDTs validate the same way as host-side DDTs. + for tbl in scheme_ddt_tables: + all_errors.extend(_validate_ddt_table(tbl, ddt_index, log)) + for tbl in host_tables: + if tbl.table_type == 'host': + all_errors.extend(_validate_host_table(tbl, modules_tree, log)) + elif tbl.table_type == 'ddt': + all_errors.extend(_validate_ddt_table(tbl, ddt_index, log)) + elif tbl.table_type == 'control': + log.info( + "Skipping control table '%s' (no Fortran source backs " + "control variables)", tbl.table_name, + ) + # scheme tables were already rejected above; suite tables aren't + # expected here but would be silent-skipped by omission. if all_errors: log.warning("%d validation error(s) found.", len(all_errors)) @@ -1037,9 +1610,31 @@ def _build_parser() -> argparse.ArgumentParser: ) parser.add_argument( '--scheme-files', - required=True, + required=False, + default='', metavar='FILE[,FILE...]', - help='Comma-separated scheme metadata (.meta) files', + help=( + 'Comma-separated scheme metadata (.meta) files. May contain ' + 'type = scheme tables and type = ddt tables (schemes routinely ' + "co-locate their own DDTs in the same .meta file); host / " + 'control / suite tables are rejected (pass them via ' + '--host-files instead). At least one of --scheme-files or ' + '--host-files must be supplied.' + ), + ) + parser.add_argument( + '--host-files', + required=False, + default='', + metavar='FILE[,FILE...]', + help=( + 'Comma-separated host metadata (.meta) files. ' + 'type = host / type = ddt tables in these files are validated ' + 'against module-level decls and derived-type definitions in ' + 'the same --source-files. type = control is silent-skipped ' + '(no Fortran source backs control vars). type = scheme is ' + 'rejected (pass via --scheme-files instead).' + ), ) parser.add_argument( '--source-files', @@ -1123,9 +1718,10 @@ def main(argv: Optional[List[str]] = None) -> int: scheme_files = [f.strip() for f in args.scheme_files.split(',') if f.strip()] source_files = [f.strip() for f in args.source_files.split(',') if f.strip()] + host_files = [f.strip() for f in args.host_files.split(',') if f.strip()] try: - errors = validate(scheme_files, source_files) + errors = validate(scheme_files, source_files, host_files=host_files) except CCPPError as exc: _LOGGER.error("%s", exc) return 2 diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 5d7b393d..2f2b1b42 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -254,6 +254,58 @@ def _apply_transform_formula(formula_fn, var_expr: str, kind: str) -> str: return formula_fn().format(kind=kind_suffix, var=var_expr) +#: Map from CCPP metadata ``type =`` to the Fortran kind-cast intrinsic. +#: Only the numeric types that legitimately carry distinct kinds in +#: CCPP physics are mapped; everything else (logical, character, DDT) +#: is rejected by :func:`_kind_cast_expr` because either the kind itself +#: is dimensioned differently (character ``len=``) or kinds don't apply. +_KIND_CAST_INTRINSIC = { + 'real': 'real', + 'integer': 'int', + 'complex': 'cmplx', +} + + +def _kind_cast_expr( + var_type: str, + var_expr: str, + target_kind: str, + local: str, + std_name: str, + scheme_name: str, +) -> str: + """Build a Fortran expression that casts *var_expr* to *target_kind*. + + Used when host and scheme metadata differ in ``kind`` only -- no unit + conversion, no vertical flip -- so the transformation temporary + needs an explicit precision cast. Without this the temp would be + declared but never assigned (see :func:`_resolve_one_arg`). + + Returns a Fortran expression like ``real(con_pi, kind=kind_phys)``. + + Raises + ------ + CCPPError + When *var_type* is not one of the numeric types listed in + :data:`_KIND_CAST_INTRINSIC` (kinds on logical / character are + handled via separate metadata pathways, and DDT kinds don't + apply -- a kind mismatch on those types indicates the metadata + is wrong rather than something the cap should bridge). + """ + intrinsic = _KIND_CAST_INTRINSIC.get(var_type.strip().lower()) + if intrinsic is None: + raise CCPPError( + "Variable '{}' (standard_name='{}', scheme '{}'): host and " + "scheme metadata differ in 'kind' but the variable type " + "'{}' has no defined kind-cast intrinsic. Kind differences " + "are supported for real / integer / complex only; fix the " + "metadata so the kinds match.".format( + local, std_name, scheme_name, var_type, + ) + ) + return '{}({}, kind={})'.format(intrinsic, var_expr, target_kind) + + ######################################################################## # Dimension subscript helpers ######################################################################## @@ -1230,8 +1282,16 @@ def _local_name_conflict( name: str, existing_names: Set[str], ) -> str: - """Return *name* with a numeric suffix if it already exists in *existing_names*.""" - if name not in existing_names: + """Return *name* with a numeric suffix if it already exists in *existing_names*. + + Fortran identifiers are **case-insensitive**, so the collision check + is performed in lowercase: ``cp_l`` and ``CP_l`` are the same name + to the compiler and must not be emitted side-by-side as two locals. + The returned name preserves the input case (so generated source + keeps the metadata's spelling), but callers MUST add the lowercased + name to *existing_names* so subsequent calls see the collision. + """ + if name.lower() not in existing_names: return name # Split on last '_' to find the suffix ('_l' or '_p'). if '_' in name: @@ -1242,7 +1302,7 @@ def _local_name_conflict( n = 2 while True: candidate = '{}_{}{}' .format(base, n, suffix) - if candidate not in existing_names: + if candidate.lower() not in existing_names: return candidate n += 1 @@ -1673,9 +1733,20 @@ def _resolve_one_arg( # ``needs_vert_flip`` is True, so the unit-conversion formula naturally # composes the flip on the host-side RHS. For a pure-flip case (no # unit conversion) we emit a plain copy ``temp = host(...flipped)``. + # For a pure-kind case (host.kind != scheme.kind, no unit conversion, + # no vertical flip), we emit an explicit ``TYPE(host, kind=K)`` cast + # so the temp is actually assigned; without this branch the temp was + # declared and the call site referenced it, but no assignment was + # emitted -- gfortran fell back to implicit typing and yielded a + # garbage / Inf value at runtime. unit_forward = '' if needs_unit and fwd_fn is not None and intent in ('in', 'inout'): unit_forward = _apply_transform_formula(fwd_fn, call_expr, scheme_kind) + elif needs_kind and intent in ('in', 'inout'): + unit_forward = _kind_cast_expr( + scheme_var.type, call_expr, scheme_kind, + local=local, std_name=std_name, scheme_name=scheme_name, + ) elif needs_vert_flip and not needs_unit and intent in ('in', 'inout'): unit_forward = call_expr @@ -1684,23 +1755,30 @@ def _resolve_one_arg( if needs_unit and bwd_fn is not None and intent in ('out', 'inout'): unit_backward_expr = '{}_l'.format(local) unit_backward = _apply_transform_formula(bwd_fn, unit_backward_expr, host_kind) + elif needs_kind and intent in ('out', 'inout'): + unit_backward = _kind_cast_expr( + scheme_var.type, '{}_l'.format(local), host_kind, + local=local, std_name=std_name, scheme_name=scheme_name, + ) elif needs_vert_flip and not needs_unit and intent in ('out', 'inout'): unit_backward = '{}_l'.format(local) needs_transform = needs_unit or needs_kind or needs_vert_flip # ---- local variable names (transformation temp + pointer) ------------ + # ``used_local_names`` stores the LOWERCASED names so collision + # detection is Fortran-case-insensitive (see _local_name_conflict). temp_name = '' ptr_name = '' if needs_transform: candidate = '{}_l'.format(local) temp_name = _local_name_conflict(candidate, used_local_names) - used_local_names.add(temp_name) + used_local_names.add(temp_name.lower()) if optional: candidate = '{}_p'.format(local) ptr_name = _local_name_conflict(candidate, used_local_names) - used_local_names.add(ptr_name) + used_local_names.add(ptr_name.lower()) # ---- transform case -------------------------------------------------- if optional and needs_transform: diff --git a/capgen-ng/metadata/metadata_table.py b/capgen-ng/metadata/metadata_table.py index ab6e3285..568c73e3 100644 --- a/capgen-ng/metadata/metadata_table.py +++ b/capgen-ng/metadata/metadata_table.py @@ -1391,16 +1391,27 @@ def flush_table_props() -> None: for key, val in pairs: if current_section is not None: sec_type = current_section.section_type - if sec_type == SCHEME_TABLE_TYPE and key == 'active': + # ``active`` is a host-model attribute: it expresses a + # condition (referencing host standard names) under + # which a host-owned variable is valid storage. It + # belongs only on host and ddt tables. control vars + # are unconditionally framework-injected, suite tables + # are generated wholesale, and scheme args are never + # the originating storage — so all three reject it. + if (sec_type not in ('host', 'ddt') + and key == 'active'): raise ParseSyntaxError( - "'active' is a host-model-only attribute and cannot " - "appear in scheme metadata", + "'active' is a host-model attribute and may " + "only appear in host or ddt metadata; not " + "valid for {} tables".format(sec_type), token=key, context=ctx(lineno) ) - if sec_type in ('host', 'control', 'ddt') and key == 'optional': + if (sec_type != SCHEME_TABLE_TYPE + and key in ('intent', 'optional')): raise ParseSyntaxError( - "'optional' is a scheme-only attribute and cannot " - "appear in host, control, or ddt metadata", + "'{}' is a scheme-only attribute and cannot " + "appear in host, control, ddt, or suite " + "metadata".format(key), token=key, context=ctx(lineno) ) if (sec_type != SCHEME_TABLE_TYPE and diff --git a/doc/migration.md b/doc/migration.md index c9333e43..b2f48984 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -934,7 +934,9 @@ Full reference: `doc/auto_clone_constituents.md`. E2e fixture: ## 7. Validator `capgen-ng/ccpp_validator.py` — standalone Fortran-vs-metadata checker. -Validates **scheme** metadata against scheme Fortran files. +Validates **scheme** metadata against scheme Fortran files, and (since +2026-06-01) **host** and **DDT** metadata against host module-level +declarations and derived-type definitions. ### 7.1 What the validator checks @@ -989,13 +991,39 @@ metadata declares many, the "Argument count mismatch" error appends a HINT pointing at the parser rather than masquerading as a real mismatch — common cause is an unsupported signature feature. -### 7.4 Known gap +### 7.4 Host and DDT metadata validation (2026-06-01) -Host-metadata validation is not yet implemented. When invoked with -non-scheme `.meta` files, the validator silently filters to zero -schemes and reports "Validation passed." Slated for revisit after -the e2e test suite settles (`unit_conv` + `variable_transform` -complete). See `project_validator_host_check_deferred.md` (memory). +Pass `--host-files` to validate `type = host` and `type = ddt` tables +against module-level declarations and derived-type definitions in the +same `--source-files` Fortran tree: + +``` +ccpp_validator.py \ + --scheme-files scheme1.meta,scheme2.meta \ + --host-files host.meta,physics_types.meta \ + --source-files scheme1.F90,scheme2.F90,host.F90,physics_types.F90 +``` + +Per-table behaviour: + +| Table type | Check | +|---|---| +| `type = host` | For each variable, find a module-level declaration of the same `local_name` (lowercased; subscripted spellings like `tk(:,:)` strip to `tk`) in the Fortran module named by `module_name` (or `table_name` when not overridden). Compare type / kind / rank using the same rules as the scheme-side check (intent is silently ignored — host vars carry none). Missing module or missing variable → clear error. | +| `type = ddt` | For each component, find a matching member of the Fortran derived type whose name equals `table_name` (the DDT name). The type definition may live in any parsed module (a flat cross-file index is built from `--source-files`). Same type / kind / rank rules as host vars. | +| `type = control` | Silent skip with an INFO log line — control vars are framework-injected at the cap call sites, no host Fortran backs them. | +| `type = scheme` in `--host-files` | Hard error — schemes must be passed via `--scheme-files` so the validator can find the per-phase subroutines. | +| `type = host` / `control` / `suite` in `--scheme-files` | Hard error — symmetric to the rule above. Misclassified `.meta` files fail fast with a pointer at the correct flag. **`type = ddt` is allowed in `--scheme-files`**: schemes routinely co-locate their own derived-type definitions (e.g. radiation schemes carrying `ty_rad_lw` / `ty_rad_sw` in the same `.meta` as the scheme phase blocks). Scheme-co-located DDTs go through the same per-component validation as host-side DDTs. | + +**Inputs contract.** At least one of `--scheme-files` or `--host-files` +must be supplied. Passing neither raises a clear error rather than +the older silent "Validation passed." Either flag alone is fine; +both together is the common case. + +The same per-attribute rules apply that the scheme-side check uses, +which is one rule fewer than the scheme side: there is no `optional` +flag on module vars / DDT components, and host metadata carries no +`intent`, so the asymmetric-optional rule (§7.2) does not apply here. +Character `len=*` remains a wildcard against any concrete `len=N`. --- @@ -1004,7 +1032,7 @@ complete). See `project_validator_host_check_deferred.md` (memory). | Item | Status | |--------------------------------------------|-----------------------------------------------| | `ccpp_loop_counter` standard name inside nested subcycles | Maps to OUTERMOST loop var. None of cam-sima uses this; revisit if a scheme needs the innermost value. | -| Validator host-metadata check | Deferred; revisit after e2e tests stabilize. | +| Validator host-metadata check | **Landed 2026-06-01**: pass `--host-files`; see §7.4. | | Constituents overhaul (Class A/B + setters) | Discussion doc at `doc/constituents_overhaul.md`. | | Framework setters: `set_advected`, `set_diagnostic_name`, `set_default_value` | Deferred; depends on constituents-overhaul decision. | | Codegen-time scheme-registration cross-check | Deferred; would require new `registers_std_names` metadata attr. | diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index c23c539b..52640900 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -1377,9 +1377,13 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` - **Framework setter additions** — `set_advected`, `set_diagnostic_name`, `set_default_value`, possibly `set_mixing_ratio_type`. Coordinated with the overhaul. -- **Validator host-metadata check** — `ccpp_validator.py` is currently - scheme-only; revisit after the e2e test suite settles. See - `project_validator_host_check_deferred.md` (memory). +- ~~**Validator host-metadata check**~~ — **Landed 2026-06-01**: + `ccpp_validator.py --host-files` validates `type=host` and `type=ddt` + tables against module-level decls and derived-type definitions in + the same `--source-files` Fortran tree. `type=control` is silent- + skipped; `type=scheme` in `--host-files` is a hard error. Per-arg + type/kind/rank checks reuse `_check_arg_attributes`. See + `doc/migration.md` §7.4. - **Codegen-time scheme-registration cross-check** — new metadata attr `registers_std_names = a, b, c` on register-phase tables; replaces current runtime `int_unassigned` check with codegen-time error. diff --git a/end-to-end-tests/advection/CMakeLists.txt b/end-to-end-tests/advection/CMakeLists.txt index 6f10b58b..2b69dda0 100644 --- a/end-to-end-tests/advection/CMakeLists.txt +++ b/end-to-end-tests/advection/CMakeLists.txt @@ -21,10 +21,12 @@ list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_F # Run ccpp_validator ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${SCHEME_FORTRAN_FILES} - METADATA_FILES ${SCHEME_METADATA_FILES}) + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} - METADATA_FILES ${HOST_METADATA_FILES}) + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") # Enable trace output in auto-generated caps set(CCPP_TRACE ON) diff --git a/end-to-end-tests/advection_auto_clone/CMakeLists.txt b/end-to-end-tests/advection_auto_clone/CMakeLists.txt index c04bd7cc..30884200 100644 --- a/end-to-end-tests/advection_auto_clone/CMakeLists.txt +++ b/end-to-end-tests/advection_auto_clone/CMakeLists.txt @@ -28,10 +28,12 @@ set(EXTRA_FLAGS ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${SCHEME_FORTRAN_FILES} METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME" EXTRA_FLAGS ${EXTRA_FLAGS}) ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST" EXTRA_FLAGS ${EXTRA_FLAGS}) # Enable trace output in auto-generated caps diff --git a/end-to-end-tests/capgen_ng/CMakeLists.txt b/end-to-end-tests/capgen_ng/CMakeLists.txt index 74f57bbd..04411444 100644 --- a/end-to-end-tests/capgen_ng/CMakeLists.txt +++ b/end-to-end-tests/capgen_ng/CMakeLists.txt @@ -32,10 +32,12 @@ list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) # Run ccpp_validator ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${SCHEME_FORTRAN_FILES} - METADATA_FILES ${SCHEME_METADATA_FILES}) + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} - METADATA_FILES ${HOST_METADATA_FILES}) + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") # Run ccpp_capgen_ng ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} diff --git a/end-to-end-tests/chunked_data/CMakeLists.txt b/end-to-end-tests/chunked_data/CMakeLists.txt index 894e6833..85124125 100644 --- a/end-to-end-tests/chunked_data/CMakeLists.txt +++ b/end-to-end-tests/chunked_data/CMakeLists.txt @@ -21,10 +21,12 @@ list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) # Run ccpp_validator ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${SCHEME_FORTRAN_FILES} - METADATA_FILES ${SCHEME_METADATA_FILES}) + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} - METADATA_FILES ${HOST_METADATA_FILES}) + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") # Run ccpp_capgen_ng ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} diff --git a/end-to-end-tests/cmake/ccpp_capgen.cmake b/end-to-end-tests/cmake/ccpp_capgen.cmake index 76bca88f..8647a851 100644 --- a/end-to-end-tests/cmake/ccpp_capgen.cmake +++ b/end-to-end-tests/cmake/ccpp_capgen.cmake @@ -1,10 +1,22 @@ # CMake wrapper for ccpp_validator.py # # SOURCE_FILES - CMake list of Fortran source files -# METADATA_FILES - CMake list of corresponding metadata files +# METADATA_FILES - CMake list of scheme metadata files. +# TYPE - Type of metadata: SCHEME or HOST. +# SCHEME metadata is validated against +# the per-phase subroutine signatures +# in SOURCE_FILES. type=scheme and +# type=ddt tables are validated; types +# control, host, suite are hard errors. +# HOST metadata: type=host and type=ddt +# tables get module-level/derived-type +# validation against SOURCE_FILES; +# type=control is silent-skipped; +# type=scheme is rejected as a hard error. +# function(ccpp_validator) set(optionalArgs) - set(oneValueArgs VERBOSITY) + set(oneValueArgs VERBOSITY TYPE) set(multi_value_keywords SOURCE_FILES METADATA_FILES EXTRA_FLAGS) cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN}) @@ -22,10 +34,24 @@ function(ccpp_validator) list(APPEND CCPP_VALIDATOR_CMD_LIST "--source-files" "${SOURCE_FILES_SEPARATED}") if(NOT DEFINED arg_METADATA_FILES) - message(FATAL_ERROR "function(ccpp_capgen): METADATA_FILES not set.") + message(FATAL_ERROR "function(ccpp_validator): METADATA_FILES not set.") endif() list(JOIN arg_METADATA_FILES "," METADATA_FILES_SEPARATED) - list(APPEND CCPP_VALIDATOR_CMD_LIST "--scheme-files" "${METADATA_FILES_SEPARATED}") + + if(NOT DEFINED arg_TYPE) + message(FATAL_ERROR "function(ccpp_validator): TYPE must be HOST or SCHEME") + endif() + string(TOUPPER "${arg_TYPE}" _type) + if(NOT (_type MATCHES "^(HOST|SCHEME)$")) + message(FATAL_ERROR "function(ccpp_validator): TYPE must be HOST or SCHEME") + endif() + + if(_type MATCHES "^HOST$") + list(APPEND CCPP_VALIDATOR_CMD_LIST "--host-files" "${METADATA_FILES_SEPARATED}") + endif() + if(_type MATCHES "^SCHEME$") + list(APPEND CCPP_VALIDATOR_CMD_LIST "--scheme-files" "${METADATA_FILES_SEPARATED}") + endif() if(DEFINED arg_VERBOSITY) string(REPEAT "--verbose " ${arg_VERBOSITY} VERBOSE_PARAMS_SEPARATED) diff --git a/end-to-end-tests/ddthost/CMakeLists.txt b/end-to-end-tests/ddthost/CMakeLists.txt index a07d795c..aaa10fdc 100644 --- a/end-to-end-tests/ddthost/CMakeLists.txt +++ b/end-to-end-tests/ddthost/CMakeLists.txt @@ -21,10 +21,12 @@ list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) # Run ccpp_validator ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${SCHEME_FORTRAN_FILES} - METADATA_FILES ${SCHEME_METADATA_FILES}) + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} - METADATA_FILES ${HOST_METADATA_FILES}) + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") # Run ccpp_capgen_ng ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} diff --git a/end-to-end-tests/ddthost/host_ccpp_ddt.F90 b/end-to-end-tests/ddthost/host_ccpp_ddt.F90 index b60c81af..d9427c74 100644 --- a/end-to-end-tests/ddthost/host_ccpp_ddt.F90 +++ b/end-to-end-tests/ddthost/host_ccpp_ddt.F90 @@ -9,7 +9,7 @@ module host_ccpp_ddt type, public :: ccpp_info_t integer :: col_start ! horizontal_loop_begin integer :: col_end ! horizontal_loop_end - character(len=512) :: errmsg ! ccpp_error_message + character(len=256) :: errmsg ! ccpp_error_message integer :: errflg ! ccpp_error_code end type ccpp_info_t diff --git a/end-to-end-tests/instances/CMakeLists.txt b/end-to-end-tests/instances/CMakeLists.txt index 1b995067..f64ebf24 100644 --- a/end-to-end-tests/instances/CMakeLists.txt +++ b/end-to-end-tests/instances/CMakeLists.txt @@ -21,10 +21,12 @@ list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) # Run ccpp_validator ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${SCHEME_FORTRAN_FILES} - METADATA_FILES ${SCHEME_METADATA_FILES}) + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} - METADATA_FILES ${HOST_METADATA_FILES}) + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") # Run ccpp_capgen_ng ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} diff --git a/end-to-end-tests/instances_advection/CMakeLists.txt b/end-to-end-tests/instances_advection/CMakeLists.txt index 25bc8687..698050e3 100644 --- a/end-to-end-tests/instances_advection/CMakeLists.txt +++ b/end-to-end-tests/instances_advection/CMakeLists.txt @@ -20,12 +20,15 @@ list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) +# Run ccpp_validator ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${SCHEME_FORTRAN_FILES} - METADATA_FILES ${SCHEME_METADATA_FILES}) + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} - METADATA_FILES ${HOST_METADATA_FILES}) + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} diff --git a/end-to-end-tests/nested_suite/CMakeLists.txt b/end-to-end-tests/nested_suite/CMakeLists.txt index f5f21345..bd449ba7 100644 --- a/end-to-end-tests/nested_suite/CMakeLists.txt +++ b/end-to-end-tests/nested_suite/CMakeLists.txt @@ -21,10 +21,12 @@ list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) # Run ccpp_validator ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${SCHEME_FORTRAN_FILES} - METADATA_FILES ${SCHEME_METADATA_FILES}) + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} - METADATA_FILES ${HOST_METADATA_FILES}) + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") # Run ccpp_capgen_ng ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} diff --git a/end-to-end-tests/nested_suite/test_host_data.meta b/end-to-end-tests/nested_suite/test_host_data.meta index fd5c009c..2154b797 100644 --- a/end-to-end-tests/nested_suite/test_host_data.meta +++ b/end-to-end-tests/nested_suite/test_host_data.meta @@ -42,7 +42,6 @@ dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys - intent = in active = (flag_indicating_cloud_microphysics_has_graupel) [nci] standard_name = cloud_ice_number_concentration @@ -51,7 +50,6 @@ dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys - intent = in active = (flag_indicating_cloud_microphysics_has_ice) [scalar_var] standard_name = scalar_variable_for_testing diff --git a/end-to-end-tests/opt_arg/CMakeLists.txt b/end-to-end-tests/opt_arg/CMakeLists.txt index 66d27712..041f9bbb 100644 --- a/end-to-end-tests/opt_arg/CMakeLists.txt +++ b/end-to-end-tests/opt_arg/CMakeLists.txt @@ -21,10 +21,12 @@ list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) # Run ccpp_validator ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${SCHEME_FORTRAN_FILES} - METADATA_FILES ${SCHEME_METADATA_FILES}) + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} - METADATA_FILES ${HOST_METADATA_FILES}) + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") # Run ccpp_capgen_ng. Override kind_phys to REAL32 so the whole test # runs in single precision; exercises the --kind-type plumbing. diff --git a/end-to-end-tests/var_compat/CMakeLists.txt b/end-to-end-tests/var_compat/CMakeLists.txt index 6a31d273..5e006e13 100644 --- a/end-to-end-tests/var_compat/CMakeLists.txt +++ b/end-to-end-tests/var_compat/CMakeLists.txt @@ -21,10 +21,12 @@ list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) # Run ccpp_validator ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${SCHEME_FORTRAN_FILES} - METADATA_FILES ${SCHEME_METADATA_FILES}) + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} SOURCE_FILES ${HOST_FORTRAN_FILES} - METADATA_FILES ${HOST_METADATA_FILES}) + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") # Run ccpp_capgen_ng ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} diff --git a/end-to-end-tests/var_compat/test_host_data.meta b/end-to-end-tests/var_compat/test_host_data.meta index 65ce3d9d..ec691215 100644 --- a/end-to-end-tests/var_compat/test_host_data.meta +++ b/end-to-end-tests/var_compat/test_host_data.meta @@ -50,7 +50,6 @@ dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys - intent = in active = (flag_indicating_cloud_microphysics_has_graupel) [nci] standard_name = cloud_ice_number_concentration @@ -59,7 +58,6 @@ dimensions = (horizontal_dimension,vertical_layer_dimension) type = real kind = kind_phys - intent = in active = (flag_indicating_cloud_microphysics_has_ice) [scalar_var] standard_name = scalar_variable_for_testing diff --git a/unit-tests/test_metadata_table.py b/unit-tests/test_metadata_table.py index 83bdb416..fe5e282b 100644 --- a/unit-tests/test_metadata_table.py +++ b/unit-tests/test_metadata_table.py @@ -1219,6 +1219,114 @@ def test_active_expression_is_lowercased(self): ) self.assertEqual(x_var.active, '(flag_for_aerosol_input_mg_radiation)') + def test_intent_in_host_raises(self): + """'intent' on a host table is now rejected (scheme-only).""" + text = """ + [ccpp-table-properties] + name = my_host + type = host + + [ccpp-arg-table] + name = my_host + type = host + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + intent = in + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_intent_in_control_raises(self): + """'intent' on a control table is now rejected (scheme-only).""" + text = """ + [ccpp-table-properties] + name = my_ctrl + type = control + + [ccpp-arg-table] + name = my_ctrl + type = control + [ x ] + standard_name = foo + units = 1 + dimensions = () + type = integer + intent = in + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_intent_in_ddt_raises(self): + """'intent' on a ddt table is rejected (scheme-only).""" + text = """ + [ccpp-table-properties] + name = my_ddt_type + type = ddt + + [ccpp-arg-table] + name = my_ddt_type + type = ddt + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + intent = inout + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_active_in_control_raises(self): + """'active' on a control table is rejected — control vars are + unconditionally framework-injected, no active expression makes + sense for them.""" + text = """ + [ccpp-table-properties] + name = my_ctrl + type = control + + [ccpp-arg-table] + name = my_ctrl + type = control + [ x ] + standard_name = foo + units = 1 + dimensions = () + type = integer + active = my_flag + """ + with self.assertRaises((CCPPError, ParseSyntaxError)): + _parse_text(text) + + def test_active_in_ddt_allowed(self): + """'active' on a ddt table is allowed — a host DDT component is a + valid origin for the active-conditional storage contract.""" + text = """ + [ccpp-table-properties] + name = my_ddt_type + type = ddt + + [ccpp-arg-table] + name = my_ddt_type + type = ddt + [ flag ] + standard_name = my_flag + units = flag + dimensions = () + type = logical + [ x ] + standard_name = foo + units = m + dimensions = () + type = real + active = my_flag + """ + tables = _parse_text(text) + self.assertEqual(len(tables), 1) + def test_optional_in_scheme_allowed(self): """'optional' attribute in scheme metadata is valid.""" text = """ diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 5bab24a3..74625004 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -495,7 +495,7 @@ def _build_dict(self): "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" "[ mythread ]\n standard_name = thread_number\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" ) return build_flat_host_dict( _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), @@ -601,13 +601,13 @@ def test_explicit_index_substitutes_scalar_idx_placeholder(self): "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ instance ]\n standard_name = instance_number\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ ninstances ]\n standard_name = number_of_instances\n" - " units = count\n dimensions = ()\n type = integer\n intent = in\n" + " units = count\n dimensions = ()\n type = integer\n" ) hd = build_flat_host_dict( _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), @@ -681,17 +681,17 @@ def test_explicit_index_nested_ddt_two_placeholder_levels(self): "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ mythread ]\n standard_name = thread_number\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ nthreads ]\n standard_name = number_of_threads\n" - " units = count\n dimensions = ()\n type = integer\n intent = in\n" + " units = count\n dimensions = ()\n type = integer\n" "[ instance ]\n standard_name = instance_number\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ ninstances ]\n standard_name = number_of_instances\n" - " units = count\n dimensions = ()\n type = integer\n intent = in\n" + " units = count\n dimensions = ()\n type = integer\n" ) hd = build_flat_host_dict( _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), @@ -745,13 +745,13 @@ def test_explicit_index_with_literal_local_subscript(self): "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ instance ]\n standard_name = instance_number\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ ninstances ]\n standard_name = number_of_instances\n" - " units = count\n dimensions = ()\n type = integer\n intent = in\n" + " units = count\n dimensions = ()\n type = integer\n" ) hd = build_flat_host_dict( _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), @@ -797,13 +797,13 @@ def test_multiple_explicit_index_tokens_each_with_placeholder(self): "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" "[ lb ]\n standard_name = horizontal_loop_begin\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ ub ]\n standard_name = horizontal_loop_end\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ instance ]\n standard_name = instance_number\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ ninstances ]\n standard_name = number_of_instances\n" - " units = count\n dimensions = ()\n type = integer\n intent = in\n" + " units = count\n dimensions = ()\n type = integer\n" ) hd = build_flat_host_dict( _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), @@ -953,9 +953,9 @@ def test_literal_subscript_in_local_name_preserved(self): "[ccpp-table-properties]\n name = ctrl_mod\n type = control\n" "[ccpp-arg-table]\n name = ctrl_mod\n type = control\n" "[ instance ]\n standard_name = instance_number\n units = index\n" - " dimensions = ()\n type = integer\n intent = in\n" + " dimensions = ()\n type = integer\n" "[ ninstances ]\n standard_name = number_of_instances\n" - " units = count\n dimensions = ()\n type = integer\n intent = in\n" + " units = count\n dimensions = ()\n type = integer\n" ) hd = build_flat_host_dict( _parse_lines(host_src.splitlines(keepends=True), 'host.meta'), @@ -1076,6 +1076,25 @@ def test_conflict_adds_3(self): _local_name_conflict('phii_l', {'phii_l', 'phii_2_l'}), 'phii_3_l' ) + def test_case_insensitive_conflict(self): + """Fortran identifiers are case-insensitive: ``CP_l`` must be + treated as already-used when ``cp_l`` is in the existing set + (and vice versa). Regression: a HAFS_v0_hwrf_phys_ts cap + emitted both ``cp_l`` and ``CP_l`` side-by-side because the + check was string-equal rather than case-insensitive.""" + # Lower-then-upper. + self.assertEqual( + _local_name_conflict('CP_l', {'cp_l'}), 'CP_2_l', + ) + # Upper-then-lower. + self.assertEqual( + _local_name_conflict('cp_l', {'cp_l'}), 'cp_2_l', + ) + # Mixed-case existing entry too. + self.assertEqual( + _local_name_conflict('cp_l', {'Cp_L'.lower()}), 'cp_2_l', + ) + ######################################################################## # Tests: _resolve_one_arg (single argument) @@ -1860,6 +1879,95 @@ def test_len_star_host_specific_scheme_raises(self): self.assertIn('len=256', str(cm.exception)) +######################################################################## +# Tests: pure-kind transform (real-kind cast) +######################################################################## + +class TestPureKindTransform(unittest.TestCase): + """When host and scheme metadata differ in *kind* only (no unit + mismatch, no vertical flip), the resolver must emit a real/int kind + cast as ``unit_forward``. Without this the cap declares the + transformation temporary but never assigns to it -- gfortran falls + back to implicit typing at the call site and the call sees garbage + / Inf. Regression: SCM_GFS_v17_p8 / bomex started failing with + ``alon = -Infinity`` in ``setclimaer`` after the host changed + ``scm_physical_constants`` from ``kind = kind_phys`` to + ``kind = dp`` -- a pure-kind mismatch against the GFS_rrtmg_pre + scheme args, which expect ``kind_phys``.""" + + _HOST_SRC_TEMPLATE = ''' +[ccpp-table-properties] + name = phys_const + type = host +[ccpp-arg-table] + name = phys_const + type = host +[ con_pi ] + standard_name = pi + units = none + dimensions = () + type = {type} + kind = {kind} +''' + + def _hd(self, kind='dp', type_='real'): + src = self._HOST_SRC_TEMPLATE.format(kind=kind, type=type_) + return build_flat_host_dict(_parse(src), [], []) + + def _scheme_var_pi(self, intent='in', kind='kind_phys', type_='real'): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar('con_pi', ctx) + v.set_attr('standard_name', 'pi', ctx) + v.set_attr('units', 'none', ctx) + v.set_attr('dimensions', '()', ctx) + v.set_attr('type', type_, ctx) + v.set_attr('kind', kind, ctx) + v.set_attr('intent', intent, ctx) + return v + + def test_real_kind_mismatch_emits_real_cast_forward(self): + hd = self._hd(kind='dp') + scheme = self._scheme_var_pi(intent='in', kind='kind_phys') + arg = _resolve_one_arg(scheme, 'run', hd, {}, 'rrtmg', set()) + self.assertTrue(arg.needs_kind_transform) + self.assertTrue(arg.needs_transform) + self.assertEqual(arg.transform_case, 3) + # Temp must be both NAMED and ASSIGNED (the bug was that the temp + # was named but unit_forward stayed empty, so no assignment was + # emitted by the cap). + self.assertEqual(arg.temp_name, 'con_pi_l') + self.assertEqual(arg.unit_forward, 'real(con_pi, kind=kind_phys)') + + def test_real_kind_mismatch_emits_real_cast_backward_for_inout(self): + hd = self._hd(kind='dp') + scheme = self._scheme_var_pi(intent='inout', kind='kind_phys') + arg = _resolve_one_arg(scheme, 'run', hd, {}, 'rrtmg', set()) + self.assertEqual(arg.unit_forward, 'real(con_pi, kind=kind_phys)') + self.assertEqual(arg.unit_backward, 'real(con_pi_l, kind=dp)') + + def test_integer_kind_mismatch_emits_int_cast(self): + hd = self._hd(kind='int_8', type_='integer') + scheme = self._scheme_var_pi(intent='in', kind='int_4', + type_='integer') + arg = _resolve_one_arg(scheme, 'run', hd, {}, 'rrtmg', set()) + self.assertEqual(arg.unit_forward, 'int(con_pi, kind=int_4)') + + def test_unsupported_type_for_kind_cast_raises(self): + """A kind mismatch on a type without a kind-cast intrinsic (DDT, + logical) should raise a clear CCPPError pointing the user at + the metadata rather than silently emitting unassigned temps.""" + # Use a logical with two different kind names. + hd = self._hd(kind='lk1', type_='logical') + scheme = self._scheme_var_pi(intent='in', kind='lk2', + type_='logical') + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(scheme, 'run', hd, {}, 'rrtmg', set()) + msg = str(cm.exception) + self.assertIn("kind-cast intrinsic", msg) + self.assertIn("logical", msg) + + ######################################################################## # Integration tests: resolve_suite ######################################################################## diff --git a/unit-tests/test_validator.py b/unit-tests/test_validator.py index 677fb38c..2c7f57e2 100644 --- a/unit-tests/test_validator.py +++ b/unit-tests/test_validator.py @@ -9,11 +9,15 @@ import ccpp_validator as val_mod from ccpp_validator import ( + _base_local_name, _join_continuation, - _parse_subroutines, + _load_modules_tree, _load_source_tree, + _parse_modules, + _parse_subroutines, validate, ) +from metadata.parse_tools import CCPPError _SAMPLE_DIR = os.path.join(os.path.dirname(__file__), 'sample_files') _CORRECT_F90 = os.path.join(_SAMPLE_DIR, 'scheme_multipart_correct.F90') @@ -1047,6 +1051,907 @@ def test_warning_and_no_error(self): self.assertIn("Optional Fortran argument 'b'", stream.getvalue()) +class _HostValidationFixture(unittest.TestCase): + """Shared scaffolding for host/ddt validation tests. + + Builds a tmpdir with one host metadata file and one Fortran source. + Subclasses set ``META`` and ``F90`` class attributes. + """ + + META: str = '' + F90: str = '' + META_NAME: str = 'host.meta' + F90_NAME: str = 'host.F90' + + def setUp(self): + self.tmpdir = tempfile.mkdtemp() + self.meta_path = os.path.join(self.tmpdir, self.META_NAME) + self.f90_path = os.path.join(self.tmpdir, self.F90_NAME) + with open(self.meta_path, 'w') as fh: + fh.write(self.META) + with open(self.f90_path, 'w') as fh: + fh.write(self.F90) + + def tearDown(self): + import shutil + shutil.rmtree(self.tmpdir, ignore_errors=True) + + +class TestParseModulesAndDDTs(unittest.TestCase): + """Standalone parser-level tests beyond the doctest example.""" + + def test_module_var_after_use_only(self): + src = textwrap.dedent("""\ + module m + use kinds, only: kind_phys + implicit none + integer :: nlev + real(kind=kind_phys) :: cp + end module m + """) + mods = _parse_modules(src) + self.assertEqual(sorted(mods['m'].vars.keys()), ['cp', 'nlev']) + self.assertEqual(mods['m'].vars['cp'].type_, 'real') + self.assertEqual(mods['m'].vars['cp'].kind_, 'kind_phys') + + def test_subroutine_locals_excluded(self): + src = textwrap.dedent("""\ + module m + integer :: mod_var + contains + subroutine helper(x) + integer, intent(in) :: x + real :: local_only + end subroutine helper + end module m + """) + mods = _parse_modules(src) + self.assertIn('mod_var', mods['m'].vars) + self.assertNotIn('local_only', mods['m'].vars) + self.assertNotIn('x', mods['m'].vars) + + def test_ddt_block_components(self): + src = textwrap.dedent("""\ + module types + type :: physics_t + real(kind=kind_phys) :: tk(:,:) + integer :: nlay + logical :: has_water + end type physics_t + end module types + """) + mods = _parse_modules(src) + comps = mods['types'].ddts['physics_t'] + self.assertEqual(sorted(comps.keys()), ['has_water', 'nlay', 'tk']) + self.assertEqual(comps['tk'].rank, 2) + self.assertEqual(comps['tk'].kind_, 'kind_phys') + self.assertEqual(comps['has_water'].type_, 'logical') + + def test_ddt_with_attrs_on_type_decl(self): + src = textwrap.dedent("""\ + module types + type, public :: opaque_t + integer :: a + end type opaque_t + end module types + """) + mods = _parse_modules(src) + self.assertIn('opaque_t', mods['types'].ddts) + + def test_ddt_index_collects_across_modules(self): + src1 = ('module a\n type :: t1\n integer :: x\n end type t1\n' + 'end module a\n') + src2 = ('module b\n type :: t2\n real :: y\n end type t2\n' + 'end module b\n') + tmp = tempfile.mkdtemp() + try: + p1 = os.path.join(tmp, 'a.F90') + p2 = os.path.join(tmp, 'b.F90') + with open(p1, 'w') as fh: fh.write(src1) + with open(p2, 'w') as fh: fh.write(src2) + modules, ddt_index = _load_modules_tree([p1, p2]) + finally: + import shutil + shutil.rmtree(tmp, ignore_errors=True) + self.assertEqual(sorted(modules.keys()), ['a', 'b']) + self.assertEqual(sorted(ddt_index.keys()), ['t1', 't2']) + + +class TestBaseLocalName(unittest.TestCase): + + def test_plain(self): + self.assertEqual(_base_local_name('foo'), 'foo') + + def test_case_folded(self): + self.assertEqual(_base_local_name('Foo_Bar'), 'foo_bar') + + def test_strips_subscript(self): + self.assertEqual(_base_local_name('tk(:,:)'), 'tk') + + def test_strips_named_subscript(self): + self.assertEqual( + _base_local_name('dqdt(:,:,index_of_cloud)'), 'dqdt', + ) + + +class TestValidateHostCorrect(_HostValidationFixture): + """type=host with matching Fortran module → no errors.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ nlev ] + standard_name = vertical_layer_dimension + long_name = number of layers + units = count + dimensions = () + type = integer + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + integer :: nlev + real(kind=kind_phys) :: cp + end module my_host + """) + + def test_no_errors(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateHostTypeMismatch(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + integer :: cp + end module my_host + """) + + def test_type_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + # Type mismatch surfaces; an incidental kind mismatch may also + # surface (integer Fortran has no kind metadata). + type_errs = [e for e in errs if 'type mismatch' in e] + self.assertEqual(len(type_errs), 1, errs) + self.assertIn("cp", type_errs[0]) + + +class TestValidateHostKindMismatch(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + real :: cp + end module my_host + """) + + def test_kind_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("kind mismatch", errs[0]) + + +class TestValidateHostRankMismatch(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + real(kind=kind_phys) :: tk(:) + end module my_host + """) + + def test_rank_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("rank mismatch", errs[0]) + + +class TestValidateHostMissingVar(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + integer :: nlev + end module my_host + """) + + def test_missing_decl_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("not found", errs[0]) + self.assertIn("cp", errs[0]) + + +class TestValidateHostMissingModule(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module different_module + implicit none + real(kind=kind_phys) :: cp + end module different_module + """) + + def test_module_not_found_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("not found in any source file", errs[0]) + self.assertIn("my_host", errs[0]) + + +class TestValidateHostModuleNameOverride(_HostValidationFixture): + """module_name override in [ccpp-table-properties] honoured.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_table + type = host + module_name = host_state + [ccpp-arg-table] + name = my_table + type = host + [ cp ] + standard_name = specific_heat_of_dry_air_at_constant_pressure + long_name = specific heat + units = J kg-1 K-1 + dimensions = () + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module host_state + implicit none + real(kind=kind_phys) :: cp + end module host_state + """) + + def test_no_errors_with_override(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateDDTCorrect(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_t + type = ddt + [ccpp-arg-table] + name = physics_t + type = ddt + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + [ nlay ] + standard_name = vertical_layer_dimension + long_name = number of layers + units = count + dimensions = () + type = integer + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: physics_t + real(kind=kind_phys) :: tk(:,:) + integer :: nlay + end type physics_t + end module phys_types + """) + + META_NAME = 'physics_t.meta' + F90_NAME = 'phys_types.F90' + + def test_no_errors(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateDDTSlicedLocalName(_HostValidationFixture): + """A sliced metadata local_name expresses a reduced-rank view of a + higher-rank Fortran component. The rank check must consult the + subscript width — not just len(metadata.dimensions) — when computing + the expected Fortran rank. Mirrors the + end-to-end-tests/ddthost/test_host_data shape: rank-3 Fortran ``q`` + bound under TWO standard names — the bare ``q`` (3-D mixing ratio) + and the sliced ``q(:,:,index_of_water_vapor_specific_humidity)`` + (2-D view of one tracer).""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_state + type = ddt + [ccpp-arg-table] + name = physics_state + type = ddt + [ q ] + standard_name = constituent_mixing_ratio + long_name = q + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension, number_of_tracers) + type = real | kind = kind_phys + [ q(:,:,index_of_water_vapor_specific_humidity) ] + standard_name = water_vapor_specific_humidity + long_name = q water vapor + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: physics_state + real(kind=kind_phys), dimension(:, :, :), allocatable :: q + end type physics_state + end module phys_types + """) + + META_NAME = 'physics_state.meta' + F90_NAME = 'phys_types.F90' + + def test_no_errors_for_sliced_and_bare_view(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateHostArrayInitialiserRank(_HostValidationFixture): + """Module-level parameter declarations with an array-constructor + initialiser (``(/ 'a', 'b', 'c' /)``) must not have the + initialiser's commas counted as dimension entries — that + regression made the rank-1 ``std_name_array`` on the + end-to-end-tests/advection fixture look like rank 3. Mirrors + that shape exactly.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = test_host_data + type = host + [ccpp-arg-table] + name = test_host_data + type = host + [ num_consts ] + standard_name = number_of_constituents + units = count + dimensions = () + type = integer + [ std_name_array ] + standard_name = array_of_constituent_standard_names + units = none + dimensions = (number_of_constituents) + type = character | kind = len=32 + """) + + F90 = textwrap.dedent("""\ + module test_host_data + implicit none + integer, public, parameter :: num_consts = 3 + character(len=32), public, parameter :: std_name_array(num_consts) = (/ & + 'specific_humidity ', & + 'cloud_liquid_dry_mixing_ratio', & + 'cloud_ice_dry_mixing_ratio ' /) + end module test_host_data + """) + + META_NAME = 'test_host_data.meta' + F90_NAME = 'test_host_data.F90' + + def test_no_errors(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateDDTSlicedRankStillCaught(_HostValidationFixture): + """A truly wrong rank under a sliced spelling is still caught. The + Fortran ``q`` here is rank 2 instead of the rank-3 the metadata + implies — the subscript declares 3 entries (``(:,:,index_of_X)``) + but Fortran only carries 2.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_state + type = ddt + [ccpp-arg-table] + name = physics_state + type = ddt + [ q(:,:,index_of_water_vapor_specific_humidity) ] + standard_name = water_vapor_specific_humidity + long_name = q water vapor + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: physics_state + real(kind=kind_phys), dimension(:, :), allocatable :: q + end type physics_state + end module phys_types + """) + + META_NAME = 'physics_state.meta' + F90_NAME = 'phys_types.F90' + + def test_rank_mismatch_surfaces(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + rank_errs = [e for e in errs if 'rank mismatch' in e] + self.assertEqual(len(rank_errs), 1, errs) + self.assertIn("q(:,:,index_of_water_vapor_specific_humidity)", + rank_errs[0]) + self.assertIn("Fortran declares rank 2", rank_errs[0]) + + +class TestValidateDDTComponentMismatch(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_t + type = ddt + [ccpp-arg-table] + name = physics_t + type = ddt + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: physics_t + integer :: tk(:,:) + end type physics_t + end module phys_types + """) + + META_NAME = 'physics_t.meta' + F90_NAME = 'phys_types.F90' + + def test_type_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + type_errs = [e for e in errs if 'type mismatch' in e] + self.assertEqual(len(type_errs), 1, errs) + self.assertIn("tk", type_errs[0]) + + +class TestValidateDDTMissingComponent(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_t + type = ddt + [ccpp-arg-table] + name = physics_t + type = ddt + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: physics_t + integer :: nlay + end type physics_t + end module phys_types + """) + + META_NAME = 'physics_t.meta' + F90_NAME = 'phys_types.F90' + + def test_missing_component_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("not found as a component", errs[0]) + self.assertIn("tk", errs[0]) + + +class TestValidateDDTMissingType(_HostValidationFixture): + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = physics_t + type = ddt + [ccpp-arg-table] + name = physics_t + type = ddt + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module phys_types + implicit none + type :: different_type + real(kind=kind_phys) :: tk(:,:) + end type different_type + end module phys_types + """) + + META_NAME = 'physics_t.meta' + F90_NAME = 'phys_types.F90' + + def test_type_not_found_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("not found as a derived-type definition", errs[0]) + self.assertIn("physics_t", errs[0]) + + +class TestValidateControlSilentSkip(_HostValidationFixture): + """type=control tables are silent-skipped (no Fortran backs control vars).""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_control + type = control + [ccpp-arg-table] + name = my_control + type = control + [ suite_name ] + standard_name = suite_name + long_name = suite name + units = none + dimensions = () + type = character | kind = len=* + [ errcode ] + standard_name = ccpp_error_code + long_name = error code + units = 1 + dimensions = () + type = integer + """) + + # F90 is irrelevant — control vars never resolved against source. + F90 = textwrap.dedent("""\ + module placeholder + implicit none + end module placeholder + """) + + def test_no_errors_for_control(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + +class TestValidateSchemeInHostFilesRejected(_HostValidationFixture): + """type=scheme passed via --host-files is a hard error.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ a ] + standard_name = horizontal_dimension + long_name = horizontal dim + units = count + dimensions = () + type = integer + intent = in + """) + + F90 = textwrap.dedent("""\ + module placeholder + implicit none + end module placeholder + """) + + def test_raises_ccpperror(self): + with self.assertRaises(CCPPError) as cm: + validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertIn("--host-files", str(cm.exception)) + self.assertIn("my_scheme", str(cm.exception)) + + +class TestValidateNonSchemeInSchemeFilesRejected(_HostValidationFixture): + """type=host / type=control / type=ddt in --scheme-files is rejected. + + Symmetric to the scheme-in-host-files rejection above: each CLI + flag has a single responsibility so misclassified .meta files fail + fast with a clear pointer at the right flag. + """ + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ nlev ] + standard_name = vertical_layer_dimension + long_name = number of layers + units = count + dimensions = () + type = integer + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + integer :: nlev + end module my_host + """) + + def test_host_in_scheme_files_rejected(self): + with self.assertRaises(CCPPError) as cm: + validate([self.meta_path], [self.f90_path]) + msg = str(cm.exception) + self.assertIn("--scheme-files", msg) + self.assertIn("--host-files", msg) + self.assertIn("my_host", msg) + self.assertIn("type = host", msg) + + def test_control_in_scheme_files_rejected(self): + # Replace the host fixture's meta with a control-typed one and + # check the same rejection path covers control too. + with open(self.meta_path, 'w') as fh: + fh.write(textwrap.dedent("""\ + [ccpp-table-properties] + name = my_control + type = control + [ccpp-arg-table] + name = my_control + type = control + [ suite_name ] + standard_name = suite_name + long_name = suite name + units = none + dimensions = () + type = character | kind = len=* + """)) + with self.assertRaises(CCPPError) as cm: + validate([self.meta_path], [self.f90_path]) + msg = str(cm.exception) + self.assertIn("my_control", msg) + self.assertIn("type = control", msg) + + def test_suite_in_scheme_files_rejected(self): + # type = suite is the third rejected type (host / control / suite). + with open(self.meta_path, 'w') as fh: + fh.write(textwrap.dedent("""\ + [ccpp-table-properties] + name = my_suite + type = suite + [ccpp-arg-table] + name = my_suite + type = suite + """)) + with self.assertRaises(CCPPError) as cm: + validate([self.meta_path], [self.f90_path]) + msg = str(cm.exception) + self.assertIn("my_suite", msg) + self.assertIn("type = suite", msg) + + +class TestValidateSchemeWithCoLocatedDDT(_HostValidationFixture): + """A scheme metadata file may contain its own type = ddt tables — + schemes routinely co-locate the DDTs they define (e.g. radiation + schemes carrying ty_rad_lw / ty_rad_sw type definitions in the same + .meta as the scheme phase blocks). Both the scheme phase signature + AND the DDT components must be validated.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_scheme + type = scheme + [ccpp-arg-table] + name = my_scheme_run + type = scheme + [ workspace ] + standard_name = scheme_workspace + long_name = workspace + units = none + dimensions = () + type = ty_ws + intent = inout + [ errmsg ] + standard_name = ccpp_error_message + long_name = error message + units = none + dimensions = () + type = character | kind = len=* + intent = out + [ errflg ] + standard_name = ccpp_error_code + long_name = error code + units = 1 + dimensions = () + type = integer + intent = out + [ccpp-table-properties] + name = ty_ws + type = ddt + [ccpp-arg-table] + name = ty_ws + type = ddt + [ nlay ] + standard_name = vertical_layer_dimension + long_name = number of layers + units = count + dimensions = () + type = integer + [ tk ] + standard_name = air_temperature + long_name = temperature + units = K + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + """) + + F90 = textwrap.dedent("""\ + module my_scheme + implicit none + type :: ty_ws + integer :: nlay + real(kind=kind_phys) :: tk(:,:) + end type ty_ws + contains + subroutine my_scheme_run(workspace, errmsg, errflg) + type(ty_ws), intent(inout) :: workspace + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + end subroutine my_scheme_run + end module my_scheme + """) + + META_NAME = 'my_scheme.meta' + F90_NAME = 'my_scheme.F90' + + def test_no_errors_when_both_match(self): + errs = validate([self.meta_path], [self.f90_path]) + self.assertEqual(errs, []) + + def test_ddt_component_mismatch_surfaces(self): + # Break the DDT component type and confirm the validator + # catches it (proves the DDT pass actually runs on scheme files). + broken_f90 = textwrap.dedent("""\ + module my_scheme + implicit none + type :: ty_ws + integer :: nlay + integer :: tk(:,:) + end type ty_ws + contains + subroutine my_scheme_run(workspace, errmsg, errflg) + type(ty_ws), intent(inout) :: workspace + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + end subroutine my_scheme_run + end module my_scheme + """) + with open(self.f90_path, 'w') as fh: + fh.write(broken_f90) + errs = validate([self.meta_path], [self.f90_path]) + type_errs = [e for e in errs if 'type mismatch' in e] + self.assertEqual(len(type_errs), 1, errs) + self.assertIn("tk", type_errs[0]) + self.assertIn("ty_ws", type_errs[0]) + + +class TestValidateRequiresAtLeastOneInputs(unittest.TestCase): + """validate() must error when neither scheme_files nor host_files is supplied.""" + + def test_both_empty_raises(self): + with self.assertRaises(CCPPError) as cm: + validate([], [], host_files=[]) + self.assertIn("at least one", str(cm.exception)) + self.assertIn("--scheme-files", str(cm.exception)) + self.assertIn("--host-files", str(cm.exception)) + + def test_default_host_files_kwarg_also_raises(self): + # Same condition reached via the default value of host_files. + with self.assertRaises(CCPPError): + validate([], []) + + def load_tests(loader, tests, ignore): tests.addTests(doctest.DocTestSuite(val_mod)) return tests From a99ca8a9448bf54dac283e30b2859d7c2473509c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Jun 2026 15:46:54 -0600 Subject: [PATCH 46/74] capgen-ng/ccpp_capgen_ng.py: add return_state argument --- capgen-ng/ccpp_capgen_ng.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index dcc50c8c..fa53946f 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -825,7 +825,8 @@ def capgen( logger: Optional[logging.Logger] = None, no_host_introspection: bool = False, trace: bool = False, -) -> None: + return_state: bool = False, +): """Programmatic entry point for the cap generator. Mirrors the CLI behaviour. Both the CLI and programmatic paths call @@ -1128,6 +1129,16 @@ def capgen( log.info("Cap generation complete.") + # When *return_state* is requested, hand the resolved state back + # to the caller so external tools (host-side compat adapters, + # debug utilities, downstream code generators) can consume the + # in-memory ``host_dict`` and ``suite_resolutions`` without + # re-running the load + resolve passes. Returns ``None`` + # otherwise: the canonical signature is "side effects only". + if return_state: + return host_dict, suite_resolutions + return None + def main(argv: Optional[List[str]] = None) -> int: """Command-line entry point. From 16ef2d4d290058c42a737434cd60179e6cee9758 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 3 Jun 2026 14:10:49 -0600 Subject: [PATCH 47/74] Add ccpp_constituent_minimum_values to framework-resolved variables --- capgen-ng/generator/suite_resolver.py | 3 +++ unit-tests/test_suite_resolver.py | 36 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 2f2b1b42..c18a7aa3 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -130,6 +130,7 @@ _CONST_TEND_ARRAY_STD = 'ccpp_constituent_tendencies' _CONST_PROPS_ARRAY_STD = 'ccpp_constituent_properties' _CONST_NUM_STD = 'number_of_ccpp_constituents' +_CONST_MINVAL_STD = 'ccpp_constituent_minimum_values' _TEND_PREFIX = 'tendency_of_' _INDEX_PREFIX = 'index_of_' @@ -139,6 +140,7 @@ _CONST_TEND_ARRAY_STD, _CONST_PROPS_ARRAY_STD, _CONST_NUM_STD, + _CONST_MINVAL_STD, }) # Per-instance constituent object name in ccpp_host_constituents. Schemes @@ -153,6 +155,7 @@ _CONST_TEND_ARRAY_STD: 'vars_layer_tend', _CONST_PROPS_ARRAY_STD: 'const_metadata', _CONST_NUM_STD: 'num_layer_vars', + _CONST_MINVAL_STD: 'vars_minvalue', } diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 74625004..50adfd05 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -3611,6 +3611,7 @@ def test_tendency_call_expr(self): '1:nlev, index_of_cloud_liquid_water_mixing_ratio)', ) + def test_instance_number_in_used_dim_std_names(self): # Drives the group cap to inject instance_number as a dummy arg. for name in ('cldliq', 'tend_cldliq'): @@ -3729,6 +3730,41 @@ def test_no_const_dim_when_arg_does_not_reference_it(self): self.assertIsNotNone(arg) self.assertEqual(arg.used_const_dim_std_names, set()) + def test_minimum_values_routes_to_vars_minvalue(self): + # ``ccpp_constituent_minimum_values`` is a framework-named std + # whose value is per-constituent and lives on + # ``ccpp_model_constituents_t%vars_minvalue(:)``. The resolver + # must route it through Path 1b (framework-name) — the + # ``vars_minvalue`` member, not Path 2 (constituent auto- + # provisioning). Drives cam-sima's ``qneg`` scheme: under the + # original capgen contract this was a host-USE'd module array; + # capgen-ng exposes it through the per-instance object. + from generator.suite_resolver import _resolve_constituent_arg + hd = _load_full_host_dict() + suite_var = self._scheme_var( + 'qmin', 'ccpp_constituent_minimum_values', + '(number_of_ccpp_constituents)', + intent='in', + ) + arg = _resolve_constituent_arg( + suite_var, 'run', hd, {}, 'qneg', 'mysuite', + ) + self.assertIsNotNone(arg) + self.assertEqual(arg.source, 'constituent') + inst_local = hd['instance_number'].local_name + self.assertEqual( + arg.call_expr, + 'ccpp_model_constituents_obj({})%vars_minvalue(:)'.format( + inst_local), + ) + # number_of_ccpp_constituents goes on the dedicated channel. + self.assertEqual(arg.used_const_dim_std_names, + {'number_of_ccpp_constituents'}) + self.assertNotIn('number_of_ccpp_constituents', + arg.used_dim_std_names) + self.assertNotIn('number_of_ccpp_constituents', + arg.constituent_extra_symbols) + class TestConstSubscriptHelper(unittest.TestCase): """``_const_dim_part`` / ``_build_const_subscript``: From 831865664c0b44433fe339191332b7e6646baf5b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 3 Jun 2026 15:48:25 -0600 Subject: [PATCH 48/74] Cap long Fortran names --- capgen-ng/generator/host_constituents.py | 18 +++--- capgen-ng/generator/suite_resolver.py | 65 +++++++++++++++++++- unit-tests/test_suite_resolver.py | 76 ++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 11 deletions(-) diff --git a/capgen-ng/generator/host_constituents.py b/capgen-ng/generator/host_constituents.py index 55b29a3e..11be6e8e 100644 --- a/capgen-ng/generator/host_constituents.py +++ b/capgen-ng/generator/host_constituents.py @@ -32,7 +32,7 @@ from typing import List, Optional, Set, Tuple from metadata.parse_tools import open_if_changed -from generator.suite_resolver import SuiteResolution +from generator.suite_resolver import SuiteResolution, _index_symbol_name _INDENT = ' ' @@ -282,10 +282,11 @@ def _initialize_constituents_lines( lines.append('{}nullify(const_obj_ptr)'.format(i2)) lines.append('') for std_name in index_names: + index_sym = _index_symbol_name(std_name) lines.append( - "{}call {}({})%const_index(index_of_{}, '{}', " + "{}call {}({})%const_index({}, '{}', " "errcode=errflg, errmsg=errmsg)".format( - i2, _CONST_OBJ, inst_idx, std_name, std_name, + i2, _CONST_OBJ, inst_idx, index_sym, std_name, ) ) lines.append('{}if (errflg /= 0) return'.format(i2)) @@ -294,8 +295,8 @@ def _initialize_constituents_lines( # explicitly so the host sees the bad registration at init time # instead of crashing on a -huge(1) subscript later. lines.append( - '{}if (index_of_{} == int_unassigned) then'.format( - i2, std_name, + '{}if ({} == int_unassigned) then'.format( + i2, index_sym, ) ) lines.append('{}errflg = 1'.format(i2 + _INDENT)) @@ -543,7 +544,7 @@ def _deallocate_lines( # instead. lines.append('{}deallocate({})'.format(i3, _CONST_OBJ)) for std_name in index_names: - lines.append('{}index_of_{} = 0'.format(i3, std_name)) + lines.append('{}{} = 0'.format(i3, _index_symbol_name(std_name))) lines.append('{}end if'.format(i2)) lines.append('') lines.append('{}end subroutine ccpp_deallocate_dynamic_constituents'.format(i1)) @@ -591,7 +592,7 @@ def _generate_host_constituents( # Publics: state + routines. publics = [_CONST_OBJ] - publics += ['index_of_{}'.format(n) for n in index_names] + publics += [_index_symbol_name(n) for n in index_names] publics += [ 'ccpp_register_constituents', 'ccpp_initialize_constituents', @@ -649,7 +650,8 @@ def _generate_host_constituents( if register_suites: lines.append('') for std_name in index_names: - lines.append('{}integer :: index_of_{} = 0'.format(_INDENT, std_name)) + lines.append('{}integer :: {} = 0'.format( + _INDENT, _index_symbol_name(std_name))) if index_names: max_len = max(len(n) for n in index_names) lines.append('') diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index c18a7aa3..ba25bc2d 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -134,6 +134,60 @@ _TEND_PREFIX = 'tendency_of_' _INDEX_PREFIX = 'index_of_' +# Fortran 2008 caps user-defined identifiers at 63 characters. Several +# CCPP standard-name conventions (notably CAM-SIMA's +# ``_wrt_moist_air_and_condensed_water`` constituent suffix) produce +# base names ≥55 chars, which blow the limit once the ``index_of_`` +# prefix is prepended. The helper below mangles overlong names to a +# deterministic 63-char form so every emitter and resolver call site +# sees the same symbol; short names are returned unchanged. +_FORTRAN_ID_LIMIT = 63 + + +def _index_symbol_name(base_std_name: str) -> str: + """Return the Fortran local name for ``index_of_``. + + Identity for inputs whose ``index_of_`` form fits Fortran's 63-char + identifier limit; otherwise truncates the base and appends a short + SHA-1 hash so distinct std-names map to distinct symbols. The + chosen layout is:: + + index_of__<8-hex-sha1> + + where ``max_base_len = 63 - len('index_of_') - 1 - 8 = 45``. All + emit/reference sites (host_constituents.py public/declaration/ + reset/const_index/init-guard, suite_resolver.py auto-provisioned + subscript, Path 1a call_expr) MUST route through this helper to + keep the symbol consistent within a single capgen-ng run. The + underlying std_name is still passed to ``const_index`` as a + string literal, so the framework lookup keys remain unchanged -- + only the Fortran-side mapping symbol is mangled. + + Examples + -------- + >>> _index_symbol_name('water_vapor') + 'index_of_water_vapor' + >>> name = _index_symbol_name( + ... 'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water') + >>> len(name) <= 63 + True + >>> name.startswith('index_of_') + True + >>> name == _index_symbol_name( + ... 'cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water') + True + """ + full = _INDEX_PREFIX + base_std_name + if len(full) <= _FORTRAN_ID_LIMIT: + return full + import hashlib + sha8 = hashlib.sha1(base_std_name.encode('utf-8')).hexdigest()[:8] + # 63 - len('index_of_') - 1 (sep) - 8 (sha) = 45 + max_base_len = _FORTRAN_ID_LIMIT - len(_INDEX_PREFIX) - 1 - 8 + return '{}{}_{}'.format( + _INDEX_PREFIX, base_std_name[:max_base_len], sha8, + ) + # Std names directly satisfied by host-constituents-module-owned symbols. _FRAMEWORK_CONST_STDS = frozenset({ _CONST_BASE_ARRAY_STD, @@ -2002,9 +2056,14 @@ def _common_kwargs(base_expr, subscript, call_expr, # ---- Path 1a: index_of_ — module-level integer, no per-instance -- if is_index_name: + # Mangle long std_names down to a Fortran-legal 63-char symbol; + # identity for short names, so existing fixtures are unaffected. + # ``_INDEX_PREFIX`` is already part of std_name -- strip then + # re-add via the helper for uniform truncation. + index_sym = _index_symbol_name(std_name[len(_INDEX_PREFIX):]) return ResolvedArg(**_common_kwargs( - base_expr=std_name, subscript='', call_expr=std_name, - used_host_std=set(), extra_symbols={std_name}, + base_expr=index_sym, subscript='', call_expr=index_sym, + used_host_std=set(), extra_symbols={index_sym}, )) # ---- Path 1b: framework-named std_name → DDT member ----------------- @@ -2058,7 +2117,7 @@ def _common_kwargs(base_expr, subscript, call_expr, leading_sub, used_host_std = _build_call_subscript( scheme_dims, phase, host_dict, suite_vars=suite_vars, ) - index_sym = '{}{}'.format(_INDEX_PREFIX, base_std) + index_sym = _index_symbol_name(base_std) if leading_sub: subscript = leading_sub[:-1] + ', ' + index_sym + ')' else: diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 50adfd05..ebb849c5 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -3766,6 +3766,82 @@ def test_minimum_values_routes_to_vars_minvalue(self): arg.constituent_extra_symbols) +class TestIndexSymbolNameMangling(unittest.TestCase): + """``_index_symbol_name`` keeps short ``index_of_`` names + intact, but mangles overlong CAM-SIMA-style names down to the + Fortran 63-char identifier limit with a deterministic SHA hash so + every emit/reference site agrees on the symbol.""" + + def test_short_name_passes_through(self): + from generator.suite_resolver import _index_symbol_name + self.assertEqual(_index_symbol_name('water_vapor'), + 'index_of_water_vapor') + + def test_overlong_name_truncated_and_hashed(self): + from generator.suite_resolver import _index_symbol_name + long_base = ('cloud_liquid_water_mixing_ratio_' + 'wrt_moist_air_and_condensed_water') + sym = _index_symbol_name(long_base) + # Must be a Fortran-legal identifier (≤ 63 chars), prefixed + # with index_of_, and stable across calls. + self.assertLessEqual(len(sym), 63) + self.assertTrue(sym.startswith('index_of_')) + self.assertEqual(sym, _index_symbol_name(long_base)) + + def test_distinct_bases_distinct_symbols(self): + # Two CAM-SIMA constituents share the same long suffix; the + # hash component must keep their symbols distinct so the + # framework's per-constituent integer storage doesn't alias. + from generator.suite_resolver import _index_symbol_name + a = _index_symbol_name( + 'cloud_liquid_water_mixing_ratio_' + 'wrt_moist_air_and_condensed_water') + b = _index_symbol_name( + 'water_vapor_mixing_ratio_' + 'wrt_moist_air_and_condensed_water') + self.assertNotEqual(a, b) + + def test_used_in_auto_provisioned_call_expr(self): + # End-to-end check: the auto-provisioning subscript path + # (Path 2) routes the index_of_ token through the helper, + # so the call_expr that lands in the group cap is a legal + # Fortran symbol. + from generator.suite_resolver import ( + _resolve_constituent_arg, _index_symbol_name, + ) + long_base = ('cloud_liquid_water_mixing_ratio_' + 'wrt_moist_air_and_condensed_water') + hd = _load_full_host_dict() + scheme_var = self._scheme_var_for_mangling( + 'cldliq', long_base, + '(horizontal_dimension, vertical_layer_dimension)', + ) + scheme_var.set_attr('advected', 'True', _ctx()) + arg = _resolve_constituent_arg( + scheme_var, 'run', hd, {}, 'consumer', 'mysuite', + ) + self.assertIsNotNone(arg) + expected_index_sym = _index_symbol_name(long_base) + self.assertIn(expected_index_sym, arg.constituent_extra_symbols) + # The long raw form must NOT appear (would blow the Fortran limit). + self.assertNotIn('index_of_' + long_base, + arg.constituent_extra_symbols) + self.assertIn(expected_index_sym, arg.call_expr) + + @staticmethod + def _scheme_var_for_mangling(local, std_name, dims, intent='in'): + from metadata.metadata_table import MetaVar + ctx = _ctx() + v = MetaVar(local, ctx) + v.set_attr('standard_name', std_name, ctx) + v.set_attr('units', 'none', ctx) + v.set_attr('dimensions', dims, ctx) + v.set_attr('type', 'real', ctx) + v.set_attr('kind', 'kind_phys', ctx) + v.set_attr('intent', intent, ctx) + return v + + class TestConstSubscriptHelper(unittest.TestCase): """``_const_dim_part`` / ``_build_const_subscript``: ``number_of_ccpp_constituents`` becomes ``':'`` and is routed From 9550b77325e2a40dc6ceedfccce91a7012fb8ba8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 3 Jun 2026 18:04:01 -0600 Subject: [PATCH 49/74] Fix constituents imports for auto-clone constituents legacy shim --- capgen-ng/generator/host_constituents.py | 7 +++- capgen-ng/generator/suite_cap.py | 7 ++-- capgen-ng/generator/suite_resolver.py | 19 ++++++++++ unit-tests/test_host_constituents.py | 35 +++++++++++++++++ unit-tests/test_suite_resolver.py | 48 ++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 6 deletions(-) diff --git a/capgen-ng/generator/host_constituents.py b/capgen-ng/generator/host_constituents.py index 11be6e8e..75b16e52 100644 --- a/capgen-ng/generator/host_constituents.py +++ b/capgen-ng/generator/host_constituents.py @@ -70,8 +70,11 @@ def _all_index_names(suite_results: List[SuiteResolution]) -> List[str]: def _suites_with_register_consts( suite_results: List[SuiteResolution], ) -> List[str]: - return [suite_resolution.suite_name for suite_resolution in suite_results - if suite_resolution.constituent_register_calls] + # auto-clone-constituents: predicate is centralised on + # SuiteResolution so this module never reads legacy-shim state + # directly; see ``SuiteResolution.needs_dynamic_constituents_buffer``. + return [sr.suite_name for sr in suite_results + if sr.needs_dynamic_constituents_buffer] def _dyn_const_array_name(suite_name: str) -> str: diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 260bafa9..ea874dda 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -225,10 +225,9 @@ def _register_uses( # Per-suite dynamic-constituent buffer is owned by ccpp_host_constituents # and written into here. Pull in the constituent property type plus the # buffer symbol. - # auto-clone-constituents: the synthesised %instantiate calls - # also need the constituent property type and the buffer symbol, - # so include them when the auto-clone list is non-empty. - if suite_res.constituent_register_calls or suite_res.auto_cloned_constituents: + # auto-clone-constituents: predicate centralised on SuiteResolution so + # this site does not read legacy-shim state directly. + if suite_res.needs_dynamic_constituents_buffer: uses.setdefault(_CONST_MOD, set()).add(_CONST_PROP_TYPE) buf = '{}_dynamic_constituents'.format(suite_name) uses.setdefault('ccpp_host_constituents', set()).add(buf) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index ba25bc2d..58e8769d 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -1330,6 +1330,25 @@ class SuiteResolution: # dynamic-constituents buffer. Empty when the shim is off. auto_cloned_constituents: List[AutoCloneEntry] = field(default_factory=list) + # auto-clone-constituents: this property exists so consumer + # emitters (host_constituents, suite_cap) never have to read + # ``auto_cloned_constituents`` themselves. When the legacy + # auto-clone shim retires, drop the second clause below and the + # property collapses to ``bool(self.constituent_register_calls)``; + # every consumer keeps working without changes. + @property + def needs_dynamic_constituents_buffer(self) -> bool: + """True iff the per-suite ``_dynamic_constituents`` buffer + must be declared and populated for this suite. + + Single source of truth for the predicate. + """ + return bool( + self.constituent_register_calls + # auto-clone-constituents: + or self.auto_cloned_constituents + ) + ######################################################################## # Argument resolution helpers diff --git a/unit-tests/test_host_constituents.py b/unit-tests/test_host_constituents.py index 685ec874..c2ad807f 100644 --- a/unit-tests/test_host_constituents.py +++ b/unit-tests/test_host_constituents.py @@ -94,6 +94,41 @@ def test_register_suites_listed(self): ['reg_consts'], ) + def test_register_suites_include_auto_cloned_only(self): + # auto-clone-constituents: the legacy shim populates + # ``SuiteResolution.auto_cloned_constituents`` but leaves + # ``constituent_register_calls`` empty (no real register + # scheme exists). host_constituents.F90 still has to declare + # the per-suite ``_dynamic_constituents`` buffer + # because the suite cap emits a USE on it. Regression for + # CAM-SIMA kessler_test build (2026-06-03). + from generator.suite_resolver import SuiteResolution, AutoCloneEntry + sr = SuiteResolution( + suite_name='kessler_test', + auto_cloned_constituents=[AutoCloneEntry( + std_name='water_vapor', long_name='', diag_name='qv', + units='kg kg-1', vertical_dim='vertical_layer_dimension', + advected=True, molar_mass=0.0, default_value=None, + min_value=None, water_species=None, mixing_ratio_type=None, + )], + ) + self.assertEqual( + _suites_with_register_consts([sr]), + ['kessler_test'], + ) + + def test_register_suites_excludes_pure_consumer(self): + # auto-clone-constituents: no register calls AND no auto-cloned + # entries -> suite stays off the list so the buffer is not + # declared. Companion to test_register_suites_include_auto_cloned_only; + # delete together when the shim retires. + from generator.suite_resolver import SuiteResolution + sr = SuiteResolution(suite_name='pure_consumer') + self.assertEqual( + _suites_with_register_consts([sr]), + [], + ) + class TestModuleSkippedWhenNoConstituents(unittest.TestCase): """``_generate_host_constituents`` returns ``None`` when nothing touches diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index ebb849c5..3d675f5b 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -3898,6 +3898,54 @@ def test_full_3d_subscript(self): self.assertEqual(used_const_dim, {'number_of_ccpp_constituents'}) +# auto-clone-constituents: the entire class below exists because the +# legacy auto-clone shim introduces a second source for "this suite +# needs a per-suite ``_dynamic_constituents`` buffer". When +# the shim retires, this class can be deleted -- the property collapses +# to ``bool(self.constituent_register_calls)`` and the existing +# register-path tests elsewhere already cover that case. +class TestNeedsDynamicConstituentsBufferProperty(unittest.TestCase): + """``SuiteResolution.needs_dynamic_constituents_buffer`` is the + single source of truth for "this suite needs a + ``_dynamic_constituents`` buffer". Centralising the rule + here keeps legacy-shim state (``auto_cloned_constituents``) out of + every generator emitter; consumers reference the property instead + of OR-ing the two underlying fields. When the legacy auto-clone + shim retires, only this property's body changes.""" + + def test_false_when_neither(self): + # auto-clone-constituents: empty-state baseline. + from generator.suite_resolver import SuiteResolution + sr = SuiteResolution(suite_name='s') + self.assertFalse(sr.needs_dynamic_constituents_buffer) + + def test_true_for_register_calls(self): + # auto-clone-constituents: register-only path -- verifies the + # property still fires correctly for non-shim registrations + # after the OR-abstraction landed. + from generator.suite_resolver import SuiteResolution + sr = SuiteResolution( + suite_name='s', + constituent_register_calls=[('register_constituents', 'register')], + ) + self.assertTrue(sr.needs_dynamic_constituents_buffer) + + def test_true_for_auto_cloned_only(self): + # auto-clone-constituents: shim-only path -- regression for + # the CAM-SIMA kessler_test build (2026-06-03). + from generator.suite_resolver import SuiteResolution, AutoCloneEntry + sr = SuiteResolution( + suite_name='s', + auto_cloned_constituents=[AutoCloneEntry( + std_name='water_vapor', long_name='', diag_name='qv', + units='kg kg-1', vertical_dim='vertical_layer_dimension', + advected=True, molar_mass=0.0, default_value=None, + min_value=None, water_species=None, mixing_ratio_type=None, + )], + ) + self.assertTrue(sr.needs_dynamic_constituents_buffer) + + class TestConstituentResolverErrors(unittest.TestCase): """Mismatched constituent-flag + intent + std-name combinations error.""" From 047741269b1001b7b43547f587a7e64d26bd3fd1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 3 Jun 2026 20:54:19 -0600 Subject: [PATCH 50/74] Rename internal hardcoded errflg to errcode, update tests --- capgen-ng/ccpp_capgen_ng.py | 1 + capgen-ng/generator/host_constituents.py | 44 +- doc/constituents_overhaul.md | 71 ++ end-to-end-tests/advection/cld_ice.F90 | 18 +- end-to-end-tests/advection/cld_ice.meta | 6 +- end-to-end-tests/advection/cld_liq.F90 | 26 +- end-to-end-tests/advection/cld_liq.meta | 6 +- end-to-end-tests/advection/const_indices.F90 | 32 +- end-to-end-tests/advection/const_indices.meta | 4 +- end-to-end-tests/advection/dlc_liq.F90 | 14 +- end-to-end-tests/advection/dlc_liq.meta | 2 +- end-to-end-tests/advection/test_host.F90 | 684 +++++++++--------- end-to-end-tests/advection/test_host.meta | 2 +- end-to-end-tests/advection/test_host_data.F90 | 10 +- .../advection_auto_clone/cld_ice.F90 | 18 +- .../advection_auto_clone/cld_ice.meta | 6 +- .../advection_auto_clone/cld_liq.F90 | 24 +- .../advection_auto_clone/cld_liq.meta | 6 +- .../advection_auto_clone/const_indices.F90 | 32 +- .../advection_auto_clone/const_indices.meta | 4 +- .../advection_auto_clone/dlc_liq.F90 | 14 +- .../advection_auto_clone/dlc_liq.meta | 2 +- .../advection_auto_clone/test_host.F90 | 682 ++++++++--------- .../advection_auto_clone/test_host.meta | 2 +- .../advection_auto_clone/test_host_data.F90 | 10 +- .../instances_advection/cld_liq.F90 | 24 +- .../instances_advection/cld_liq.meta | 6 +- end-to-end-tests/instances_advection/main.F90 | 54 +- .../instances_advection/main.meta | 2 +- .../sample_files/control_unit_conv.meta | 7 + unit-tests/test_control_validation.py | 3 +- unit-tests/test_host_constituents.py | 24 +- 32 files changed, 960 insertions(+), 880 deletions(-) diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index fa53946f..5b112ed3 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -684,6 +684,7 @@ def _load_metadata_files( # Required control variables: (standard_name, expected_fortran_type, description) _REQUIRED_CTRL_VARS = [ ('suite_name', 'character', 'drives suite dispatch'), + ('group_name', 'character', 'drives per-group dispatch inside ccpp_physics_* (each suite_cap emits a select case on this name)'), ('horizontal_loop_begin', 'integer', 'lower horizontal slice bound at scheme call sites'), ('horizontal_loop_end', 'integer', 'upper horizontal slice bound at scheme call sites'), ('thread_number', 'integer', 'current thread number (pass 1 if single-threaded)'), diff --git a/capgen-ng/generator/host_constituents.py b/capgen-ng/generator/host_constituents.py index 75b16e52..f003fe27 100644 --- a/capgen-ng/generator/host_constituents.py +++ b/capgen-ng/generator/host_constituents.py @@ -110,7 +110,7 @@ def _instance_signature( sig.append(inst_local) if ninst_local: sig.append(ninst_local) - sig += ['errflg', 'errmsg'] + sig += ['errcode', 'errmsg'] return sig @@ -148,7 +148,7 @@ def _register_constituents_lines( lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) if ninst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, ninst_local)) - lines.append('{}integer, intent(out) :: errflg'.format(i2)) + lines.append('{}integer, intent(out) :: errcode'.format(i2)) lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) lines.append('') lines.append('{}integer :: num_consts, index'.format(i2)) @@ -157,7 +157,7 @@ def _register_constituents_lines( )) lines.append('') lines.append("{}errmsg = ''".format(i2)) - lines.append('{}errflg = 0'.format(i2)) + lines.append('{}errcode = 0'.format(i2)) lines.append('') # Allocate the object array on first call (idempotent across instances). lines.append('{}if (.not. allocated({})) then'.format(i2, _CONST_OBJ)) @@ -188,12 +188,12 @@ def _register_constituents_lines( lines.append('{}do index = 1, size(host_constituents, 1)'.format(i2)) lines.append('{}const_prop => host_constituents(index)'.format(i3)) lines.append( - '{}call {}({})%new_field(const_prop, errcode=errflg, errmsg=errmsg)'.format( + '{}call {}({})%new_field(const_prop, errcode=errcode, errmsg=errmsg)'.format( i3, _CONST_OBJ, inst_idx, ) ) lines.append('{}nullify(const_prop)'.format(i3)) - lines.append('{}if (errflg /= 0) return'.format(i3)) + lines.append('{}if (errcode /= 0) return'.format(i3)) lines.append('{}end do'.format(i2)) for sname in register_suites: buf = _dyn_const_array_name(sname) @@ -214,18 +214,18 @@ def _register_constituents_lines( ) ) lines.append( - '{}call {}({})%new_field(const_prop, errcode=errflg, errmsg=errmsg)'.format( + '{}call {}({})%new_field(const_prop, errcode=errcode, errmsg=errmsg)'.format( i3 + _INDENT * 2, _CONST_OBJ, inst_idx, ) ) lines.append('{}nullify(const_prop)'.format(i3 + _INDENT * 2)) - lines.append('{}if (errflg /= 0) return'.format(i3 + _INDENT * 2)) + lines.append('{}if (errcode /= 0) return'.format(i3 + _INDENT * 2)) lines.append('{}end do'.format(i3 + _INDENT)) lines.append('{}end if'.format(i3)) lines.append('{}end if'.format(i2)) lines.append('') lines.append( - '{}call {}({})%lock_table(errcode=errflg, errmsg=errmsg)'.format( + '{}call {}({})%lock_table(errcode=errcode, errmsg=errmsg)'.format( i2, _CONST_OBJ, inst_idx, ) ) @@ -257,7 +257,7 @@ def _initialize_constituents_lines( lines.append('{}integer, intent(in) :: ncols, num_layers'.format(i2)) if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) - lines.append('{}integer, intent(out) :: errflg'.format(i2)) + lines.append('{}integer, intent(out) :: errcode'.format(i2)) lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) lines.append('') lines.append('{}type({}), pointer :: const_obj_ptr => null()'.format( @@ -265,14 +265,14 @@ def _initialize_constituents_lines( )) lines.append('') lines.append("{}errmsg = ''".format(i2)) - lines.append('{}errflg = 0'.format(i2)) + lines.append('{}errcode = 0'.format(i2)) lines.append('') lines.append( - '{}call {}({})%lock_data(ncols, num_layers, errcode=errflg, errmsg=errmsg)'.format( + '{}call {}({})%lock_data(ncols, num_layers, errcode=errcode, errmsg=errmsg)'.format( i2, _CONST_OBJ, inst_idx, ) ) - lines.append('{}if (errflg /= 0) return'.format(i2)) + lines.append('{}if (errcode /= 0) return'.format(i2)) # Cache the singleton pointer in ccpp_scheme_utils (cam-sima compat). # Only the FIRST call across instances actually sets it (the routine # is guarded internally); other instances see the first instance's @@ -288,11 +288,11 @@ def _initialize_constituents_lines( index_sym = _index_symbol_name(std_name) lines.append( "{}call {}({})%const_index({}, '{}', " - "errcode=errflg, errmsg=errmsg)".format( + "errcode=errcode, errmsg=errmsg)".format( i2, _CONST_OBJ, inst_idx, index_sym, std_name, ) ) - lines.append('{}if (errflg /= 0) return'.format(i2)) + lines.append('{}if (errcode /= 0) return'.format(i2)) # %const_index doesn't error on a miss — it sets the integer to # int_unassigned and leaves errcode unchanged. Surface that case # explicitly so the host sees the bad registration at init time @@ -302,7 +302,7 @@ def _initialize_constituents_lines( i2, index_sym, ) ) - lines.append('{}errflg = 1'.format(i2 + _INDENT)) + lines.append('{}errcode = 1'.format(i2 + _INDENT)) lines.append( "{}errmsg = 'ccpp_initialize_constituents: constituent " "''{}'' is referenced by a scheme but is not in the " @@ -327,16 +327,16 @@ def _is_scheme_constituent_lines(suite_results: List[SuiteResolution]) -> List[s lines: List[str] = [''] lines.append( '{}subroutine ccpp_is_scheme_constituent(var_name, ' - 'constituent_exists, errflg, errmsg)'.format(i1) + 'constituent_exists, errcode, errmsg)'.format(i1) ) lines.append('') lines.append('{}character(len=*), intent(in) :: var_name'.format(i2)) lines.append('{}logical, intent(out) :: constituent_exists'.format(i2)) - lines.append('{}integer, intent(out) :: errflg'.format(i2)) + lines.append('{}integer, intent(out) :: errcode'.format(i2)) lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) lines.append('') lines.append("{}errmsg = ''".format(i2)) - lines.append('{}errflg = 0'.format(i2)) + lines.append('{}errcode = 0'.format(i2)) lines.append('') if index_names: lines.append( @@ -363,7 +363,7 @@ def _wrap_method_sub( sig = [n for n, _, _ in extra_args] if inst_local: sig.append(inst_local) - sig += ['errflg', 'errmsg'] + sig += ['errcode', 'errmsg'] lines: List[str] = [''] lines.append('{}subroutine {}({})'.format(i1, sub_name, ', '.join(sig))) lines.append('') @@ -371,15 +371,15 @@ def _wrap_method_sub( lines.append('{}{}'.format(i2, decl)) if inst_local: lines.append('{}integer, intent(in) :: {}'.format(i2, inst_local)) - lines.append('{}integer, intent(out) :: errflg'.format(i2)) + lines.append('{}integer, intent(out) :: errcode'.format(i2)) lines.append('{}character(len=*), intent(out) :: errmsg'.format(i2)) lines.append('') lines.append("{}errmsg = ''".format(i2)) - lines.append('{}errflg = 0'.format(i2)) + lines.append('{}errcode = 0'.format(i2)) lines.append('') call_args = [call for _, _, call in extra_args] if errcode_call: - call_args += ['errcode=errflg', 'errmsg=errmsg'] + call_args += ['errcode=errcode', 'errmsg=errmsg'] lines.append('{}call {}({})%{}({})'.format( i2, _CONST_OBJ, inst_idx, method, ', '.join(call_args), )) diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index a709364c..97cc3d75 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -619,6 +619,77 @@ shim. Remove the rewrite once known consumers are migrated. - **Position relative to Proposals A/B/C**: orthogonal — none of the three proposed touching the buffer. Independently adopted. +### 4.14 Capgen-ng: error-output keyword inconsistency across emitted public API (OPEN — observation) + +- **Location**: `capgen-ng/generator/host_cap.py:370,446,*` (lifecycle + subs) vs `capgen-ng/generator/host_constituents.py` (the entire + constituent wrapper family). +- **Symptom**: the public Fortran argument carrying the CCPP error + flag does not have a consistent name across the cap's surface area. + - **Lifecycle subs** (`ccpp_register`, `ccpp_init`, `ccpp_physics_*`, + `ccpp_final`, plus the five `ccpp_physics_suite_*` introspection + routines) read the host's `ccpp_error_code` control-var + `local_name` from `host_dict` and use *that* as the public arg + name. A host that calls `[errflg]` `errflg` ends up with + `subroutine ccpp_register(suite_name, errflg, errmsg)`; a host + that calls `[errcode]` `errcode` ends up with + `subroutine ccpp_register(suite_name, errcode, errmsg)`. This is + the host-controlled side. + - **Constituent wrappers** (`ccpp_register_constituents`, + `ccpp_initialize_constituents`, `ccpp_is_scheme_constituent`, + `ccpp_number_constituents`, `ccpp_gather_constituents`, + `ccpp_update_constituents`, `ccpp_const_get_index`) hard-code + `errcode` regardless of what the host declared. As of + 2026-06-03 the hard-code is `errcode` (renamed from `errflg` for + consistency with the framework methods on + `ccpp_constituent_properties_t` and `ccpp_model_constituents_t`, + which all expose `errcode=`). Before 2026-06-03 it was `errflg`, + which broke any host whose control-var convention was + `errcode` -- including the CAM-SIMA build. +- **Resulting cross-cutting hazard**: in a host where the + `ccpp_error_code` local name happens to be `errflg`, the caller + writes + ```fortran + call ccpp_register(suite_name, errflg=errflg, errmsg=errmsg) ! host-name keyword + call ccpp_register_constituents(host_consts, errcode=errflg, errmsg=errmsg) ! hardcoded keyword + ``` + Two different keyword names for the same conceptual argument on + adjacent calls. Confusing but compiles; the host's local variable + is bound by name to whichever keyword the callee defines. +- **Why the constituent wrappers are hardcoded**: the wrappers are + thin shims around framework methods + (`ccpp_model_constituents_t%new_field`, `%lock_table`, + `%num_constituents`, etc.) that all take `errcode=` per + `capgen-ng/src/ccpp_constituent_prop_mod.F90`. Hardcoding `errcode` + on the wrapper means the wrapper body just forwards + `errcode=errcode` instead of `errcode=` -- one + less host-dict lookup, but at the cost of breaking the + "host names what they want" contract. +- **Options to resolve**: + - (a) Plumb the host's `ccpp_error_code` local name through + `host_constituents.py` the same way `host_cap.py` does (via + `_ctrl_local(host_dict, 'ccpp_error_code') or 'errcode'`). + The constituent wrappers' public arg then tracks the host's + convention. Adds 1 dictionary lookup per emitted sub; no + other change. + - (b) Standardise the lifecycle subs on `errcode` too, ignoring + the host's `ccpp_error_code` local name. Simpler internally + but breaks every existing host that ships + `[errflg] standard_name = ccpp_error_code`. + - (c) Status quo (the constituent wrappers' `errcode` hardcode): + document it loudly and live with the cross-API split. +- **Status**: currently option (c). The post-rename build of CAM-SIMA + works because CAM-SIMA's caller code uses `errcode=errflg` (passing + its local var `errflg` to the hardcoded keyword `errcode`). Hosts + with the opposite convention (`[errcode] standard_name = + ccpp_error_code` -> lifecycle subs expose `errcode=`, + constituent wrappers also expose `errcode=`) coincidentally see + consistent keywords today; the hazard is invisible for them. +- **Recommended fix**: option (a). Lines up with the lifecycle + emitter's already-established host-driven pattern. Trivial + implementation cost; eliminates the cross-cutting confusion for + any host whose `ccpp_error_code` local name is not `errcode`. + --- ## 5. Property classification (Class A vs Class B) diff --git a/end-to-end-tests/advection/cld_ice.F90 b/end-to-end-tests/advection/cld_ice.F90 index bf19b979..e3fc2abd 100644 --- a/end-to-end-tests/advection/cld_ice.F90 +++ b/end-to-end-tests/advection/cld_ice.F90 @@ -49,7 +49,7 @@ end subroutine cld_ice_register !! \htmlinclude arg_table_cld_ice_run.html !! subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & - errmsg, errflg) + errmsg, errcode) integer, intent(in) :: ncol real(kind=kind_phys), intent(in) :: timestep @@ -58,7 +58,7 @@ subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & real(kind=kind_phys), intent(in) :: ps(:) real(kind=kind_phys), intent(inout) :: cld_ice_array(:, :) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: icol @@ -66,7 +66,7 @@ subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & real(kind=kind_phys) :: frz errmsg = '' - errflg = 0 + errcode = 0 ! Apply state-of-the-art thermodynamics :) do icol = 1, ncol @@ -87,14 +87,14 @@ end subroutine cld_ice_run !> \section arg_table_cld_ice_init Argument Table !! \htmlinclude arg_table_cld_ice_init.html !! - subroutine cld_ice_init(tfreeze, errmsg, errflg) + subroutine cld_ice_init(tfreeze, errmsg, errcode) real(kind=kind_phys), intent(in) :: tfreeze character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 + errcode = 0 tcld = tfreeze - 20.0_kind_phys end subroutine cld_ice_init @@ -109,13 +109,13 @@ end subroutine cld_ice_init !! and the subroutine are parsed correctly. !! @{ - subroutine cld_ice_final(errmsg, errflg) + subroutine cld_ice_final(errmsg, errcode) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 + errcode = 0 end subroutine cld_ice_final diff --git a/end-to-end-tests/advection/cld_ice.meta b/end-to-end-tests/advection/cld_ice.meta index bd3cf24a..2200f2c8 100644 --- a/end-to-end-tests/advection/cld_ice.meta +++ b/end-to-end-tests/advection/cld_ice.meta @@ -82,7 +82,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -108,7 +108,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -127,7 +127,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection/cld_liq.F90 b/end-to-end-tests/advection/cld_liq.F90 index d019f152..c0d00a43 100644 --- a/end-to-end-tests/advection/cld_liq.F90 +++ b/end-to-end-tests/advection/cld_liq.F90 @@ -18,15 +18,15 @@ module cld_liq !> \section arg_table_cld_liq_register Argument Table !! \htmlinclude arg_table_cld_liq_register.html !! - subroutine cld_liq_register(dyn_const, errmsg, errflg) + subroutine cld_liq_register(dyn_const, errmsg, errcode) type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 - allocate(dyn_const(2), stat=errflg) - if (errflg /= 0) then + errcode = 0 + allocate(dyn_const(2), stat=errcode) + if (errcode /= 0) then errmsg = 'Error allocating dyn_const in cld_liq_register' return end if @@ -34,14 +34,14 @@ subroutine cld_liq_register(dyn_const, errmsg, errflg) diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & vertical_dim='vertical_layer_dimension', advected=.true., & water_species=.true., mixing_ratio_type='dry', & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) call dyn_const(2)%instantiate(std_name="cloud_liquid_dry_mixing_ratio", long_name='Cloud liquid dry mixing ratio', & diag_name='CLDLIQ', units='kg kg-1', default_value=0._kind_phys, & vertical_dim='vertical_layer_dimension', advected=.true., & ! Defer setting water_species later in the test !water_species=.true., mixing_ratio_type='dry', & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) end subroutine cld_liq_register @@ -49,7 +49,7 @@ end subroutine cld_liq_register !! \htmlinclude arg_table_cld_liq_run.html !! subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & - cld_liq_array, cld_liq_tend, errmsg, errflg) + cld_liq_array, cld_liq_tend, errmsg, errcode) integer, intent(in) :: ncol real(kind=kind_phys), intent(in) :: timestep @@ -60,7 +60,7 @@ subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & real(kind=kind_phys), intent(inout) :: cld_liq_array(:, :) real(kind=kind_phys), intent(out) :: cld_liq_tend(:, :) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: icol @@ -68,7 +68,7 @@ subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & real(kind=kind_phys) :: cond errmsg = '' - errflg = 0 + errcode = 0 ! Apply state-of-the-art thermodynamics :) do icol = 1, ncol @@ -91,15 +91,15 @@ end subroutine cld_liq_run !> \section arg_table_cld_liq_init Argument Table !! \htmlinclude arg_table_cld_liq_init.html !! - subroutine cld_liq_init(tfreeze, tcld, errmsg, errflg) + subroutine cld_liq_init(tfreeze, tcld, errmsg, errcode) real(kind=kind_phys), intent(in) :: tfreeze real(kind=kind_phys), intent(out) :: tcld character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 + errcode = 0 tcld = tfreeze - 20.0_kind_phys end subroutine cld_liq_init diff --git a/end-to-end-tests/advection/cld_liq.meta b/end-to-end-tests/advection/cld_liq.meta index db43c5ef..1abc40d0 100644 --- a/end-to-end-tests/advection/cld_liq.meta +++ b/end-to-end-tests/advection/cld_liq.meta @@ -19,7 +19,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -94,7 +94,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -126,7 +126,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection/const_indices.F90 b/end-to-end-tests/advection/const_indices.F90 index bc3b46a7..5c77e29c 100644 --- a/end-to-end-tests/advection/const_indices.F90 +++ b/end-to-end-tests/advection/const_indices.F90 @@ -17,7 +17,7 @@ module const_indices !! \htmlinclude arg_table_const_indices_run.html !! subroutine const_indices_run(const_std_name, num_consts, test_stdname_array, & - const_index, const_inds, errmsg, errflg) + const_index, const_inds, errmsg, errcode) use ccpp_constituent_prop_mod, only: int_unassigned use ccpp_scheme_utils, only: ccpp_constituent_index use ccpp_scheme_utils, only: ccpp_constituent_indices @@ -28,28 +28,28 @@ subroutine const_indices_run(const_std_name, num_consts, test_stdname_array, & integer, intent(out) :: const_index integer, intent(out) :: const_inds(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: indx integer :: test_indx errmsg = '' - errflg = 0 + errcode = 0 ! Find the constituent index for - call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) - if (errflg == 0) then - call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) + call ccpp_constituent_index(const_std_name, const_index, errcode, errmsg) + if (errcode == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errcode, errmsg) end if ! Check that a non-registered constituent is detectable but ! does not cause an error - if (errflg == 0) then - call ccpp_constituent_index('unobtainium', test_indx, errflg, errmsg) + if (errcode == 0) then + call ccpp_constituent_index('unobtainium', test_indx, errcode, errmsg) if (test_indx /= int_unassigned) then - if (errflg == 0) then + if (errcode == 0) then ! Do not add an error if one is already reported - errflg = 2 + errcode = 2 write(errmsg, '(2a,i0,a,i0)') "ccpp_constituent_index called for ", & "'unobtainium' returned an index of ", test_indx, ", not ", & int_unassigned @@ -63,7 +63,7 @@ end subroutine const_indices_run !! \htmlinclude arg_table_const_indices_init.html !! subroutine const_indices_init(const_std_name, num_consts, test_stdname_array, & - const_index, const_inds, errmsg, errflg) + const_index, const_inds, errmsg, errcode) use ccpp_scheme_utils, only: ccpp_constituent_index, & ccpp_constituent_indices @@ -73,18 +73,18 @@ subroutine const_indices_init(const_std_name, num_consts, test_stdname_array, & integer, intent(out) :: const_index integer, intent(out) :: const_inds(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: indx errmsg = '' - errflg = 0 + errcode = 0 ! Find the constituent index for - call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) - if (errflg == 0) then - call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) + call ccpp_constituent_index(const_std_name, const_index, errcode, errmsg) + if (errcode == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errcode, errmsg) end if end subroutine const_indices_init diff --git a/end-to-end-tests/advection/const_indices.meta b/end-to-end-tests/advection/const_indices.meta index a4cc98e2..147e2ccb 100644 --- a/end-to-end-tests/advection/const_indices.meta +++ b/end-to-end-tests/advection/const_indices.meta @@ -47,7 +47,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -99,7 +99,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection/dlc_liq.F90 b/end-to-end-tests/advection/dlc_liq.F90 index 20ff4b7b..134e3aed 100644 --- a/end-to-end-tests/advection/dlc_liq.F90 +++ b/end-to-end-tests/advection/dlc_liq.F90 @@ -16,25 +16,25 @@ module dlc_liq !> \section arg_table_dlc_liq_init Argument Table !! \htmlinclude arg_table_dlc_liq_init.html !! - subroutine dlc_liq_init(dyn_const, errmsg, errflg) + subroutine dlc_liq_init(dyn_const, errmsg, errcode) type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode character(len=256) :: stdname errmsg = '' - errflg = 0 - allocate(dyn_const(1), stat=errflg) - if (errflg /= 0) then + errcode = 0 + allocate(dyn_const(1), stat=errcode) + if (errcode /= 0) then errmsg = 'Error allocating dyn_const in dlc_liq_init' return end if call dyn_const(1)%instantiate(std_name="dyn_const3", long_name='dyn const3', & diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & vertical_dim='vertical_layer_dimension', advected=.true., & - errcode=errflg, errmsg=errmsg) - call dyn_const(1)%standard_name(stdname, errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) + call dyn_const(1)%standard_name(stdname, errcode=errcode, errmsg=errmsg) end subroutine dlc_liq_init diff --git a/end-to-end-tests/advection/dlc_liq.meta b/end-to-end-tests/advection/dlc_liq.meta index fedb6243..41a69db9 100644 --- a/end-to-end-tests/advection/dlc_liq.meta +++ b/end-to-end-tests/advection/dlc_liq.meta @@ -20,7 +20,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection/test_host.F90 b/end-to-end-tests/advection/test_host.F90 index ed4cf557..dfbd3c54 100644 --- a/end-to-end-tests/advection/test_host.F90 +++ b/end-to-end-tests/advection/test_host.F90 @@ -27,26 +27,26 @@ module test_prog private :: check_suite private :: advect_constituents ! Move data around - private :: check_errflg + private :: check_errcode contains - subroutine check_errflg(subname, errflg, errmsg, errflg_final) - ! If errflg is not zero, print an error message + subroutine check_errcode(subname, errcode, errmsg, errcode_final) + ! If errcode is not zero, print an error message character(len=*), intent(in) :: subname - integer, intent(in) :: errflg + integer, intent(in) :: errcode character(len=*), intent(in) :: errmsg - integer, intent(out) :: errflg_final + integer, intent(out) :: errcode_final - if (errflg /= 0) then - write(6, '(a,i0,4a)') "Error ", errflg, " from ", trim(subname), & + if (errcode /= 0) then + write(6, '(a,i0,4a)') "Error ", errcode, " from ", trim(subname), & ':', trim(errmsg) !Notify test script that a failure occurred: - errflg_final = -1 !Notify test script that a failure occured + errcode_final = -1 !Notify test script that a failure occured end if - end subroutine check_errflg + end subroutine check_errcode logical function check_suite(test_suite) use test_host_ccpp_cap, only: ccpp_physics_suite_part_list @@ -57,20 +57,20 @@ logical function check_suite(test_suite) type(suite_info), intent(in) :: test_suite ! Local variables logical :: check - integer :: errflg + integer :: errcode character(len=512) :: errmsg character(len=128), allocatable :: test_list(:) check_suite = .true. ! First, check the suite parts call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then + errmsg, errcode) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_parts, 'part names', & suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -78,13 +78,13 @@ logical function check_suite(test_suite) end if ! Check the input variables call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.true., output_vars=.false.) - if (errflg == 0) then + errmsg, errcode, input_vars=.true., output_vars=.false.) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_input_vars, & 'input variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -92,13 +92,13 @@ logical function check_suite(test_suite) end if ! Check the output variables call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.false., output_vars=.true.) - if (errflg == 0) then + errmsg, errcode, input_vars=.false., output_vars=.true.) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_output_vars, & 'output variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -106,13 +106,13 @@ logical function check_suite(test_suite) end if ! Check all required variables call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then + errmsg, errcode) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_required_vars, & 'required variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -187,8 +187,8 @@ subroutine test_host(retval, test_suites) character(len=256) :: const_str character(len=512) :: errmsg character(len=512) :: expected_error - integer :: errflg - integer :: errflg_final ! Used to notify testing script of test failure + integer :: errcode + integer :: errcode_final ! Used to notify testing script of test failure real(kind=kind_phys), pointer :: const_ptr(:, :, :) real(kind=kind_phys) :: default_value real(kind=kind_phys) :: check_value @@ -197,7 +197,7 @@ subroutine test_host(retval, test_suites) ! Initialized "final" error flag used to report a failure to the larged ! testing script: - errflg_final = 0 + errcode_final = 0 ! Gather and test the inspection routines num_suites = size(test_suites) @@ -226,36 +226,36 @@ subroutine test_host(retval, test_suites) return end if - errflg = 0 + errcode = 0 errmsg = '' ! Check that is_scheme_constituent works as expected call ccpp_is_scheme_constituent('specific_humidity', & - is_constituent, errflg, errmsg) - call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & - errmsg, errflg_final) + is_constituent, errcode, errmsg) + call check_errcode(subname // "_ccpp_is_scheme_constituent", errcode, & + errmsg, errcode_final) ! specific_humidity should not be an existing constituent if (is_constituent) then write(6, *) "ERROR: specific humidity is already a constituent" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if call ccpp_is_scheme_constituent('cloud_ice_dry_mixing_ratio', & - is_constituent, errflg, errmsg) - call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & - errmsg, errflg_final) + is_constituent, errcode, errmsg) + call check_errcode(subname // "_ccpp_is_scheme_constituent", errcode, & + errmsg, errcode_final) ! cloud_ice_dry_mixing_ratio should be an existing constituent if (.not. is_constituent) then write(6, *) "ERROR: cloud_ice_dry_mixing ratio not found in ", & "host cap constituent list" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if ! Use the suite information to call the register phase do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_register(suite_name=test_suites(sind)%suite_name, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(4a)') 'ERROR in register of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) exit @@ -276,19 +276,19 @@ subroutine test_host(retval, test_suites) long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) call host_constituents(2)%instantiate(std_name="specific_humidity", & long_name="Specific humidity", diag_name='H2O', units="kg kg", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) - call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) - if (errflg == 0) then + errcode=errcode, errmsg=errmsg) + call check_errcode(subname // '.initialize', errcode, errmsg, errcode_final) + if (errcode == 0) then call ccpp_register_constituents(host_constituents, & - errmsg=errmsg, errflg=errflg) + errmsg=errmsg, errcode=errcode) end if ! Check the error - if (errflg == 0) then + if (errcode == 0) then write(6, '(2a)') 'ERROR register_constituents: expected this error: ', & trim(expected_error) else @@ -301,14 +301,14 @@ subroutine test_host(retval, test_suites) ! Now try again but with a compatible constituent - should be ignored when ! the constituents object is created ! Use the suite information to call the register phase - errflg = 0 + errcode = 0 call ccpp_deallocate_dynamic_constituents() deallocate(host_constituents) do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_register(suite_name=test_suites(sind)%suite_name, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(4a)') 'ERROR in register of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) exit @@ -320,12 +320,12 @@ subroutine test_host(retval, test_suites) long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) call host_constituents(2)%instantiate(std_name="specific_humidity", & long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) call host_constituents(3)%instantiate( & std_name='cloud_ice_dry_mixing_ratio', & long_name='Cloud ice dry mixing ratio', & @@ -336,23 +336,23 @@ subroutine test_host(retval, test_suites) default_value=0._kind_phys, & !water_species=.true., & mixing_ratio_type='dry', & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) - call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) - if (errflg == 0) then + call check_errcode(subname // '.initialize', errcode, errmsg, errcode_final) + if (errcode == 0) then call ccpp_register_constituents(host_constituents, & - errmsg=errmsg, errflg=errflg) + errmsg=errmsg, errcode=errcode) end if - if (errflg /= 0) then + if (errcode /= 0) then write(6, '(2a)') 'ERROR register_constituents: ', trim(errmsg) retval = .false. return end if ! Check number of advected constituents - if (errflg == 0) then + if (errcode == 0) then call ccpp_number_constituents(num_advected, errmsg=errmsg, & - errflg=errflg) - call check_errflg(subname // ".num_advected", errflg, errmsg, errflg_final) + errcode=errcode) + call check_errcode(subname // ".num_advected", errcode, errmsg, errcode_final) end if if (num_advected /= 6) then write(6, '(a,i0)') "ERROR: num advected constituents = ", num_advected @@ -360,11 +360,11 @@ subroutine test_host(retval, test_suites) return end if ! Initialize constituent data - call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, errflg=errflg, errmsg=errmsg) + call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, errcode=errcode, errmsg=errmsg) ! Stop tests here if initialization failed (as all other tests will likely ! fail as well: - if (errflg /= 0) then + if (errcode /= 0) then retval = .false. return end if @@ -373,47 +373,47 @@ subroutine test_host(retval, test_suites) const_ptr => ccpp_constituents_array() ! Check if the specific humidity index can be found: - call ccpp_const_get_index('specific_humidity', const_index=index, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_specific_humidity", errflg, errmsg, & - errflg_final) + call ccpp_const_get_index('specific_humidity', const_index=index, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_specific_humidity", errcode, errmsg, & + errcode_final) ! Check if the cloud liquid index can be found: call ccpp_const_get_index(stdname='cloud_liquid_dry_mixing_ratio', & - const_index=index_liq, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_cld_liq", errflg, errmsg, & - errflg_final) + const_index=index_liq, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_cld_liq", errcode, errmsg, & + errcode_final) ! Check if the cloud ice index can be found: call ccpp_const_get_index(stdname='cloud_ice_dry_mixing_ratio', & - const_index=index_ice, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_cld_ice", errflg, errmsg, & - errflg_final) + const_index=index_ice, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_cld_ice", errcode, errmsg, & + errcode_final) ! Check if the dynamic constituents indices can be found - call ccpp_const_get_index(stdname='dyn_const1', const_index=index_dyn1, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_dyn_const1", errflg, errmsg, & - errflg_final) - call ccpp_const_get_index(stdname='dyn_const2_wrt_moist_air', const_index=index_dyn2, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_dyn_const2", errflg, errmsg, & - errflg_final) - call ccpp_const_get_index(stdname='dyn_const3_wrt_moist_air_and_condensed_water', const_index=index_dyn3, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_dyn_const3", errflg, errmsg, & - errflg_final) + call ccpp_const_get_index(stdname='dyn_const1', const_index=index_dyn1, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const1", errcode, errmsg, & + errcode_final) + call ccpp_const_get_index(stdname='dyn_const2_wrt_moist_air', const_index=index_dyn2, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const2", errcode, errmsg, & + errcode_final) + call ccpp_const_get_index(stdname='dyn_const3_wrt_moist_air_and_condensed_water', const_index=index_dyn3, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const3", errcode, errmsg, & + errcode_final) ! Load up the test array indices - call ccpp_const_get_index(stdname=const_std_name, const_index=test_scalar_const_index, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // "." // const_std_name, errflg, errmsg, & - errflg_final) + call ccpp_const_get_index(stdname=const_std_name, const_index=test_scalar_const_index, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // "." // const_std_name, errcode, errmsg, & + errcode_final) do sind = 1, num_consts call ccpp_const_get_index(stdname=std_name_array(sind), & - const_index=test_const_indices(sind), errflg=errflg, errmsg=errmsg) - call check_errflg(subname // "." // std_name_array(sind), errflg, errmsg, & - errflg_final) + const_index=test_const_indices(sind), errcode=errcode, errmsg=errmsg) + call check_errcode(subname // "." // std_name_array(sind), errcode, errmsg, & + errcode_final) end do ! Stop tests here if the index checks failed, as all other tests will ! likely fail as well: - if (errflg_final /= 0) then + if (errcode_final /= 0) then retval = .false. return end if @@ -426,265 +426,265 @@ subroutine test_host(retval, test_suites) const_props => ccpp_model_const_properties() ! Standard name: - call const_props(index)%standard_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index)%standard_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get standard_name for specific_humidity, index = ", & index, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'specific_humidity') then write(6, *) "ERROR: standard name, '", trim(const_str), & "' should be 'specific_humidity'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check standard name for a dynamic constituent - call const_props(index_dyn2)%standard_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%standard_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get standard_name for dyn_const2, index = ", & index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'dyn_const2_wrt_moist_air') then write(6, *) "ERROR: standard name, '", trim(const_str), & "' should be 'dyn_const2_wrt_moist_air'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Long name: - call const_props(index_liq)%long_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%long_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get long_name for cld_liq index = ", & index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'Cloud liquid dry mixing ratio') then write(6, *) "ERROR: long name, '", trim(const_str), & "' should be 'Cloud liquid dry mixing ratio'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check long name for a dynamic constituent - call const_props(index_dyn1)%long_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%long_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get long_name for dyn_const1 index = ", & index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'dyn const1') then write(6, *) "ERROR: long name, '", trim(const_str), & "' should be 'dyn const1'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Diagnostic name: - call const_props(index_liq)%diagnostic_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%diagnostic_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get diagnostic name for cld_liq index = ", & index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'CLDLIQ') then write(6, *) "ERROR: diagnostic name, '", trim(const_str), & "' should be 'CLDLIQ'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check default diagnostic name is set correctly - call const_props(index_ice)%diagnostic_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%diagnostic_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get diagnostic name for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'CLDICE') then write(6, *) "ERROR: diagnostic name, '", trim(const_str), & "' should be 'CLDICE'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check diagnostic name of a dynamic constituent - call const_props(index_dyn2)%diagnostic_name(const_str, errflg, & + call const_props(index_dyn2)%diagnostic_name(const_str, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get diagnostic name for dyn_const2 index = ", & index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'DYNCONST2') then write(6, *) "ERROR: diagnostic name, '", trim(const_str), & "' should be 'DYNCONST2'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Mass mixing ratio: - call const_props(index_ice)%is_mass_mixing_ratio(const_log, errflg, & + call const_props(index_ice)%is_mass_mixing_ratio(const_log, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get mass mixing ratio prop for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: cloud ice is not a mass mixing_ratio" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check mass mixing ratio for a dynamic constituent - call const_props(index_dyn2)%is_mass_mixing_ratio(const_log, errflg, & + call const_props(index_dyn2)%is_mass_mixing_ratio(const_log, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get mass mixing ratio prop for dyn_const2 index = ", & index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const2 is not a mass mixing_ratio" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Dry mixing ratio: - call const_props(index_ice)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: cloud ice mass_mixing_ratio is not dry" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check wet mixing ratio for dynamic constituent 1 - call const_props(index_dyn1)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for dyn_const1 index = ", index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (const_log) then write(6, *) "ERROR: dyn_const1 is dry and should be wet" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_dyn1)%is_wet(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%is_wet(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get wet prop for dyn_const1 index = ", index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const1 is not wet but should be" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check moist mixing ratio for dynamic constituent 2 - call const_props(index_dyn2)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for dyn_const2 index = ", index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (const_log) then write(6, *) "ERROR: dyn_const2 is dry and should be moist" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_dyn2)%is_moist(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%is_moist(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get moist prop for dyn_const2 index = ", index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const2 is not moist but should be" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check dry mixing ratio for dynamic constituent 3 - call const_props(index_dyn3)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn3)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for dyn_const3 index = ", index_dyn3, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const3 is not dry and should be" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- @@ -694,71 +694,71 @@ subroutine test_host(retval, test_suites) ! ------------------- ! Check that a constituent's minimum value defaults to zero: - call const_props(index_dyn2)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get minimum value for dyn_const2 index = ", index_dyn2, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 0._kind_phys) then ! Should be zero write(6, *) "ERROR: 'minimum' should default to zero for all ", & "constituents unless set by host model or scheme metadata." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that a constituent instantiated with a specified minimum value ! actually contains that minimum value property: - call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get minimum value for dyn_const1 index = ", index_dyn1, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 1000._kind_phys) then !Should be 1000 write(6, *) "ERROR: 'minimum' should give a value of 1000 ", & "for dyn_const1, as was set during instantiation." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent's minimum value works ! as expected: - call const_props(index_dyn1)%set_minimum(1._kind_phys, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%set_minimum(1._kind_phys, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set minimum value for dyn_const1 index = ", index_dyn1, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_dyn1)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get minimum value for dyn_const1 index = ", & index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 1._kind_phys) then ! Should now be one write(6, *) "ERROR: 'set_minimum' did not set constituent", & " minimum value correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ---------------------- @@ -767,52 +767,52 @@ subroutine test_host(retval, test_suites) ! Check that a constituent instantiated with a specified molecular ! weight actually contains that molecular weight property value: - call const_props(index)%molar_mass(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index)%molar_mass(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get molecular weight for specific humidity index = ", & index, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 2000._kind_phys) then ! Should be 2000 write(6, *) "ERROR: 'molar_mass' should give a value of 2000 ", & "for specific humidity, as was set during instantiation." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent's molecular weight works ! as expected: - call const_props(index_ice)%set_molar_mass(1._kind_phys, errflg, & + call const_props(index_ice)%set_molar_mass(1._kind_phys, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set molecular weight for cld_ice index = ", index_ice, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_ice)%molar_mass(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_ice)%molar_mass(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get molecular weight for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 1._kind_phys) then ! Should be equal to one write(6, *) "ERROR: 'set_molar_mass' did not set constituent", & " molecular weight value correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- @@ -820,51 +820,51 @@ subroutine test_host(retval, test_suites) ! ------------------- ! Check that being thermodynamically active defaults to False: - call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%is_thermo_active(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get thermo_active prop for cld_ice index = ", index_ice, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check) then ! Should be False write(6, *) "ERROR: 'is_thermo_active' should default to False ", & "for all constituents unless set by host model." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent to be thermodynamically active works ! as expected: - call const_props(index_ice)%set_thermo_active(.true., errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%set_thermo_active(.true., errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set thermo_active prop for cld_ice index = ", index_ice, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_ice)%is_thermo_active(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get thermo_active prop for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (.not. check) then ! Should now be True write(6, *) "ERROR: 'set_thermo_active' did not set", & " thermo_active constituent property correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- @@ -873,149 +873,149 @@ subroutine test_host(retval, test_suites) ! ------------------- ! Check that being a water species defaults to False: - call const_props(index_liq)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get water_species prop for cld_liq index = ", index_liq, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check) then ! Should be False write(6, *) "ERROR: 'is_water_species' should default to False ", & "for all constituents unless set by host model." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent to be a water species works ! as expected: - call const_props(index_liq)%set_water_species(.true., errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%set_water_species(.true., errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set water_species prop for cld_liq index = ", index_liq, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_liq)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_liq)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get water_species prop for cld_liq index = ", & index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (.not. check) then ! Should now be True write(6, *) "ERROR: 'set_water_species' did not set", & " water_species constituent property correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent to be a water species via the ! instantiate call works as expected - call const_props(index_dyn1)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + call const_props(index_dyn1)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & "trying to get water_species prop for dyn_const1 index = ", & index_dyn1, trim(errmsg) end if - if (errflg == 0) then + if (errcode == 0) then if (.not. check) then ! Should now be True write(6, *) "ERROR: 'water_species=.true. did not set", & " water_species constituent property correctly" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_dyn2)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + call const_props(index_dyn2)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & "trying to get water_species prop for dyn_const2 index = ", & index_dyn2, trim(errmsg) end if - if (errflg == 0) then + if (errcode == 0) then if (check) then ! Should now be False write(6, *) "ERROR: 'water_species=.false. did not set", & " water_species constituent property correctly" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- ! Check that setting a constituent's default value works as expected - call const_props(index_liq)%has_default(has_default, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%has_default(has_default, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to check for default for cld_liq index = ", index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. has_default) then write(6, *) "ERROR: cloud_liquid_dry_mixing_ratio should have default but doesn't" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_ice)%has_default(has_default, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%has_default(has_default, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to check for default for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. has_default) then write(6, *) "ERROR: cloud ice_dry_mixing_ratio should have default but doesn't" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_ice)%default_value(default_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%default_value(default_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to grab default for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (default_value /= 0.0_kind_phys) then write(6, *) "ERROR: cloud ice mass_mixing_ratio default is ", default_value, & " but should be 0.0" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ++++++++++++++++++++++++++++++++++ ! Set error flag to the "final" value, because any error ! above will likely result in a large number of failures ! below: - errflg = errflg_final + errcode = errcode_final ! Call ccpp_init do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_init(suite_name=test_suites(sind)%suite_name, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(4a)') 'ERROR in initialize of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) exit @@ -1025,13 +1025,13 @@ subroutine test_host(retval, test_suites) ! Call ccpp_physics_init do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_physics_init( & suite_name=test_suites(sind)%suite_name, & group_name='all', col_start=1, col_end=ncols, & thread_num=1, nthreads=1, nphys_threads=1, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(4a)') 'ERROR in initialize of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) exit @@ -1041,21 +1041,21 @@ subroutine test_host(retval, test_suites) ! Check indices call check_constituent_indices(test_scalar_const_index, test_const_indices, & - errmsg, errflg) - call check_errflg(subname // " check suite indices", errflg, errmsg, & - errflg_final) + errmsg, errcode) + call check_errcode(subname // " check suite indices", errcode, errmsg, & + errcode_final) ! Loop over time steps do time_step = 1, num_time_steps ! Initialize the timestep do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_physics_timestep_init( & suite_name=test_suites(sind)%suite_name, & group_name='all', col_start=1, col_end=ncols, & thread_num=1, nthreads=1, nphys_threads=1, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & trim(errmsg) end if @@ -1063,21 +1063,21 @@ subroutine test_host(retval, test_suites) end do do col_start = 1, ncols, 5 - if (errflg /= 0) then + if (errcode /= 0) then continue end if col_end = min(col_start + 4, ncols) do sind = 1, num_suites do index = 1, size(test_suites(sind)%suite_parts) - if (errflg == 0) then + if (errcode == 0) then call ccpp_physics_run( & suite_name=test_suites(sind)%suite_name, & group_name=test_suites(sind)%suite_parts(index), & col_start=col_start, col_end=col_end, & thread_num=1, nthreads=1, nphys_threads=1, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(5a)') trim(test_suites(sind)%suite_name), & '/', trim(test_suites(sind)%suite_parts(index)),& ': ', trim(errmsg) @@ -1089,19 +1089,19 @@ subroutine test_host(retval, test_suites) end do ! Check indices call check_constituent_indices(test_scalar_const_index, test_const_indices, & - errmsg, errflg) - call check_errflg(subname // " check suite indices", errflg, errmsg, & - errflg_final) + errmsg, errcode) + call check_errcode(subname // " check suite indices", errcode, errmsg, & + errcode_final) do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_physics_timestep_final( & suite_name=test_suites(sind)%suite_name, & group_name='all', col_start=1, col_end=ncols, & thread_num=1, nthreads=1, nphys_threads=1, & - errmsg=errmsg, errflg=errflg) + errmsg=errmsg, errcode=errcode) end if - if (errflg /= 0) then + if (errcode /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & trim(errmsg) exit @@ -1109,19 +1109,19 @@ subroutine test_host(retval, test_suites) end do ! Run "dycore" - if (errflg == 0) then + if (errcode == 0) then call advect_constituents() end if end do ! End time step loop do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_physics_final( & suite_name=test_suites(sind)%suite_name, & group_name='all', col_start=1, col_end=ncols, & thread_num=1, nthreads=1, nphys_threads=1, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & trim(errmsg) write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & @@ -1132,10 +1132,10 @@ subroutine test_host(retval, test_suites) end do do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_final(suite_name=test_suites(sind)%suite_name, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & trim(errmsg) write(6, '(2a)') 'An error occurred in ccpp_final, ', & @@ -1148,24 +1148,24 @@ subroutine test_host(retval, test_suites) call ccpp_deallocate_dynamic_constituents() deallocate(host_constituents) - if (errflg == 0) then + if (errcode == 0) then ! Run finished without error, check answers if (compare_data(num_advected)) then write(6, *) 'Answers are correct!' - errflg = 0 + errcode = 0 else write(6, *) 'Answers are not correct!' - errflg = -1 + errcode = -1 end if end if - ! Make sure "final" flag is non-zero if "errflg" is: - if (errflg /= 0) then - errflg_final = -1 ! Notify test script that a failure occured + ! Make sure "final" flag is non-zero if "errcode" is: + if (errcode /= 0) then + errcode_final = -1 ! Notify test script that a failure occured end if ! Set return value to False if any errors were found: - retval = errflg_final == 0 + retval = errcode_final == 0 end subroutine test_host diff --git a/end-to-end-tests/advection/test_host.meta b/end-to-end-tests/advection/test_host.meta index ab33172f..e69dafd9 100644 --- a/end-to-end-tests/advection/test_host.meta +++ b/end-to-end-tests/advection/test_host.meta @@ -30,7 +30,7 @@ dimensions = () type = character kind = len=512 -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection/test_host_data.F90 b/end-to-end-tests/advection/test_host_data.F90 index f360ad79..4bcb753b 100644 --- a/end-to-end-tests/advection/test_host_data.F90 +++ b/end-to-end-tests/advection/test_host_data.F90 @@ -29,26 +29,26 @@ module test_host_data contains - subroutine check_constituent_indices(test_index, test_indices, errmsg, errflg) + subroutine check_constituent_indices(test_index, test_indices, errmsg, errcode) ! Check constituent indices against what was found by suite ! indices are passed in rather than looked up to avoid a dependency loop ! Dummy arguments integer, intent(in) :: test_index ! scalar const index from host integer, intent(in) :: test_indices(:) ! array_test_indices from host character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode ! Local variable integer :: indx integer :: emstrt - errflg = 0 + errcode = 0 errmsg = '' if (test_index /= const_index) then emstrt = len_trim(errmsg) + 1 write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_index_check for ', & const_std_name, test_index, ' /= ', const_index - errflg = errflg + 1 + errcode = errcode + 1 end if do indx = 1, num_consts if (test_indices(indx) /= const_inds(indx)) then @@ -59,7 +59,7 @@ subroutine check_constituent_indices(test_index, test_indices, errmsg, errflg) end if write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_indices_check for ', & std_name_array(indx), test_indices(indx), ' /= ', const_inds(indx) - errflg = errflg + 1 + errcode = errcode + 1 end if end do diff --git a/end-to-end-tests/advection_auto_clone/cld_ice.F90 b/end-to-end-tests/advection_auto_clone/cld_ice.F90 index bf19b979..e3fc2abd 100644 --- a/end-to-end-tests/advection_auto_clone/cld_ice.F90 +++ b/end-to-end-tests/advection_auto_clone/cld_ice.F90 @@ -49,7 +49,7 @@ end subroutine cld_ice_register !! \htmlinclude arg_table_cld_ice_run.html !! subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & - errmsg, errflg) + errmsg, errcode) integer, intent(in) :: ncol real(kind=kind_phys), intent(in) :: timestep @@ -58,7 +58,7 @@ subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & real(kind=kind_phys), intent(in) :: ps(:) real(kind=kind_phys), intent(inout) :: cld_ice_array(:, :) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: icol @@ -66,7 +66,7 @@ subroutine cld_ice_run(ncol, timestep, temp, qv, ps, cld_ice_array, & real(kind=kind_phys) :: frz errmsg = '' - errflg = 0 + errcode = 0 ! Apply state-of-the-art thermodynamics :) do icol = 1, ncol @@ -87,14 +87,14 @@ end subroutine cld_ice_run !> \section arg_table_cld_ice_init Argument Table !! \htmlinclude arg_table_cld_ice_init.html !! - subroutine cld_ice_init(tfreeze, errmsg, errflg) + subroutine cld_ice_init(tfreeze, errmsg, errcode) real(kind=kind_phys), intent(in) :: tfreeze character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 + errcode = 0 tcld = tfreeze - 20.0_kind_phys end subroutine cld_ice_init @@ -109,13 +109,13 @@ end subroutine cld_ice_init !! and the subroutine are parsed correctly. !! @{ - subroutine cld_ice_final(errmsg, errflg) + subroutine cld_ice_final(errmsg, errcode) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 + errcode = 0 end subroutine cld_ice_final diff --git a/end-to-end-tests/advection_auto_clone/cld_ice.meta b/end-to-end-tests/advection_auto_clone/cld_ice.meta index df0a925d..7496da74 100644 --- a/end-to-end-tests/advection_auto_clone/cld_ice.meta +++ b/end-to-end-tests/advection_auto_clone/cld_ice.meta @@ -83,7 +83,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -109,7 +109,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -128,7 +128,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection_auto_clone/cld_liq.F90 b/end-to-end-tests/advection_auto_clone/cld_liq.F90 index 0eaffbdb..586a68b3 100644 --- a/end-to-end-tests/advection_auto_clone/cld_liq.F90 +++ b/end-to-end-tests/advection_auto_clone/cld_liq.F90 @@ -18,15 +18,15 @@ module cld_liq !> \section arg_table_cld_liq_register Argument Table !! \htmlinclude arg_table_cld_liq_register.html !! - subroutine cld_liq_register(dyn_const, errmsg, errflg) + subroutine cld_liq_register(dyn_const, errmsg, errcode) type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 - allocate(dyn_const(1), stat=errflg) - if (errflg /= 0) then + errcode = 0 + allocate(dyn_const(1), stat=errcode) + if (errcode /= 0) then errmsg = 'Error allocating dyn_const in cld_liq_register' return end if @@ -34,7 +34,7 @@ subroutine cld_liq_register(dyn_const, errmsg, errflg) diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & vertical_dim='vertical_layer_dimension', advected=.true., & water_species=.true., mixing_ratio_type='dry', & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) end subroutine cld_liq_register @@ -42,7 +42,7 @@ end subroutine cld_liq_register !! \htmlinclude arg_table_cld_liq_run.html !! subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & - cld_liq_tend, errmsg, errflg) + cld_liq_tend, errmsg, errcode) integer, intent(in) :: ncol real(kind=kind_phys), intent(in) :: timestep @@ -52,7 +52,7 @@ subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & real(kind=kind_phys), intent(in) :: ps(:) real(kind=kind_phys), intent(out) :: cld_liq_tend(:, :) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: icol @@ -60,7 +60,7 @@ subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & real(kind=kind_phys) :: cond errmsg = '' - errflg = 0 + errcode = 0 ! Apply state-of-the-art thermodynamics :) do icol = 1, ncol @@ -82,18 +82,18 @@ end subroutine cld_liq_run !> \section arg_table_cld_liq_init Argument Table !! \htmlinclude arg_table_cld_liq_init.html !! - subroutine cld_liq_init(tfreeze, cld_liq_array, tcld, errmsg, errflg) + subroutine cld_liq_init(tfreeze, cld_liq_array, tcld, errmsg, errcode) real(kind=kind_phys), intent(in) :: tfreeze real(kind=kind_phys), intent(inout) :: cld_liq_array(:, :) real(kind=kind_phys), intent(out) :: tcld character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode ! This routine currently does nothing errmsg = '' - errflg = 0 + errcode = 0 cld_liq_array = 0.0_kind_phys tcld = tfreeze - 20.0_kind_phys diff --git a/end-to-end-tests/advection_auto_clone/cld_liq.meta b/end-to-end-tests/advection_auto_clone/cld_liq.meta index 0b73cfa0..7013aea8 100644 --- a/end-to-end-tests/advection_auto_clone/cld_liq.meta +++ b/end-to-end-tests/advection_auto_clone/cld_liq.meta @@ -19,7 +19,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -86,7 +86,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -128,7 +128,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection_auto_clone/const_indices.F90 b/end-to-end-tests/advection_auto_clone/const_indices.F90 index bc3b46a7..5c77e29c 100644 --- a/end-to-end-tests/advection_auto_clone/const_indices.F90 +++ b/end-to-end-tests/advection_auto_clone/const_indices.F90 @@ -17,7 +17,7 @@ module const_indices !! \htmlinclude arg_table_const_indices_run.html !! subroutine const_indices_run(const_std_name, num_consts, test_stdname_array, & - const_index, const_inds, errmsg, errflg) + const_index, const_inds, errmsg, errcode) use ccpp_constituent_prop_mod, only: int_unassigned use ccpp_scheme_utils, only: ccpp_constituent_index use ccpp_scheme_utils, only: ccpp_constituent_indices @@ -28,28 +28,28 @@ subroutine const_indices_run(const_std_name, num_consts, test_stdname_array, & integer, intent(out) :: const_index integer, intent(out) :: const_inds(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: indx integer :: test_indx errmsg = '' - errflg = 0 + errcode = 0 ! Find the constituent index for - call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) - if (errflg == 0) then - call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) + call ccpp_constituent_index(const_std_name, const_index, errcode, errmsg) + if (errcode == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errcode, errmsg) end if ! Check that a non-registered constituent is detectable but ! does not cause an error - if (errflg == 0) then - call ccpp_constituent_index('unobtainium', test_indx, errflg, errmsg) + if (errcode == 0) then + call ccpp_constituent_index('unobtainium', test_indx, errcode, errmsg) if (test_indx /= int_unassigned) then - if (errflg == 0) then + if (errcode == 0) then ! Do not add an error if one is already reported - errflg = 2 + errcode = 2 write(errmsg, '(2a,i0,a,i0)') "ccpp_constituent_index called for ", & "'unobtainium' returned an index of ", test_indx, ", not ", & int_unassigned @@ -63,7 +63,7 @@ end subroutine const_indices_run !! \htmlinclude arg_table_const_indices_init.html !! subroutine const_indices_init(const_std_name, num_consts, test_stdname_array, & - const_index, const_inds, errmsg, errflg) + const_index, const_inds, errmsg, errcode) use ccpp_scheme_utils, only: ccpp_constituent_index, & ccpp_constituent_indices @@ -73,18 +73,18 @@ subroutine const_indices_init(const_std_name, num_consts, test_stdname_array, & integer, intent(out) :: const_index integer, intent(out) :: const_inds(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: indx errmsg = '' - errflg = 0 + errcode = 0 ! Find the constituent index for - call ccpp_constituent_index(const_std_name, const_index, errflg, errmsg) - if (errflg == 0) then - call ccpp_constituent_indices(test_stdname_array, const_inds, errflg, errmsg) + call ccpp_constituent_index(const_std_name, const_index, errcode, errmsg) + if (errcode == 0) then + call ccpp_constituent_indices(test_stdname_array, const_inds, errcode, errmsg) end if end subroutine const_indices_init diff --git a/end-to-end-tests/advection_auto_clone/const_indices.meta b/end-to-end-tests/advection_auto_clone/const_indices.meta index a4cc98e2..147e2ccb 100644 --- a/end-to-end-tests/advection_auto_clone/const_indices.meta +++ b/end-to-end-tests/advection_auto_clone/const_indices.meta @@ -47,7 +47,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -99,7 +99,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection_auto_clone/dlc_liq.F90 b/end-to-end-tests/advection_auto_clone/dlc_liq.F90 index 20ff4b7b..134e3aed 100644 --- a/end-to-end-tests/advection_auto_clone/dlc_liq.F90 +++ b/end-to-end-tests/advection_auto_clone/dlc_liq.F90 @@ -16,25 +16,25 @@ module dlc_liq !> \section arg_table_dlc_liq_init Argument Table !! \htmlinclude arg_table_dlc_liq_init.html !! - subroutine dlc_liq_init(dyn_const, errmsg, errflg) + subroutine dlc_liq_init(dyn_const, errmsg, errcode) type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode character(len=256) :: stdname errmsg = '' - errflg = 0 - allocate(dyn_const(1), stat=errflg) - if (errflg /= 0) then + errcode = 0 + allocate(dyn_const(1), stat=errcode) + if (errcode /= 0) then errmsg = 'Error allocating dyn_const in dlc_liq_init' return end if call dyn_const(1)%instantiate(std_name="dyn_const3", long_name='dyn const3', & diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & vertical_dim='vertical_layer_dimension', advected=.true., & - errcode=errflg, errmsg=errmsg) - call dyn_const(1)%standard_name(stdname, errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) + call dyn_const(1)%standard_name(stdname, errcode=errcode, errmsg=errmsg) end subroutine dlc_liq_init diff --git a/end-to-end-tests/advection_auto_clone/dlc_liq.meta b/end-to-end-tests/advection_auto_clone/dlc_liq.meta index fedb6243..41a69db9 100644 --- a/end-to-end-tests/advection_auto_clone/dlc_liq.meta +++ b/end-to-end-tests/advection_auto_clone/dlc_liq.meta @@ -20,7 +20,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection_auto_clone/test_host.F90 b/end-to-end-tests/advection_auto_clone/test_host.F90 index cd41f0f7..7845ca79 100644 --- a/end-to-end-tests/advection_auto_clone/test_host.F90 +++ b/end-to-end-tests/advection_auto_clone/test_host.F90 @@ -27,26 +27,26 @@ module test_prog private :: check_suite private :: advect_constituents ! Move data around - private :: check_errflg + private :: check_errcode contains - subroutine check_errflg(subname, errflg, errmsg, errflg_final) - ! If errflg is not zero, print an error message + subroutine check_errcode(subname, errcode, errmsg, errcode_final) + ! If errcode is not zero, print an error message character(len=*), intent(in) :: subname - integer, intent(in) :: errflg + integer, intent(in) :: errcode character(len=*), intent(in) :: errmsg - integer, intent(out) :: errflg_final + integer, intent(out) :: errcode_final - if (errflg /= 0) then - write(6, '(a,i0,4a)') "Error ", errflg, " from ", trim(subname), & + if (errcode /= 0) then + write(6, '(a,i0,4a)') "Error ", errcode, " from ", trim(subname), & ':', trim(errmsg) !Notify test script that a failure occurred: - errflg_final = -1 !Notify test script that a failure occured + errcode_final = -1 !Notify test script that a failure occured end if - end subroutine check_errflg + end subroutine check_errcode logical function check_suite(test_suite) use test_host_ccpp_cap, only: ccpp_physics_suite_part_list @@ -57,20 +57,20 @@ logical function check_suite(test_suite) type(suite_info), intent(in) :: test_suite ! Local variables logical :: check - integer :: errflg + integer :: errcode character(len=512) :: errmsg character(len=128), allocatable :: test_list(:) check_suite = .true. ! First, check the suite parts call ccpp_physics_suite_part_list(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then + errmsg, errcode) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_parts, 'part names', & suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -78,13 +78,13 @@ logical function check_suite(test_suite) end if ! Check the input variables call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.true., output_vars=.false.) - if (errflg == 0) then + errmsg, errcode, input_vars=.true., output_vars=.false.) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_input_vars, & 'input variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -92,13 +92,13 @@ logical function check_suite(test_suite) end if ! Check the output variables call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg, input_vars=.false., output_vars=.true.) - if (errflg == 0) then + errmsg, errcode, input_vars=.false., output_vars=.true.) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_output_vars, & 'output variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -106,13 +106,13 @@ logical function check_suite(test_suite) end if ! Check all required variables call ccpp_physics_suite_variables(test_suite%suite_name, test_list, & - errmsg, errflg) - if (errflg == 0) then + errmsg, errcode) + if (errcode == 0) then check = check_list(test_list, test_suite%suite_required_vars, & 'required variable names', suite_name=test_suite%suite_name) else check = .false. - write(6, '(a,i0,2a)') 'ERROR ', errflg, ': ', trim(errmsg) + write(6, '(a,i0,2a)') 'ERROR ', errcode, ': ', trim(errmsg) end if check_suite = check_suite .and. check if (allocated(test_list)) then @@ -187,8 +187,8 @@ subroutine test_host(retval, test_suites) character(len=256) :: const_str character(len=512) :: errmsg character(len=512) :: expected_error - integer :: errflg - integer :: errflg_final ! Used to notify testing script of test failure + integer :: errcode + integer :: errcode_final ! Used to notify testing script of test failure real(kind=kind_phys), pointer :: const_ptr(:, :, :) real(kind=kind_phys) :: default_value real(kind=kind_phys) :: check_value @@ -197,7 +197,7 @@ subroutine test_host(retval, test_suites) ! Initialized "final" error flag used to report a failure to the larged ! testing script: - errflg_final = 0 + errcode_final = 0 ! Gather and test the inspection routines num_suites = size(test_suites) @@ -226,36 +226,36 @@ subroutine test_host(retval, test_suites) return end if - errflg = 0 + errcode = 0 errmsg = '' ! Check that is_scheme_constituent works as expected call ccpp_is_scheme_constituent('specific_humidity', & - is_constituent, errflg, errmsg) - call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & - errmsg, errflg_final) + is_constituent, errcode, errmsg) + call check_errcode(subname // "_ccpp_is_scheme_constituent", errcode, & + errmsg, errcode_final) ! specific_humidity should not be an existing constituent if (is_constituent) then write(6, *) "ERROR: specific humidity is already a constituent" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if call ccpp_is_scheme_constituent('cloud_ice_dry_mixing_ratio', & - is_constituent, errflg, errmsg) - call check_errflg(subname // "_ccpp_is_scheme_constituent", errflg, & - errmsg, errflg_final) + is_constituent, errcode, errmsg) + call check_errcode(subname // "_ccpp_is_scheme_constituent", errcode, & + errmsg, errcode_final) ! cloud_ice_dry_mixing_ratio should be an existing constituent if (.not. is_constituent) then write(6, *) "ERROR: cloud_ice_dry_mixing ratio not found in ", & "host cap constituent list" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if ! Use the suite information to call the register phase do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_register(suite_name=test_suites(sind)%suite_name, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(4a)') 'ERROR in register of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) exit @@ -276,19 +276,19 @@ subroutine test_host(retval, test_suites) long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) call host_constituents(2)%instantiate(std_name="specific_humidity", & long_name="Specific humidity", diag_name='H2O', units="kg kg", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) - call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) - if (errflg == 0) then + errcode=errcode, errmsg=errmsg) + call check_errcode(subname // '.initialize', errcode, errmsg, errcode_final) + if (errcode == 0) then call ccpp_register_constituents(host_constituents, & - errmsg=errmsg, errflg=errflg) + errmsg=errmsg, errcode=errcode) end if ! Check the error - if (errflg == 0) then + if (errcode == 0) then write(6, '(2a)') 'ERROR register_constituents: expected this error: ', & trim(expected_error) else @@ -301,14 +301,14 @@ subroutine test_host(retval, test_suites) ! Now try again but with a compatible constituent - should be ignored when ! the constituents object is created ! Use the suite information to call the register phase - errflg = 0 + errcode = 0 call ccpp_deallocate_dynamic_constituents() deallocate(host_constituents) do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_register(suite_name=test_suites(sind)%suite_name, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(4a)') 'ERROR in register of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) exit @@ -320,27 +320,27 @@ subroutine test_host(retval, test_suites) long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) call host_constituents(2)%instantiate(std_name="specific_humidity", & long_name="Specific humidity", diag_name='H2O', units="kg kg-1", & vertical_dim="vertical_layer_dimension", advected=.true., & min_value=1000._kind_phys, molar_mass=2000._kind_phys, & - errcode=errflg, errmsg=errmsg) - call check_errflg(subname // '.initialize', errflg, errmsg, errflg_final) - if (errflg == 0) then + errcode=errcode, errmsg=errmsg) + call check_errcode(subname // '.initialize', errcode, errmsg, errcode_final) + if (errcode == 0) then call ccpp_register_constituents(host_constituents, & - errmsg=errmsg, errflg=errflg) + errmsg=errmsg, errcode=errcode) end if - if (errflg /= 0) then + if (errcode /= 0) then write(6, '(2a)') 'ERROR register_constituents: ', trim(errmsg) retval = .false. return end if ! Check number of advected constituents - if (errflg == 0) then + if (errcode == 0) then call ccpp_number_constituents(num_advected, errmsg=errmsg, & - errflg=errflg) - call check_errflg(subname // ".num_advected", errflg, errmsg, errflg_final) + errcode=errcode) + call check_errcode(subname // ".num_advected", errcode, errmsg, errcode_final) end if if (num_advected /= 6) then write(6, '(a,i0)') "ERROR: num advected constituents = ", num_advected @@ -348,11 +348,11 @@ subroutine test_host(retval, test_suites) return end if ! Initialize constituent data - call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, errflg=errflg, errmsg=errmsg) + call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, errcode=errcode, errmsg=errmsg) ! Stop tests here if initialization failed (as all other tests will likely ! fail as well: - if (errflg /= 0) then + if (errcode /= 0) then retval = .false. return end if @@ -361,47 +361,47 @@ subroutine test_host(retval, test_suites) const_ptr => ccpp_constituents_array() ! Check if the specific humidity index can be found: - call ccpp_const_get_index('specific_humidity', const_index=index, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_specific_humidity", errflg, errmsg, & - errflg_final) + call ccpp_const_get_index('specific_humidity', const_index=index, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_specific_humidity", errcode, errmsg, & + errcode_final) ! Check if the cloud liquid index can be found: call ccpp_const_get_index(stdname='cloud_liquid_dry_mixing_ratio', & - const_index=index_liq, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_cld_liq", errflg, errmsg, & - errflg_final) + const_index=index_liq, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_cld_liq", errcode, errmsg, & + errcode_final) ! Check if the cloud ice index can be found: call ccpp_const_get_index(stdname='cloud_ice_dry_mixing_ratio', & - const_index=index_ice, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_cld_ice", errflg, errmsg, & - errflg_final) + const_index=index_ice, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_cld_ice", errcode, errmsg, & + errcode_final) ! Check if the dynamic constituents indices can be found - call ccpp_const_get_index(stdname='dyn_const1', const_index=index_dyn1, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_dyn_const1", errflg, errmsg, & - errflg_final) - call ccpp_const_get_index(stdname='dyn_const2_wrt_moist_air', const_index=index_dyn2, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_dyn_const2", errflg, errmsg, & - errflg_final) - call ccpp_const_get_index(stdname='dyn_const3_wrt_moist_air_and_condensed_water', const_index=index_dyn3, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // ".index_dyn_const3", errflg, errmsg, & - errflg_final) + call ccpp_const_get_index(stdname='dyn_const1', const_index=index_dyn1, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const1", errcode, errmsg, & + errcode_final) + call ccpp_const_get_index(stdname='dyn_const2_wrt_moist_air', const_index=index_dyn2, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const2", errcode, errmsg, & + errcode_final) + call ccpp_const_get_index(stdname='dyn_const3_wrt_moist_air_and_condensed_water', const_index=index_dyn3, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // ".index_dyn_const3", errcode, errmsg, & + errcode_final) ! Load up the test array indices - call ccpp_const_get_index(stdname=const_std_name, const_index=test_scalar_const_index, errflg=errflg, errmsg=errmsg) - call check_errflg(subname // "." // const_std_name, errflg, errmsg, & - errflg_final) + call ccpp_const_get_index(stdname=const_std_name, const_index=test_scalar_const_index, errcode=errcode, errmsg=errmsg) + call check_errcode(subname // "." // const_std_name, errcode, errmsg, & + errcode_final) do sind = 1, num_consts call ccpp_const_get_index(stdname=std_name_array(sind), & - const_index=test_const_indices(sind), errflg=errflg, errmsg=errmsg) - call check_errflg(subname // "." // std_name_array(sind), errflg, errmsg, & - errflg_final) + const_index=test_const_indices(sind), errcode=errcode, errmsg=errmsg) + call check_errcode(subname // "." // std_name_array(sind), errcode, errmsg, & + errcode_final) end do ! Stop tests here if the index checks failed, as all other tests will ! likely fail as well: - if (errflg_final /= 0) then + if (errcode_final /= 0) then retval = .false. return end if @@ -414,265 +414,265 @@ subroutine test_host(retval, test_suites) const_props => ccpp_model_const_properties() ! Standard name: - call const_props(index)%standard_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index)%standard_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get standard_name for specific_humidity, index = ", & index, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'specific_humidity') then write(6, *) "ERROR: standard name, '", trim(const_str), & "' should be 'specific_humidity'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check standard name for a dynamic constituent - call const_props(index_dyn2)%standard_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%standard_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get standard_name for dyn_const2, index = ", & index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'dyn_const2_wrt_moist_air') then write(6, *) "ERROR: standard name, '", trim(const_str), & "' should be 'dyn_const2_wrt_moist_air'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Long name: - call const_props(index_liq)%long_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%long_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get long_name for cld_liq index = ", & index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'Cloud liquid dry mixing ratio') then write(6, *) "ERROR: long name, '", trim(const_str), & "' should be 'Cloud liquid dry mixing ratio'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check long name for a dynamic constituent - call const_props(index_dyn1)%long_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%long_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get long_name for dyn_const1 index = ", & index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'dyn const1') then write(6, *) "ERROR: long name, '", trim(const_str), & "' should be 'dyn const1'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Diagnostic name: - call const_props(index_liq)%diagnostic_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%diagnostic_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get diagnostic name for cld_liq index = ", & index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'CLDLIQ') then write(6, *) "ERROR: diagnostic name, '", trim(const_str), & "' should be 'CLDLIQ'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check default diagnostic name is set correctly - call const_props(index_ice)%diagnostic_name(const_str, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%diagnostic_name(const_str, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get diagnostic name for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'cld_ice_array') then write(6, *) "ERROR: diagnostic name, '", trim(const_str), & "' should be 'cld_ice_array'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check diagnostic name of a dynamic constituent - call const_props(index_dyn2)%diagnostic_name(const_str, errflg, & + call const_props(index_dyn2)%diagnostic_name(const_str, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get diagnostic name for dyn_const2 index = ", & index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (trim(const_str) /= 'DYNCONST2') then write(6, *) "ERROR: diagnostic name, '", trim(const_str), & "' should be 'DYNCONST2'" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Mass mixing ratio: - call const_props(index_ice)%is_mass_mixing_ratio(const_log, errflg, & + call const_props(index_ice)%is_mass_mixing_ratio(const_log, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get mass mixing ratio prop for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: cloud ice is not a mass mixing_ratio" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check mass mixing ratio for a dynamic constituent - call const_props(index_dyn2)%is_mass_mixing_ratio(const_log, errflg, & + call const_props(index_dyn2)%is_mass_mixing_ratio(const_log, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get mass mixing ratio prop for dyn_const2 index = ", & index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const2 is not a mass mixing_ratio" - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Dry mixing ratio: - call const_props(index_ice)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: cloud ice mass_mixing_ratio is not dry" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check wet mixing ratio for dynamic constituent 1 - call const_props(index_dyn1)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for dyn_const1 index = ", index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (const_log) then write(6, *) "ERROR: dyn_const1 is dry and should be wet" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_dyn1)%is_wet(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%is_wet(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get wet prop for dyn_const1 index = ", index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const1 is not wet but should be" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check moist mixing ratio for dynamic constituent 2 - call const_props(index_dyn2)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for dyn_const2 index = ", index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (const_log) then write(6, *) "ERROR: dyn_const2 is dry and should be moist" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_dyn2)%is_moist(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%is_moist(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get moist prop for dyn_const2 index = ", index_dyn2, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const2 is not moist but should be" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check dry mixing ratio for dynamic constituent 3 - call const_props(index_dyn3)%is_dry(const_log, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn3)%is_dry(const_log, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get dry prop for dyn_const3 index = ", index_dyn3, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. const_log) then write(6, *) "ERROR: dyn_const3 is not dry and should be" - errflg_final = -1 + errcode_final = -1 end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- @@ -682,71 +682,71 @@ subroutine test_host(retval, test_suites) ! ------------------- ! Check that a constituent's minimum value defaults to zero: - call const_props(index_dyn2)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn2)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get minimum value for dyn_const2 index = ", index_dyn2, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 0._kind_phys) then ! Should be zero write(6, *) "ERROR: 'minimum' should default to zero for all ", & "constituents unless set by host model or scheme metadata." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that a constituent instantiated with a specified minimum value ! actually contains that minimum value property: - call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get minimum value for dyn_const1 index = ", index_dyn1, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 1000._kind_phys) then !Should be 1000 write(6, *) "ERROR: 'minimum' should give a value of 1000 ", & "for dyn_const1, as was set during instantiation." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent's minimum value works ! as expected: - call const_props(index_dyn1)%set_minimum(1._kind_phys, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_dyn1)%set_minimum(1._kind_phys, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set minimum value for dyn_const1 index = ", index_dyn1, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_dyn1)%minimum(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_dyn1)%minimum(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get minimum value for dyn_const1 index = ", & index_dyn1, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 1._kind_phys) then ! Should now be one write(6, *) "ERROR: 'set_minimum' did not set constituent", & " minimum value correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ---------------------- @@ -755,52 +755,52 @@ subroutine test_host(retval, test_suites) ! Check that a constituent instantiated with a specified molecular ! weight actually contains that molecular weight property value: - call const_props(index)%molar_mass(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index)%molar_mass(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get molecular weight for specific humidity index = ", & index, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 2000._kind_phys) then ! Should be 2000 write(6, *) "ERROR: 'molar_mass' should give a value of 2000 ", & "for specific humidity, as was set during instantiation." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent's molecular weight works ! as expected: - call const_props(index_ice)%set_molar_mass(1._kind_phys, errflg, & + call const_props(index_ice)%set_molar_mass(1._kind_phys, errcode, & errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set molecular weight for cld_ice index = ", index_ice, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_ice)%molar_mass(check_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_ice)%molar_mass(check_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get molecular weight for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (check_value /= 1._kind_phys) then ! Should be equal to one write(6, *) "ERROR: 'set_molar_mass' did not set constituent", & " molecular weight value correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- @@ -808,51 +808,51 @@ subroutine test_host(retval, test_suites) ! ------------------- ! Check that being thermodynamically active defaults to False: - call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%is_thermo_active(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get thermo_active prop for cld_ice index = ", index_ice, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check) then ! Should be False write(6, *) "ERROR: 'is_thermo_active' should default to False ", & "for all constituents unless set by host model." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent to be thermodynamically active works ! as expected: - call const_props(index_ice)%set_thermo_active(.true., errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%set_thermo_active(.true., errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set thermo_active prop for cld_ice index = ", index_ice, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_ice)%is_thermo_active(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_ice)%is_thermo_active(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get thermo_active prop for cld_ice index = ", & index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (.not. check) then ! Should now be True write(6, *) "ERROR: 'set_thermo_active' did not set", & " thermo_active constituent property correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- @@ -861,149 +861,149 @@ subroutine test_host(retval, test_suites) ! ------------------- ! Check that being a water species defaults to False: - call const_props(index_liq)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to get water_species prop for cld_liq index = ", index_liq, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (check) then ! Should be False write(6, *) "ERROR: 'is_water_species' should default to False ", & "for all constituents unless set by host model." - errflg_final = -1 ! Notify test script that a failure occured + errcode_final = -1 ! Notify test script that a failure occured end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent to be a water species works ! as expected: - call const_props(index_liq)%set_water_species(.true., errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%set_water_species(.true., errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to set water_species prop for cld_liq index = ", index_liq, & trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then - call const_props(index_liq)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + if (errcode == 0) then + call const_props(index_liq)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & " trying to get water_species prop for cld_liq index = ", & index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if end if - if (errflg == 0) then + if (errcode == 0) then if (.not. check) then ! Should now be True write(6, *) "ERROR: 'set_water_species' did not set", & " water_species constituent property correctly." - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! Check that setting a constituent to be a water species via the ! instantiate call works as expected - call const_props(index_dyn1)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + call const_props(index_dyn1)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & "trying to get water_species prop for dyn_const1 index = ", & index_dyn1, trim(errmsg) end if - if (errflg == 0) then + if (errcode == 0) then if (.not. check) then ! Should now be True write(6, *) "ERROR: 'water_species=.true. did not set", & " water_species constituent property correctly" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_dyn2)%is_water_species(check, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errflg, & + call const_props(index_dyn2)%is_water_species(check, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,a,i0,/,a)') "ERROR: Error, ", errcode, & "trying to get water_species prop for dyn_const2 index = ", & index_dyn2, trim(errmsg) end if - if (errflg == 0) then + if (errcode == 0) then if (check) then ! Should now be False write(6, *) "ERROR: 'water_species=.false. did not set", & " water_species constituent property correctly" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ------------------- ! Check that setting a constituent's default value works as expected - call const_props(index_liq)%has_default(has_default, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_liq)%has_default(has_default, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to check for default for cld_liq index = ", index_liq, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (has_default) then write(6, *) "ERROR: cloud liquid mass_mixing_ratio should not have default but does" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_ice)%has_default(has_default, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%has_default(has_default, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to check for default for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (.not. has_default) then write(6, *) "ERROR: cloud ice_dry_mixing_ratio should have default but doesn't" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if - call const_props(index_ice)%default_value(default_value, errflg, errmsg) - if (errflg /= 0) then - write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errflg, " trying ", & + call const_props(index_ice)%default_value(default_value, errcode, errmsg) + if (errcode /= 0) then + write(6, '(a,i0,2a,i0,/,a)') "ERROR: Error, ", errcode, " trying ", & "to grab default for cld_ice index = ", index_ice, trim(errmsg) - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if - if (errflg == 0) then + if (errcode == 0) then if (default_value /= 0.0_kind_phys) then write(6, *) "ERROR: cloud ice mass_mixing_ratio default is ", default_value, & " but should be 0.0" - errflg_final = -1 ! Notify test script that a failure occurred + errcode_final = -1 ! Notify test script that a failure occurred end if else ! Reset error flag to continue testing other properties: - errflg = 0 + errcode = 0 end if ! ++++++++++++++++++++++++++++++++++ ! Set error flag to the "final" value, because any error ! above will likely result in a large number of failures ! below: - errflg = errflg_final + errcode = errcode_final ! Call ccpp_init do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_init(suite_name=test_suites(sind)%suite_name, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(4a)') 'ERROR in initialize of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) exit @@ -1013,13 +1013,13 @@ subroutine test_host(retval, test_suites) ! Call ccpp_physics_init do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_physics_init( & suite_name=test_suites(sind)%suite_name, & group_name='all', col_start=1, col_end=ncols, & thread_num=1, nthreads=1, nphys_threads=1, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(4a)') 'ERROR in initialize of ', & trim(test_suites(sind)%suite_name), ': ', trim(errmsg) exit @@ -1029,21 +1029,21 @@ subroutine test_host(retval, test_suites) ! Check indices call check_constituent_indices(test_scalar_const_index, test_const_indices, & - errmsg, errflg) - call check_errflg(subname // " check suite indices", errflg, errmsg, & - errflg_final) + errmsg, errcode) + call check_errcode(subname // " check suite indices", errcode, errmsg, & + errcode_final) ! Loop over time steps do time_step = 1, num_time_steps ! Initialize the timestep do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_physics_timestep_init( & suite_name=test_suites(sind)%suite_name, & group_name='all', col_start=1, col_end=ncols, & thread_num=1, nthreads=1, nphys_threads=1, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & trim(errmsg) end if @@ -1051,21 +1051,21 @@ subroutine test_host(retval, test_suites) end do do col_start = 1, ncols, 5 - if (errflg /= 0) then + if (errcode /= 0) then continue end if col_end = min(col_start + 4, ncols) do sind = 1, num_suites do index = 1, size(test_suites(sind)%suite_parts) - if (errflg == 0) then + if (errcode == 0) then call ccpp_physics_run( & suite_name=test_suites(sind)%suite_name, & group_name=test_suites(sind)%suite_parts(index), & col_start=col_start, col_end=col_end, & thread_num=1, nthreads=1, nphys_threads=1, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(5a)') trim(test_suites(sind)%suite_name), & '/', trim(test_suites(sind)%suite_parts(index)),& ': ', trim(errmsg) @@ -1077,19 +1077,19 @@ subroutine test_host(retval, test_suites) end do ! Check indices call check_constituent_indices(test_scalar_const_index, test_const_indices, & - errmsg, errflg) - call check_errflg(subname // " check suite indices", errflg, errmsg, & - errflg_final) + errmsg, errcode) + call check_errcode(subname // " check suite indices", errcode, errmsg, & + errcode_final) do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_physics_timestep_final( & suite_name=test_suites(sind)%suite_name, & group_name='all', col_start=1, col_end=ncols, & thread_num=1, nthreads=1, nphys_threads=1, & - errmsg=errmsg, errflg=errflg) + errmsg=errmsg, errcode=errcode) end if - if (errflg /= 0) then + if (errcode /= 0) then write(6, '(3a)') trim(test_suites(sind)%suite_name), ': ', & trim(errmsg) exit @@ -1097,19 +1097,19 @@ subroutine test_host(retval, test_suites) end do ! Run "dycore" - if (errflg == 0) then + if (errcode == 0) then call advect_constituents() end if end do ! End time step loop do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_physics_final( & suite_name=test_suites(sind)%suite_name, & group_name='all', col_start=1, col_end=ncols, & thread_num=1, nthreads=1, nphys_threads=1, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & trim(errmsg) write(6, '(2a)') 'An error occurred in ccpp_physics_final, ', & @@ -1120,10 +1120,10 @@ subroutine test_host(retval, test_suites) end do do sind = 1, num_suites - if (errflg == 0) then + if (errcode == 0) then call ccpp_final(suite_name=test_suites(sind)%suite_name, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(6, '(3a)') test_suites(sind)%suite_parts(index), ': ', & trim(errmsg) write(6, '(2a)') 'An error occurred in ccpp_final, ', & @@ -1136,24 +1136,24 @@ subroutine test_host(retval, test_suites) call ccpp_deallocate_dynamic_constituents() deallocate(host_constituents) - if (errflg == 0) then + if (errcode == 0) then ! Run finished without error, check answers if (compare_data(num_advected)) then write(6, *) 'Answers are correct!' - errflg = 0 + errcode = 0 else write(6, *) 'Answers are not correct!' - errflg = -1 + errcode = -1 end if end if - ! Make sure "final" flag is non-zero if "errflg" is: - if (errflg /= 0) then - errflg_final = -1 ! Notify test script that a failure occured + ! Make sure "final" flag is non-zero if "errcode" is: + if (errcode /= 0) then + errcode_final = -1 ! Notify test script that a failure occured end if ! Set return value to False if any errors were found: - retval = errflg_final == 0 + retval = errcode_final == 0 end subroutine test_host diff --git a/end-to-end-tests/advection_auto_clone/test_host.meta b/end-to-end-tests/advection_auto_clone/test_host.meta index ab33172f..e69dafd9 100644 --- a/end-to-end-tests/advection_auto_clone/test_host.meta +++ b/end-to-end-tests/advection_auto_clone/test_host.meta @@ -30,7 +30,7 @@ dimensions = () type = character kind = len=512 -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/advection_auto_clone/test_host_data.F90 b/end-to-end-tests/advection_auto_clone/test_host_data.F90 index f360ad79..4bcb753b 100644 --- a/end-to-end-tests/advection_auto_clone/test_host_data.F90 +++ b/end-to-end-tests/advection_auto_clone/test_host_data.F90 @@ -29,26 +29,26 @@ module test_host_data contains - subroutine check_constituent_indices(test_index, test_indices, errmsg, errflg) + subroutine check_constituent_indices(test_index, test_indices, errmsg, errcode) ! Check constituent indices against what was found by suite ! indices are passed in rather than looked up to avoid a dependency loop ! Dummy arguments integer, intent(in) :: test_index ! scalar const index from host integer, intent(in) :: test_indices(:) ! array_test_indices from host character(len=*), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode ! Local variable integer :: indx integer :: emstrt - errflg = 0 + errcode = 0 errmsg = '' if (test_index /= const_index) then emstrt = len_trim(errmsg) + 1 write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_index_check for ', & const_std_name, test_index, ' /= ', const_index - errflg = errflg + 1 + errcode = errcode + 1 end if do indx = 1, num_consts if (test_indices(indx) /= const_inds(indx)) then @@ -59,7 +59,7 @@ subroutine check_constituent_indices(test_index, test_indices, errmsg, errflg) end if write(errmsg(emstrt:), '(2a,i0,a,i0)') 'const_indices_check for ', & std_name_array(indx), test_indices(indx), ' /= ', const_inds(indx) - errflg = errflg + 1 + errcode = errcode + 1 end if end do diff --git a/end-to-end-tests/instances_advection/cld_liq.F90 b/end-to-end-tests/instances_advection/cld_liq.F90 index 25b05b9f..3200505f 100644 --- a/end-to-end-tests/instances_advection/cld_liq.F90 +++ b/end-to-end-tests/instances_advection/cld_liq.F90 @@ -18,15 +18,15 @@ module cld_liq !> \section arg_table_cld_liq_register Argument Table !! \htmlinclude arg_table_cld_liq_register.html !! - subroutine cld_liq_register(dyn_const, errmsg, errflg) + subroutine cld_liq_register(dyn_const, errmsg, errcode) type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 - allocate(dyn_const(1), stat=errflg) - if (errflg /= 0) then + errcode = 0 + allocate(dyn_const(1), stat=errcode) + if (errcode /= 0) then errmsg = 'Error allocating dyn_const in cld_liq_register' return end if @@ -34,7 +34,7 @@ subroutine cld_liq_register(dyn_const, errmsg, errflg) diag_name='CLDLIQ', units='kg kg-1', default_value=0._kind_phys, & vertical_dim='vertical_layer_dimension', advected=.true., & mixing_ratio_type='dry', & - errcode=errflg, errmsg=errmsg) + errcode=errcode, errmsg=errmsg) end subroutine cld_liq_register @@ -42,7 +42,7 @@ end subroutine cld_liq_register !! \htmlinclude arg_table_cld_liq_run.html !! subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & - cld_liq_array, cld_liq_tend, errmsg, errflg) + cld_liq_array, cld_liq_tend, errmsg, errcode) integer, intent(in) :: ncol real(kind=kind_phys), intent(in) :: timestep @@ -53,7 +53,7 @@ subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & real(kind=kind_phys), intent(inout) :: cld_liq_array(:, :) real(kind=kind_phys), intent(out) :: cld_liq_tend(:, :) character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode !---------------------------------------------------------------- integer :: icol @@ -61,7 +61,7 @@ subroutine cld_liq_run(ncol, timestep, tcld, temp, qv, ps, & real(kind=kind_phys) :: cond errmsg = '' - errflg = 0 + errcode = 0 do icol = 1, ncol do ilev = 1, size(temp, 2) @@ -83,15 +83,15 @@ end subroutine cld_liq_run !> \section arg_table_cld_liq_init Argument Table !! \htmlinclude arg_table_cld_liq_init.html !! - subroutine cld_liq_init(tfreeze, tcld, errmsg, errflg) + subroutine cld_liq_init(tfreeze, tcld, errmsg, errcode) real(kind=kind_phys), intent(in) :: tfreeze real(kind=kind_phys), intent(out) :: tcld character(len=512), intent(out) :: errmsg - integer, intent(out) :: errflg + integer, intent(out) :: errcode errmsg = '' - errflg = 0 + errcode = 0 tcld = tfreeze - 20.0_kind_phys end subroutine cld_liq_init diff --git a/end-to-end-tests/instances_advection/cld_liq.meta b/end-to-end-tests/instances_advection/cld_liq.meta index db43c5ef..1abc40d0 100644 --- a/end-to-end-tests/instances_advection/cld_liq.meta +++ b/end-to-end-tests/instances_advection/cld_liq.meta @@ -19,7 +19,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -94,7 +94,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 @@ -126,7 +126,7 @@ type = character kind = len=512 intent = out -[ errflg ] +[ errcode ] standard_name = ccpp_error_code long_name = Error flag for error handling in CCPP units = 1 diff --git a/end-to-end-tests/instances_advection/main.F90 b/end-to-end-tests/instances_advection/main.F90 index f84f44ae..2a1e1c84 100644 --- a/end-to-end-tests/instances_advection/main.F90 +++ b/end-to-end-tests/instances_advection/main.F90 @@ -26,7 +26,7 @@ program test_instances_advection character(len=*), parameter :: ccpp_suite = 'cld_suite' character(len=512) :: errmsg - integer :: errflg + integer :: errcode integer :: nphys_threads integer :: ins integer :: tstep @@ -49,8 +49,8 @@ program test_instances_advection do ins = 1, ninstances call ccpp_register(suite_name=ccpp_suite, & instance=ins, ninstances=ninstances, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(error_unit, '(2a)') 'ccpp_register failed: ', trim(errmsg) stop 1 end if @@ -71,15 +71,15 @@ program test_instances_advection diag_name='QV', units='kg kg-1', & vertical_dim='vertical_layer_dimension', advected=.true., & default_value=0.0_kind_phys, mixing_ratio_type='wet', & - errcode=errflg, errmsg=errmsg) - if (errflg /= 0) then + errcode=errcode, errmsg=errmsg) + if (errcode /= 0) then write(error_unit, '(2a)') 'instantiate failed: ', trim(errmsg) stop 1 end if call ccpp_register_constituents(host_constituents=host_consts, & instance=ins, ninstances=ninstances, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(error_unit, '(a,i0,2a)') & 'ccpp_register_constituents failed for instance ', ins, & ': ', trim(errmsg) @@ -93,8 +93,8 @@ program test_instances_advection !----------------------------------------------------------------- do ins = 1, ninstances call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, & - instance=ins, errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + instance=ins, errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(error_unit, '(a,i0,2a)') & 'ccpp_initialize_constituents failed for instance ', ins, & ': ', trim(errmsg) @@ -108,8 +108,8 @@ program test_instances_advection !----------------------------------------------------------------- call ccpp_const_get_index(stdname='water_vapor_specific_humidity', & - const_index=idx, instance=1, errflg=errflg, errmsg=errmsg) - if (errflg /= 0) then + const_index=idx, instance=1, errcode=errcode, errmsg=errmsg) + if (errcode /= 0) then write(error_unit, '(2a)') 'ccpp_const_get_index(qv) failed: ', & trim(errmsg) stop 1 @@ -117,8 +117,8 @@ program test_instances_advection call set_index_qv(idx) call ccpp_number_constituents(num_flds=num_consts, instance=1, & - errflg=errflg, errmsg=errmsg) - if (errflg /= 0) then + errcode=errcode, errmsg=errmsg) + if (errcode /= 0) then write(error_unit, '(2a)') 'ccpp_number_constituents failed: ', & trim(errmsg) stop 1 @@ -141,8 +141,8 @@ program test_instances_advection do ins = 1, ninstances call ccpp_init(suite_name=ccpp_suite, & instance=ins, ninstances=ninstances, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(error_unit, '(a,i0,2a)') 'ccpp_init failed for instance ', & ins, ': ', trim(errmsg) stop 1 @@ -153,8 +153,8 @@ program test_instances_advection lb=1, ub=ncols, thread_num=1, nthreads=1, & nphys_threads=nphys_threads, & instance=ins, ninstances=ninstances, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(error_unit, '(a,i0,2a)') & 'ccpp_physics_init failed for instance ', ins, ': ', trim(errmsg) stop 1 @@ -170,8 +170,8 @@ program test_instances_advection group_name='physics', lb=1, ub=ncols, & thread_num=1, nthreads=1, nphys_threads=nphys_threads, & instance=ins, ninstances=ninstances, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(error_unit, '(a,i0,2a)') & 'ccpp_physics_timestep_init failed for instance ', ins, & ': ', trim(errmsg) @@ -183,8 +183,8 @@ program test_instances_advection lb=1, ub=ncols, thread_num=1, nthreads=1, & nphys_threads=nphys_threads, & instance=ins, ninstances=ninstances, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(error_unit, '(a,i0,2a)') & 'ccpp_physics_run failed for instance ', ins, ': ', trim(errmsg) stop 1 @@ -195,8 +195,8 @@ program test_instances_advection group_name='physics', lb=1, ub=ncols, & thread_num=1, nthreads=1, nphys_threads=nphys_threads, & instance=ins, ninstances=ninstances, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(error_unit, '(a,i0,2a)') & 'ccpp_physics_timestep_final failed for instance ', ins, & ': ', trim(errmsg) @@ -213,8 +213,8 @@ program test_instances_advection lb=1, ub=ncols, thread_num=1, nthreads=1, & nphys_threads=nphys_threads, & instance=ins, ninstances=ninstances, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(error_unit, '(a,i0,2a)') & 'ccpp_physics_final failed for instance ', ins, ': ', trim(errmsg) stop 1 @@ -238,8 +238,8 @@ program test_instances_advection do ins = 1, ninstances call ccpp_final(suite_name=ccpp_suite, & instance=ins, ninstances=ninstances, & - errmsg=errmsg, errflg=errflg) - if (errflg /= 0) then + errmsg=errmsg, errcode=errcode) + if (errcode /= 0) then write(error_unit, '(a,i0,2a)') 'ccpp_final failed for instance ', & ins, ': ', trim(errmsg) stop 1 diff --git a/end-to-end-tests/instances_advection/main.meta b/end-to-end-tests/instances_advection/main.meta index 9623129f..5857002a 100644 --- a/end-to-end-tests/instances_advection/main.meta +++ b/end-to-end-tests/instances_advection/main.meta @@ -57,7 +57,7 @@ dimensions = () type = character kind = len=512 -[errflg] +[errcode] standard_name = ccpp_error_code long_name = error flag for CCPP error handling units = 1 diff --git a/unit-tests/sample_files/control_unit_conv.meta b/unit-tests/sample_files/control_unit_conv.meta index 75edc085..1d73d541 100644 --- a/unit-tests/sample_files/control_unit_conv.meta +++ b/unit-tests/sample_files/control_unit_conv.meta @@ -15,6 +15,13 @@ dimensions = () type = character kind = len=256 +[ grp_name ] + standard_name = group_name + long_name = name of the CCPP group + units = none + dimensions = () + type = character + kind = len=256 [ lb ] standard_name = horizontal_loop_begin long_name = start of horizontal range for this phase diff --git a/unit-tests/test_control_validation.py b/unit-tests/test_control_validation.py index ebc6104b..2c4455d4 100644 --- a/unit-tests/test_control_validation.py +++ b/unit-tests/test_control_validation.py @@ -81,8 +81,9 @@ def test_raises_ccpp_error(self): _validate_required_control_vars('test_host', self._host_dict) def test_all_missing_vars_reported(self): - """All 7 missing required standard names appear in the error message.""" + """All 8 missing required standard names appear in the error message.""" missing = [ + 'group_name', 'horizontal_loop_begin', 'horizontal_loop_end', 'thread_number', 'number_of_threads', 'number_of_physics_threads', 'ccpp_error_code', 'ccpp_error_message', diff --git a/unit-tests/test_host_constituents.py b/unit-tests/test_host_constituents.py index c2ad807f..92f073c7 100644 --- a/unit-tests/test_host_constituents.py +++ b/unit-tests/test_host_constituents.py @@ -247,7 +247,7 @@ def test_takes_host_constituents_and_instance(self): # when the host declares the multi-instance pair. self.assertIn( 'subroutine ccpp_register_constituents(host_constituents, ' - 'inst_num, ninstances, errflg, errmsg)', + 'inst_num, ninstances, errcode, errmsg)', self.text, ) self.assertIn( @@ -300,7 +300,7 @@ def test_iterates_host_then_suite_per_instance(self): def test_lock_table_called_per_instance(self): self.assertIn( 'call ccpp_model_constituents_obj(inst_num)%lock_table(' - 'errcode=errflg, errmsg=errmsg)', + 'errcode=errcode, errmsg=errmsg)', self.text, ) @@ -314,14 +314,14 @@ def setUp(self): def test_takes_dimensions_and_instance(self): self.assertIn( 'subroutine ccpp_initialize_constituents(ncols, num_layers, ' - 'inst_num, errflg, errmsg)', + 'inst_num, errcode, errmsg)', self.text, ) def test_calls_lock_data_per_instance(self): self.assertIn( 'call ccpp_model_constituents_obj(inst_num)%lock_data(' - 'ncols, num_layers, errcode=errflg, errmsg=errmsg)', + 'ncols, num_layers, errcode=errcode, errmsg=errmsg)', self.text, ) @@ -368,7 +368,7 @@ def test_queries_index_of_X_per_instance(self): self.assertIn( "call ccpp_model_constituents_obj(inst_num)%const_index(" "index_of_cloud_liquid_water_mixing_ratio, " - "'cloud_liquid_water_mixing_ratio', errcode=errflg, errmsg=errmsg)", + "'cloud_liquid_water_mixing_ratio', errcode=errcode, errmsg=errmsg)", self.text, ) @@ -393,7 +393,7 @@ def test_post_const_index_validation_emitted(self): 'if (index_of_cloud_liquid_water_mixing_ratio == int_unassigned) then', body, ) - self.assertIn('errflg = 1', body) + self.assertIn('errcode = 1', body) self.assertIn( "errmsg = 'ccpp_initialize_constituents: constituent " "''cloud_liquid_water_mixing_ratio'' is referenced by a " @@ -411,7 +411,7 @@ def setUp(self): def test_subroutine_signature(self): self.assertIn( 'subroutine ccpp_is_scheme_constituent(var_name, ' - 'constituent_exists, errflg, errmsg)', + 'constituent_exists, errcode, errmsg)', self.text, ) @@ -456,26 +456,26 @@ def setUp(self): def test_number_constituents(self): self.assertIn( 'subroutine ccpp_number_constituents(num_flds, advected, ' - 'inst_num, errflg, errmsg)', + 'inst_num, errcode, errmsg)', self.text, ) self.assertIn( 'call ccpp_model_constituents_obj(inst_num)%num_constituents(' - 'num_flds, advected=advected, errcode=errflg, errmsg=errmsg)', + 'num_flds, advected=advected, errcode=errcode, errmsg=errmsg)', self.text, ) def test_gather_constituents(self): self.assertIn( 'call ccpp_model_constituents_obj(inst_num)%copy_in(' - 'const_array, errcode=errflg, errmsg=errmsg)', + 'const_array, errcode=errcode, errmsg=errmsg)', self.text, ) def test_update_constituents(self): self.assertIn( 'call ccpp_model_constituents_obj(inst_num)%copy_out(' - 'const_array, errcode=errflg, errmsg=errmsg)', + 'const_array, errcode=errcode, errmsg=errmsg)', self.text, ) @@ -485,7 +485,7 @@ def test_const_get_index(self): self.assertIn( 'call ccpp_model_constituents_obj(inst_num)%const_index(' 'standard_name=stdname, index=const_index, ' - 'errcode=errflg, errmsg=errmsg)', + 'errcode=errcode, errmsg=errmsg)', self.text, ) From cd82d1e9568de5cad5ea7e73a23775c436fdb150 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 4 Jun 2026 06:43:23 -0600 Subject: [PATCH 51/74] Bug fix for mangled constituent indices --- capgen-ng/generator/host_constituents.py | 15 ++- capgen-ng/generator/suite_resolver.py | 37 ++++-- unit-tests/test_host_constituents.py | 149 ++++++++++++++++++++++- 3 files changed, 190 insertions(+), 11 deletions(-) diff --git a/capgen-ng/generator/host_constituents.py b/capgen-ng/generator/host_constituents.py index f003fe27..e280a8e2 100644 --- a/capgen-ng/generator/host_constituents.py +++ b/capgen-ng/generator/host_constituents.py @@ -546,8 +546,12 @@ def _deallocate_lines( # They are deallocated in ``_final``'s last-to-leave block # instead. lines.append('{}deallocate({})'.format(i3, _CONST_OBJ)) + # Reset to the unbound sentinel (matches the declaration default) so a + # re-register / re-init cycle starts from int_unassigned and the guard + # stays effective on the second pass. for std_name in index_names: - lines.append('{}{} = 0'.format(i3, _index_symbol_name(std_name))) + lines.append('{}{} = int_unassigned'.format( + i3, _index_symbol_name(std_name))) lines.append('{}end if'.format(i2)) lines.append('') lines.append('{}end subroutine ccpp_deallocate_dynamic_constituents'.format(i1)) @@ -652,8 +656,15 @@ def _generate_host_constituents( ) if register_suites: lines.append('') + # Default to int_unassigned (NOT 0) so a constituent that is referenced + # by a scheme but never bound by ccpp_initialize_constituents (missing + # registration, or a host that never calls the init routine) is caught + # by the post-const_index guard below instead of silently surviving as + # an index of 0 and producing an out-of-bounds vars_layer subscript at + # run time. This also keeps the guard correct for any framework whose + # %const_index leaves the output unchanged on a name miss. for std_name in index_names: - lines.append('{}integer :: {} = 0'.format( + lines.append('{}integer :: {} = int_unassigned'.format( _INDENT, _index_symbol_name(std_name))) if index_names: max_len = max(len(n) for n in index_names) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 58e8769d..9f2987e5 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -1073,6 +1073,15 @@ class ResolvedArg: # :mod:`generator.host_cap`. Replaces the older trick of stuffing # them into :attr:`used_dim_std_names`. used_const_dim_std_names: Set[str] = field(default_factory=set) + # Real (un-mangled) constituent base standard names that this arg's + # subscript indexes via ``index_of_``. The Fortran symbol in + # :attr:`constituent_extra_symbols` is mangled to fit the 63-char + # identifier limit (see :func:`_index_symbol_name`), so its suffix is + # NOT a reliable source of the real standard name. This set preserves + # the real names verbatim so ``ccpp_initialize_constituents`` can pass + # them to ``%const_index`` as the lookup key (and list them in + # ``ccpp_model_const_stdnames``). + constituent_index_std_names: Set[str] = field(default_factory=set) @property def needs_transform(self) -> bool: @@ -1291,9 +1300,11 @@ class SuiteResolution: arrays. Empty when no register-phase scheme produces constituents. constituent_index_names : list of str Sorted list of base-constituent standard names that need an - ``index_of_`` integer emitted in the suite cap. Collected by - scanning every ``source='constituent'`` ResolvedArg's - ``constituent_extra_symbols`` for ``index_of_*`` tokens. Used by + ``index_of_`` integer emitted in the suite cap. These are the + REAL (un-mangled) standard names, collected from every + ``source='constituent'`` ResolvedArg's + ``constituent_index_std_names`` set -- NOT recovered from the + (possibly mangled) ``index_of_*`` Fortran symbols. Used by :mod:`generator.suite_cap` to emit the index declarations and the ``ccpp_model_constituents_object%const_index`` population calls in ``_init``. @@ -2038,7 +2049,8 @@ def _resolve_constituent_arg( def _common_kwargs(base_expr, subscript, call_expr, used_host_std, extra_symbols, - used_const_dim_std=None): + used_const_dim_std=None, + index_std_names=None): used_host_std = set(used_host_std) if inst_local: used_host_std.add(_INSTANCE_NUM_STD) @@ -2071,6 +2083,8 @@ def _common_kwargs(base_expr, subscript, call_expr, constituent_extra_symbols=extra_symbols, used_const_dim_std_names=(set(used_const_dim_std) if used_const_dim_std else set()), + constituent_index_std_names=(set(index_std_names) + if index_std_names else set()), ) # ---- Path 1a: index_of_ — module-level integer, no per-instance -- @@ -2079,10 +2093,12 @@ def _common_kwargs(base_expr, subscript, call_expr, # identity for short names, so existing fixtures are unaffected. # ``_INDEX_PREFIX`` is already part of std_name -- strip then # re-add via the helper for uniform truncation. - index_sym = _index_symbol_name(std_name[len(_INDEX_PREFIX):]) + index_base = std_name[len(_INDEX_PREFIX):] + index_sym = _index_symbol_name(index_base) return ResolvedArg(**_common_kwargs( base_expr=index_sym, subscript='', call_expr=index_sym, used_host_std=set(), extra_symbols={index_sym}, + index_std_names={index_base}, )) # ---- Path 1b: framework-named std_name → DDT member ----------------- @@ -2148,6 +2164,7 @@ def _common_kwargs(base_expr, subscript, call_expr, base_expr=base_expr, subscript=subscript, call_expr=call_expr, used_host_std=used_host_std, extra_symbols={index_sym, _CONST_OBJ_VAR}, + index_std_names={base_std}, )) @@ -2299,9 +2316,13 @@ def resolve_suite( if arg.source != 'constituent' or arg.is_constituent_arg: continue uses_constituents = True - for sym in arg.constituent_extra_symbols: - if sym.startswith(_INDEX_PREFIX): - index_names.add(sym[len(_INDEX_PREFIX):]) + # Collect the REAL (un-mangled) base standard names, not + # the mangled symbol suffix. ``ccpp_initialize_constituents`` + # passes these verbatim to ``%const_index``; a mangled key + # would never match a registered constituent and leave the + # ``index_of_`` integer at its 0 default -> out-of-bounds + # subscript at run time. + index_names.update(arg.constituent_index_std_names) constituent_index_names = sorted(index_names) # Under option A the constituent object is generator-owned (lives in diff --git a/unit-tests/test_host_constituents.py b/unit-tests/test_host_constituents.py index 92f073c7..25d2513b 100644 --- a/unit-tests/test_host_constituents.py +++ b/unit-tests/test_host_constituents.py @@ -198,7 +198,7 @@ def test_no_module_level_pointers(self): def test_index_of_X_declared(self): self.assertIn( - 'integer :: index_of_cloud_liquid_water_mixing_ratio = 0', + 'integer :: index_of_cloud_liquid_water_mixing_ratio = int_unassigned', self.consumer_text, ) self.assertIn( @@ -402,6 +402,153 @@ def test_post_const_index_validation_emitted(self): ) +def _render_long_name_constituent(): + """Resolve a single-suite scheme that consumes ONE advected base + constituent whose standard name is long enough to force + ``_index_symbol_name`` to mangle the Fortran symbol, then render the + host-constituents module. Returns ``(text, long_std_name, symbol)``. + + Regression guard for the lossy round-trip bug: the mangled + ``index_of_`` Fortran symbol must NOT leak into the ``%const_index`` + lookup string (the framework keys on the real standard name) nor into + ``ccpp_model_const_stdnames``. + """ + import tempfile + import logging + from metadata.metadata_table import parse_metadata_file + from metadata.variable_resolver import build_flat_host_dict, SchemeStore + from generator.suite_xml import parse_suite_xml + + # 57 chars -> index_of_ is 66 > 63 -> mangled (same family as the + # cam-sima kessler ``*_wrt_moist_air_and_condensed_water`` constituents). + long_std = 'water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water' + from generator.suite_resolver import _index_symbol_name + symbol = _index_symbol_name(long_std) + assert symbol != 'index_of_' + long_std, 'fixture name must mangle' + + scheme_meta = ''' +[ccpp-table-properties] + name = long_const_scheme + type = scheme +[ccpp-arg-table] + name = long_const_scheme_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in +[ nz ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer + intent = in +[ qv ] + standard_name = %s + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in + advected = .true. +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' % long_std + suite_xml = ( + '\n' + '\n' + ' \n' + ' long_const_scheme\n' + ' \n' + '\n' + ) + + here = os.path.dirname(os.path.abspath(__file__)) + samples = os.path.join(here, 'sample_files') + host_tbls = parse_metadata_file(os.path.join(samples, 'host_with_constituents.meta')) + ctrl_tbls = parse_metadata_file(os.path.join(samples, 'control_full.meta')) + fw_meta = os.path.join( + os.path.dirname(here), 'capgen-ng', 'src', 'ccpp_constituent_prop_mod.meta', + ) + ddt_tbls = parse_metadata_file(fw_meta) if os.path.isfile(fw_meta) else [] + hd = build_flat_host_dict(host_tbls, ctrl_tbls, ddt_tbls) + + with tempfile.TemporaryDirectory() as tmp: + sm = os.path.join(tmp, 'long_const_scheme.meta') + with open(sm, 'w') as fh: + fh.write(scheme_meta) + sx = os.path.join(tmp, 'suite_longc.xml') + with open(sx, 'w') as fh: + fh.write(suite_xml) + store = SchemeStore.build_from(parse_metadata_file(sm)) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + sr = resolve_suite(suite, store, hd) + text = '\n'.join(_generate_host_constituents([sr], host_dict=hd)) + return text, long_std, symbol + + +class TestLongConstituentNameRoundTrip(unittest.TestCase): + """Regression: a constituent std name long enough to mangle the Fortran + ``index_of_`` symbol must still key ``%const_index`` (and the + ``ccpp_model_const_stdnames`` array) on the REAL standard name. The old + code recovered the name from the mangled symbol suffix, so the lookup + string was corrupted, ``const_index`` never matched, and the index + integer stayed at its ``0`` default -> out-of-bounds constituent + subscript at run time (cam-sima kessler segfault).""" + + @classmethod + def setUpClass(cls): + cls.text, cls.long_std, cls.symbol = _render_long_name_constituent() + + def test_const_index_keys_on_real_std_name(self): + # Mangled symbol as the integer; REAL std name as the lookup string. + self.assertIn( + "%const_index({}, '{}', errcode=errcode, errmsg=errmsg)".format( + self.symbol, self.long_std, + ), + self.text, + ) + + def test_mangled_suffix_not_used_as_lookup_string(self): + # The mangled suffix must never appear as a quoted lookup key. + mangled_suffix = self.symbol[len('index_of_'):] + self.assertNotIn("'{}'".format(mangled_suffix), self.text) + + def test_stdnames_array_lists_real_name(self): + self.assertIn("'{}'".format(self.long_std), self.text) + + def test_subscript_symbol_declared_and_public(self): + # The (mangled) symbol must still be declared, public, and reset. + self.assertIn( + 'integer :: {} = int_unassigned'.format(self.symbol), self.text) + self.assertIn('public :: {}'.format(self.symbol), self.text) + + def test_index_defaults_and_resets_to_unassigned(self): + # Declaration default is the unbound sentinel (so a never-bound index + # trips the init guard instead of becoming a 0 subscript)... + self.assertIn( + 'integer :: {} = int_unassigned'.format(self.symbol), self.text) + # ...and the deallocate routine resets it to the same sentinel. + dealloc = self.text.split( + 'subroutine ccpp_deallocate_dynamic_constituents' + )[1].split('end subroutine ccpp_deallocate_dynamic_constituents')[0] + self.assertIn('{} = int_unassigned'.format(self.symbol), dealloc) + self.assertNotIn('{} = 0'.format(self.symbol), dealloc) + + class TestIsSchemeConstituent(unittest.TestCase): """``ccpp_is_scheme_constituent`` + the module-scope std-name parameter array.""" From bc6a08746c98e6df48ccb75b2b32f95d4f7f1e78 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 4 Jun 2026 09:22:42 -0600 Subject: [PATCH 52/74] Bug fix for auto-clone constituents that are also defined elsewhere as actual variables --- capgen-ng/generator/suite_resolver.py | 25 +++++ unit-tests/test_suite_resolver.py | 154 ++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 9f2987e5..426c689e 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -2124,6 +2124,31 @@ def _common_kwargs(base_expr, subscript, call_expr, if not scheme_var.is_constituent: return None # not constituent-related + # ---- Provider gate (mirrors original-capgen ConstituentVarDict.find_variable) + # A constituent-FLAGGED consumer whose standard name is actually + # PROVIDED elsewhere is an ordinary interstitial, not a constituent: + # + # * the host declares it in metadata -> host scope, or + # * an earlier scheme produced it intent=out -> already recorded + # in ``suite_vars`` (the resolver walks calls in execution order, + # so a producer that runs before this consumer is visible here). + # + # Original capgen only auto-creates a constituent when its + # ``find_variable`` returns None (nothing in host or suite scope + # provides the name). Without this gate a dry mixing ratio that is + # PRODUCED by e.g. ``wet_to_dry_water_vapor`` (intent=out, unflagged) + # but CONSUMED by ``kessler`` (advected=True) gets split into a suite + # array (producer) and a constituent column (consumer) AND spuriously + # auto-registered as a constituent. Defer to normal host/suite + # dispatch so producer and consumer share one storage location and the + # name is never registered as a constituent. Tendencies (intent=out, + # ``tendency_of_*``) are genuine constituent-tendency OUTPUTS and are + # never gated. + if not is_tendency_name and ( + std_name in suite_vars or std_name in host_dict + ): + return None + # ---- Paths 2/3: is_constituent base or tendency --------------------- if intent == 'out': if not is_tendency_name: diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 3d675f5b..d1f67d3f 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -3554,6 +3554,160 @@ def test_scheme_consts_temp_declared(self): ) +######################################################################## +# Provider gate: a constituent-flagged consumer that is actually +# provided by an earlier scheme's intent=out output is an interstitial, +# not a constituent (mirrors original-capgen find_variable). +######################################################################## + +_PROVIDER_GATE_SCHEMES = ''' +[ccpp-table-properties] + name = make_dry + type = scheme +[ccpp-arg-table] + name = make_dry_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in +[ nz ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer + intent = in +[ qv_dry ] + standard_name = water_vapor_mixing_ratio_wrt_dry_air + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-table-properties] + name = use_dry + type = scheme +[ccpp-arg-table] + name = use_dry_run + type = scheme +[ ncol ] + standard_name = horizontal_dimension + units = count + dimensions = () + type = integer + intent = in +[ nz ] + standard_name = vertical_layer_dimension + units = count + dimensions = () + type = integer + intent = in +[ qv ] + standard_name = water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water + advected = .true. + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = in +[ qv_dry ] + standard_name = water_vapor_mixing_ratio_wrt_dry_air + advected = .true. + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real | kind = kind_phys + intent = inout +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + +_PROVIDER_GATE_SUITE = ( + '\n' + '\n' + ' \n' + ' make_dry\n' + ' use_dry\n' + ' \n' + '\n' +) + + +class TestConstituentProviderGate(unittest.TestCase): + """A constituent-FLAGGED consumer arg whose standard name is produced by + an earlier scheme (intent=out, unflagged) must resolve as an ordinary + suite variable -- shared with the producer -- NOT as a constituent + column, and must not be registered as a constituent. A constituent + with no provider stays a constituent. Regression for the cam-sima + kessler dry/wet mixing-ratio over-registration.""" + + @classmethod + def setUpClass(cls): + import logging + from generator.suite_xml import parse_suite_xml + hd = _load_constituent_host_dict() + store = SchemeStore.build_from(_parse(_PROVIDER_GATE_SCHEMES)) + with tempfile.TemporaryDirectory() as tmp: + sx = os.path.join(tmp, 'suite_provgate.xml') + with open(sx, 'w') as fh: + fh.write(_PROVIDER_GATE_SUITE) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + cls.sr = resolve_suite(suite, store, hd) + run_calls = list(iter_phase_calls(cls.sr.groups[0].phase_calls['run'])) + cls.producer = {a.scheme_local_name: a for a in run_calls[0].args} + cls.consumer = {a.scheme_local_name: a for a in run_calls[1].args} + + def test_producer_output_is_suite_var(self): + self.assertEqual(self.producer['qv_dry'].source, 'suite') + + def test_provided_consumer_resolves_as_suite_not_constituent(self): + # qv_dry is flagged advected on the consumer, but make_dry provides + # it -> must be the shared suite var, not a constituent column. + qv_dry = self.consumer['qv_dry'] + self.assertEqual(qv_dry.source, 'suite') + self.assertNotIn('vars_layer', qv_dry.call_expr) + + def test_unprovided_constituent_stays_constituent(self): + qv = self.consumer['qv'] + self.assertEqual(qv.source, 'constituent') + self.assertIn('vars_layer', qv.call_expr) + + def test_index_names_exclude_provided_dry_var(self): + self.assertEqual( + self.sr.constituent_index_names, + ['water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water'], + ) + self.assertNotIn( + 'water_vapor_mixing_ratio_wrt_dry_air', + self.sr.constituent_index_names, + ) + + ######################################################################## # Constituent auto-resolution (cam-sima-style consumer schemes) ######################################################################## From d9877eb8bb36f642756ae3c5ef237b323cb7747a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 4 Jun 2026 13:02:18 -0600 Subject: [PATCH 53/74] Fix hopefully last bug arising from special handling of index_of_* variables --- capgen-ng/generator/suite_resolver.py | 76 ++++++-- unit-tests/test_suite_resolver.py | 239 ++++++++++++++++++++++++++ 2 files changed, 299 insertions(+), 16 deletions(-) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 426c689e..55e355b2 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -2033,13 +2033,35 @@ def _resolve_constituent_arg( is_index_name = std_name.startswith(_INDEX_PREFIX) is_framework_name = std_name in _FRAMEWORK_CONST_STDS or is_index_name - # Host metadata wins: if the host declares this std_name as a regular - # variable (e.g. a `protected integer` named `ntcw` with - # standard_name = index_of_..._tracer_concentration_array), defer to - # normal host-arg resolution so the scheme call uses the host's short - # local name. Constituent auto-provisioning is reserved for - # framework-named std_names the host has not claimed. - if is_framework_name and host_dict and std_name in host_dict: + # Host/suite provides it -> not a framework auto-provision. Defer to + # normal host/suite resolution when: + # + # * the host declares this std_name as a regular variable (e.g. a + # `protected integer` named `ntcw` with standard_name = + # index_of_..._tracer_concentration_array), OR + # * an earlier-phase scheme already produced it (it is in suite_vars). + # ``suite_vars`` accumulates across phases in chronological order + # (register, init, timestep_init, run, ...), so e.g. a band index + # produced by ``rrtmgp_inputs_setup_init`` (intent=out) is visible + # here when ``rrtmgp_sw_cloud_optics_run`` consumes it (intent=in). + # + # Constituent auto-provisioning is reserved for framework-named + # std_names that nothing else provides. + if is_framework_name and ( + (host_dict and std_name in host_dict) or std_name in suite_vars + ): + return None + + # A scheme that OUTPUTS ``index_of_`` is producing an ordinary index + # variable (e.g. ``rrtmgp_inputs_setup`` computing the diagnostic + # shortwave band index), NOT a constituent index -- constituent indices + # are read-only module integers bound by ``%const_index`` and are never + # written by a scheme. Defer so it becomes a suite var that later-phase + # consumers resolve via the gate above. (Index names produced by some + # OTHER scheme but consumed here have already been caught by the + # suite_vars branch above; this handles the producing arg itself, whose + # name is not yet in suite_vars on first occurrence.) + if is_index_name and intent == 'out': return None constituent_module = _constituent_module_name(suite_name) @@ -2279,12 +2301,30 @@ def resolve_suite( ) suite_vars: Dict[str, SuiteVar] = {} - resolved_groups: List[ResolvedGroup] = [] - - for group in suite.groups: - resolved_group = ResolvedGroup(group_name=group.name) - - for phase in phases: + # Pre-create one ResolvedGroup per SDF group so the phase-major loop below + # can append each phase's calls to the right group. + resolved_groups: List[ResolvedGroup] = [ + ResolvedGroup(group_name=group.name) for group in suite.groups + ] + + # Resolve PHASE-MAJOR, GROUP-MINOR (groups in SDF order within each phase), + # mirroring the runtime execution hierarchy: the host completes EVERY + # group's ``init`` before any group's ``run``, every group's + # ``timestep_init`` before any ``run``, and so on. ``suite_vars`` + # accumulates in that true execution order, so a variable produced in an + # earlier phase by ANY group is visible to a consumer in a later phase of + # ANY group (cross-group cross-phase provision). Within a single phase, + # groups resolve in SDF order, so an earlier group may provide to a later + # one (e.g. ``physics_before_coupler`` -> ``physics_after_coupler``, which + # the host runs sequentially with coupling in between); a same-phase + # consumer that precedes its producer is still correctly rejected. + # + # (The previous nesting was group-major — each group through all its + # phases — which made a variable produced by a later group's ``init`` + # invisible to an earlier group's ``run`` even though, at runtime, all + # inits precede all runs.) + for phase in phases: + for group, resolved_group in zip(suite.groups, resolved_groups): used_local_names_phase: Set[str] = set() if phase == 'run': @@ -2313,9 +2353,13 @@ def resolve_suite( if items_for_phase: resolved_group.phase_calls[phase] = items_for_phase - # Collect dimension variable USE info for this group. - resolved_group.dim_uses = _collect_dim_uses(resolved_group, host_dict, suite_vars=suite_vars) - resolved_groups.append(resolved_group) + # Collect dimension USE info once every phase/group is resolved, so + # ``suite_vars`` is complete (a dimension may reference a suite var + # produced by any group in any phase). + for resolved_group in resolved_groups: + resolved_group.dim_uses = _collect_dim_uses( + resolved_group, host_dict, suite_vars=suite_vars, + ) # Constituent register calls: gather the (scheme_name, scheme_local_name) # pairs for every register-phase arg that was flagged as a constituent. diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index d1f67d3f..0b42baa1 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -3708,6 +3708,245 @@ def test_index_names_exclude_provided_dry_var(self): ) +######################################################################## +# index_of_* disambiguation: a scheme-produced index (cross-phase) is a +# suite var, NOT a constituent index (mirrors rrtmgp band indices). +######################################################################## + +_INDEX_OF_SCHEMES = ''' +[ccpp-table-properties] + name = band_setup + type = scheme +[ccpp-arg-table] + name = band_setup_init + type = scheme +[ idx_sw ] + standard_name = index_of_shortwave_band + units = index + dimensions = () + type = integer + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-table-properties] + name = band_user + type = scheme +[ccpp-arg-table] + name = band_user_run + type = scheme +[ idx_sw ] + standard_name = index_of_shortwave_band + units = index + dimensions = () + type = integer + intent = in +[ idx_const ] + standard_name = index_of_test_constituent + units = index + dimensions = () + type = integer + intent = in +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + +# Consumer scheme deliberately listed BEFORE the producer in the SDF, to +# prove resolution is phase-driven (init before run) and NOT dependent on +# scheme text order -- exactly the rrtmgp inputs_setup / sw_cloud_optics +# arrangement. +_INDEX_OF_SUITE = ( + '\n' + '\n' + ' \n' + ' band_user\n' + ' band_setup\n' + ' \n' + '\n' +) + + +class TestIndexOfSchemeProducedVsConstituent(unittest.TestCase): + """``index_of_`` produced by a scheme (intent=out, here in the init + phase) is an ordinary suite var; a later-phase consumer (run) resolves + it from suite_vars as ``source='suite'``. A genuine ``index_of_`` + that no scheme produces stays a constituent index. Regression for the + cam-sima rrtmgp ``index_of_shortwave_band`` 'missing host variable' + failure.""" + + @classmethod + def setUpClass(cls): + import logging + from generator.suite_xml import parse_suite_xml + hd = _load_constituent_host_dict() + store = SchemeStore.build_from(_parse(_INDEX_OF_SCHEMES)) + with tempfile.TemporaryDirectory() as tmp: + sx = os.path.join(tmp, 'suite_bandtest.xml') + with open(sx, 'w') as fh: + fh.write(_INDEX_OF_SUITE) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + cls.sr = resolve_suite(suite, store, hd) + init_calls = list(iter_phase_calls(cls.sr.groups[0].phase_calls['init'])) + run_calls = list(iter_phase_calls(cls.sr.groups[0].phase_calls['run'])) + cls.producer = {a.scheme_local_name: a for a in init_calls[0].args} + cls.consumer = {a.scheme_local_name: a for a in run_calls[0].args} + + def test_init_producer_is_suite_var(self): + # The init-phase intent=out index is a regular suite var, not a + # constituent index written by a scheme. + idx = self.producer['idx_sw'] + self.assertEqual(idx.source, 'suite') + + def test_run_consumer_resolves_from_suite_vars(self): + # Cross-phase: produced in init, consumed in run -> source='suite'. + idx = self.consumer['idx_sw'] + self.assertEqual(idx.source, 'suite') + self.assertNotEqual(idx.source, 'constituent') + + def test_genuine_constituent_index_unchanged(self): + # index_of_test_constituent is produced by no scheme -> still a + # constituent index. + idx = self.consumer['idx_const'] + self.assertEqual(idx.source, 'constituent') + + def test_band_index_not_registered_as_constituent(self): + self.assertNotIn('shortwave_band', ' '.join(self.sr.constituent_index_names)) + self.assertIn('test_constituent', self.sr.constituent_index_names) + + +######################################################################## +# Cross-group cross-phase provision: a variable produced by a LATER +# group's init phase must be visible to an EARLIER group's run phase, +# because at runtime all groups' init complete before any group's run. +######################################################################## + +_XGROUP_SCHEMES = ''' +[ccpp-table-properties] + name = consumer_a + type = scheme +[ccpp-arg-table] + name = consumer_a_run + type = scheme +[ val ] + standard_name = some_setup_value + units = 1 + dimensions = () + type = real | kind = kind_phys + intent = in +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-table-properties] + name = producer_b + type = scheme +[ccpp-arg-table] + name = producer_b_init + type = scheme +[ val ] + standard_name = some_setup_value + units = 1 + dimensions = () + type = real | kind = kind_phys + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + +# The CONSUMER's group is listed FIRST and the PRODUCER's group SECOND, +# and the producer provides in the init phase while the consumer reads in +# the run phase. At runtime every group's init precedes every group's +# run, so this is valid; the resolver must reflect that (phase-major). +_XGROUP_SUITE = ( + '\n' + '\n' + ' \n' + ' consumer_a\n' + ' \n' + ' \n' + ' producer_b\n' + ' \n' + '\n' +) + + +class TestCrossGroupCrossPhaseProvision(unittest.TestCase): + """A variable produced by a later group's init phase is visible to an + earlier group's run phase (all inits precede all runs at runtime). + Resolution is phase-major, group-minor. Regression for the group-major + nesting that raised 'not provided by any prior scheme'.""" + + @classmethod + def setUpClass(cls): + import logging + from generator.suite_xml import parse_suite_xml + hd = _load_constituent_host_dict() + store = SchemeStore.build_from(_parse(_XGROUP_SCHEMES)) + with tempfile.TemporaryDirectory() as tmp: + sx = os.path.join(tmp, 'suite_xgroup.xml') + with open(sx, 'w') as fh: + fh.write(_XGROUP_SUITE) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + cls.sr = resolve_suite(suite, store, hd) + + def test_consumer_resolves_producer_from_later_group(self): + grpA = self.sr.groups[0] + run_calls = list(iter_phase_calls(grpA.phase_calls['run'])) + val = {a.scheme_local_name: a for a in run_calls[0].args}['val'] + self.assertEqual(val.source, 'suite') + + def test_producer_is_suite_var(self): + grpB = self.sr.groups[1] + init_calls = list(iter_phase_calls(grpB.phase_calls['init'])) + val = {a.scheme_local_name: a for a in init_calls[0].args}['val'] + self.assertEqual(val.source, 'suite') + + ######################################################################## # Constituent auto-resolution (cam-sima-style consumer schemes) ######################################################################## From 7a3263e729a231398dfbc323e43596abf2d032ec Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 4 Jun 2026 14:15:39 -0600 Subject: [PATCH 54/74] Fix unique names in suite-owned objects, triggered by CAM-SIMA rrtmgp test --- capgen-ng/generator/suite_data.py | 9 ++- capgen-ng/generator/suite_resolver.py | 33 ++++++++- capgen-ng/generator/suite_types.py | 11 ++- unit-tests/test_suite_resolver.py | 101 ++++++++++++++++++++++++++ unit-tests/test_suite_types.py | 2 +- 5 files changed, 147 insertions(+), 9 deletions(-) diff --git a/capgen-ng/generator/suite_data.py b/capgen-ng/generator/suite_data.py index 31187555..fc653f2e 100644 --- a/capgen-ng/generator/suite_data.py +++ b/capgen-ng/generator/suite_data.py @@ -144,9 +144,12 @@ def _collect_ddt_uses( if ddt_module_map is None or t not in ddt_module_map: raise CCPPError( "Suite-owned variable '{}' has DDT type '{}' but its " - "defining Fortran module is unknown; the DDT must appear " - "in a metadata file alongside a scheme/host/control " - "table".format(suite_var.standard_name, t) + "defining Fortran module is unknown. Declare it explicitly " + "with 'module_name = ' in the '{}' DDT's " + "[ccpp-table-properties], or co-locate the DDT table with a " + "scheme/host/control table in the same .meta file.".format( + suite_var.standard_name, t, t, + ) ) mod = ddt_module_map[t] uses.setdefault(mod, []) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 55e355b2..ae9e8728 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -58,6 +58,7 @@ ====== ================ ============ """ +import hashlib import re from dataclasses import dataclass, field from typing import Dict, List, Optional, Set, Tuple, Union @@ -912,6 +913,36 @@ def _root_symbol(access_path: str) -> str: return re.split(r'[%(]', access_path)[0] +_FORTRAN_ID_LIMIT_SV = 63 + + +def _unique_suite_field(desired: str, std_name: str, + suite_vars: Dict[str, 'SuiteVar']) -> str: + """Return a suite-data field name unique across existing suite vars. + + Two *distinct* suite-owned variables (different standard names) can be + first produced by scheme args that happen to share a local name -- e.g. + ``kdist`` for both ``longwave_gas_optics_object_for_RRTMGP`` and + ``shortwave_gas_optics_object_for_RRTMGP``, or ``hrate`` for the lw/sw + heating-rate tendencies. The generated ``ccpp__data`` DDT + declares one component per suite var named by this field, and the group + cap accesses it via the same name (``SuiteVar.access_path``), so the + field MUST be unique -- otherwise Fortran rejects the duplicate + component. + + Keep the bare name for the first occurrence (readable common case); + disambiguate a later collision with a short, deterministic + std_name-derived suffix (stable regardless of resolution order, since it + keys on the standard name rather than a counter). + """ + used = {sv.local_name for sv in suite_vars.values()} + if desired not in used: + return desired + suffix = hashlib.sha1(std_name.encode('utf-8')).hexdigest()[:8] + base = desired[:_FORTRAN_ID_LIMIT_SV - 1 - len(suffix)] + return '{}_{}'.format(base, suffix) + + ######################################################################## # Data classes ######################################################################## @@ -1604,7 +1635,7 @@ def _resolve_one_arg( inst_access = '({})'.format(inst_entry.local_name) if inst_entry else '(1)' suite_var = SuiteVar( standard_name=std_name, - local_name=local, + local_name=_unique_suite_field(local, std_name, suite_vars), type_=scheme_var.type, kind=scheme_var.kind, units=scheme_var.units, diff --git a/capgen-ng/generator/suite_types.py b/capgen-ng/generator/suite_types.py index 5baef2d1..20a46fa8 100644 --- a/capgen-ng/generator/suite_types.py +++ b/capgen-ng/generator/suite_types.py @@ -371,10 +371,13 @@ def _collect_ddt_uses( # DDT — look up the defining Fortran module. if ddt_module_map is None or t not in ddt_module_map: raise CCPPError( - "Pointer wrapper needs DDT '{}' but no defining module " - "is known. Declare the DDT via a 'type = ddt' metadata " - "table co-located with its scheme/host/control metadata " - "so build_ddt_module_map can pick it up.".format(t) + "Pointer wrapper needs DDT '{}' but its defining Fortran " + "module is unknown. Declare it explicitly with " + "'module_name = ' in the '{}' DDT's " + "[ccpp-table-properties], or co-locate the DDT table with a " + "scheme/host/control table in the same .meta file.".format( + t, t, + ) ) uses.setdefault(ddt_module_map[t], set()).add(t) return uses diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 0b42baa1..5e701861 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -3947,6 +3947,107 @@ def test_producer_is_suite_var(self): self.assertEqual(val.source, 'suite') +######################################################################## +# Suite-var field-name uniqueness: two distinct suite vars whose +# producing schemes share a local name must get distinct suite_data +# component names (rrtmgp lw/sw `kdist`, `hrate`). +######################################################################## + +_SHARED_LOCALNAME_SCHEMES = ''' +[ccpp-table-properties] + name = prod_lw + type = scheme +[ccpp-arg-table] + name = prod_lw_run + type = scheme +[ obj ] + standard_name = longwave_optics_object + units = none + dimensions = () + type = real | kind = kind_phys + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-table-properties] + name = prod_sw + type = scheme +[ccpp-arg-table] + name = prod_sw_run + type = scheme +[ obj ] + standard_name = shortwave_optics_object + units = none + dimensions = () + type = real | kind = kind_phys + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + +_SHARED_LOCALNAME_SUITE = ( + '\n' + '\n' + ' \n' + ' prod_lw\n' + ' prod_sw\n' + ' \n' + '\n' +) + + +class TestSuiteVarFieldNameUniqueness(unittest.TestCase): + """Two distinct suite-owned vars (different std names) first produced by + scheme args sharing a local name must get UNIQUE suite_data field names, + else the generated DDT has a duplicate component. Regression for the + rrtmgp lw/sw ``kdist`` / ``hrate`` collision.""" + + @classmethod + def setUpClass(cls): + import logging + from generator.suite_xml import parse_suite_xml + from generator.suite_data import _generate_suite_data + hd = _load_constituent_host_dict() + store = SchemeStore.build_from(_parse(_SHARED_LOCALNAME_SCHEMES)) + with tempfile.TemporaryDirectory() as tmp: + sx = os.path.join(tmp, 'suite_sharedln.xml') + with open(sx, 'w') as fh: + fh.write(_SHARED_LOCALNAME_SUITE) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + cls.sr = resolve_suite(suite, store, hd) + run_calls = list(iter_phase_calls(cls.sr.groups[0].phase_calls['run'])) + cls.svars = cls.sr.suite_vars + cls.data_lines = _generate_suite_data('sharedln', cls.svars) + + def test_both_suite_vars_present(self): + self.assertIn('longwave_optics_object', self.svars) + self.assertIn('shortwave_optics_object', self.svars) + + def test_field_names_distinct(self): + lw = self.svars['longwave_optics_object'].local_name + sw = self.svars['shortwave_optics_object'].local_name + self.assertNotEqual(lw, sw) + # One keeps the bare name; the other is disambiguated. + self.assertIn('obj', (lw, sw)) + + def test_no_duplicate_component_in_generated_type(self): + decls = [l for l in self.data_lines + if 'allocatable ::' in l or l.strip().startswith('real')] + names = [l.split('::')[1].strip().split('(')[0].strip() + for l in decls if '::' in l] + self.assertEqual(len(names), len(set(names)), + 'duplicate suite_data component: {}'.format(names)) + + ######################################################################## # Constituent auto-resolution (cam-sima-style consumer schemes) ######################################################################## diff --git a/unit-tests/test_suite_types.py b/unit-tests/test_suite_types.py index 819491ba..5ffa2c80 100644 --- a/unit-tests/test_suite_types.py +++ b/unit-tests/test_suite_types.py @@ -271,7 +271,7 @@ def test_external_module_parsed_from_type(self): def test_missing_ddt_in_map_raises(self): combos = {('some_ddt', '', 1)} - with self.assertRaisesRegex(CCPPError, "no defining module"): + with self.assertRaisesRegex(CCPPError, "module_name"): _collect_ddt_uses(combos, {}) From 5cf0b5dd56899c70a9b3c1de6e3f2ec8c8dc1588 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 4 Jun 2026 14:45:56 -0600 Subject: [PATCH 55/74] Bug fix for allocatable scheme data --- capgen-ng/generator/suite_data.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/capgen-ng/generator/suite_data.py b/capgen-ng/generator/suite_data.py index fc653f2e..1ae09231 100644 --- a/capgen-ng/generator/suite_data.py +++ b/capgen-ng/generator/suite_data.py @@ -331,7 +331,15 @@ def _generate_suite_data( '{}errflg = 0'.format(i2), ] for suite_var in sorted_svs: - if suite_var.dimensions: + # Allocatable suite vars are owned-and-allocated by the producing + # scheme itself (its dummy is declared ``allocatable, intent(out)``, + # so the callee performs the allocation). The suite must NOT + # pre-allocate them here -- doing so is at best redundant (the + # scheme's intent(out) auto-deallocates on entry) and at worst + # wrong (its extent may not be known until the scheme runs). The + # suite still OWNS the storage, so final_fields below deallocates + # it regardless of who allocated it. + if suite_var.dimensions and not suite_var.allocatable: dim_exprs = [ _dim_local_expr(d, suite_vars, host_dict) for d in suite_var.dimensions @@ -360,6 +368,11 @@ def _generate_suite_data( "{}errmsg = ''".format(i2), '{}errflg = 0'.format(i2), ] + # Deallocate ALL dimensioned fields here -- including allocatable + # ones that a scheme allocated itself. The storage is a component of + # the suite-owned ``ccpp_suite_data`` DDT, so the suite owns its + # teardown; the ``if (allocated(...))`` guard makes this safe whether + # the scheme allocated it, never ran, or already freed it. for suite_var in sorted_svs: if suite_var.dimensions: lines.append( From 130f2ffdcaebe154241eaf3c9e4e685de2b03691 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 4 Jun 2026 15:23:00 -0600 Subject: [PATCH 56/74] Third rrtmgp-related bugfix in capgen-ng - dimensions for suite variables --- capgen-ng/generator/suite_cap.py | 17 +++++ unit-tests/test_suite_cap.py | 106 +++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index ea874dda..8745aa85 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -30,6 +30,7 @@ ResolvedGroup, SuiteResolution, iter_phase_calls, + _root_symbol, # auto-clone-constituents: legacy-shim payload type. AutoCloneEntry, ) @@ -222,6 +223,22 @@ def _register_uses( mod = arg.module_name if mod is not None: uses.setdefault(mod, set()).add(arg.root_symbol) + # Dimension variables referenced in the arg's subscript (e.g. + # ``rad_climate(1:rad_climate_dimension)``) must also be in + # scope. Mirror the group cap's _collect_dim_uses: a host dim + # USEs its access-path root from the declaring module; a + # suite-owned dim USEs ccpp__data. (Without this the + # register subroutine references the dimension symbol with no + # IMPLICIT type.) + for dim_std in arg.used_dim_std_names: + entry = host_dict.get(dim_std) if host_dict else None + if entry is not None and entry.module_name is not None: + uses.setdefault(entry.module_name, set()).add( + _root_symbol(entry.access_path) + ) + elif dim_std in suite_res.suite_vars: + sv = suite_res.suite_vars[dim_std] + uses.setdefault(sv.module_name, set()).add('ccpp_suite_data') # Per-suite dynamic-constituent buffer is owned by ccpp_host_constituents # and written into here. Pull in the constituent property type plus the # buffer symbol. diff --git a/unit-tests/test_suite_cap.py b/unit-tests/test_suite_cap.py index b5d14890..0d3699ac 100644 --- a/unit-tests/test_suite_cap.py +++ b/unit-tests/test_suite_cap.py @@ -20,6 +20,8 @@ _load_full_host_dict, _load_scheme_store, _parse_suite, + _parse, + _sf, ) @@ -452,6 +454,110 @@ def test_write_suite_cap_threads_trace_flag(self): self.assertIn('logical, parameter :: trace = .true.', text) +# A register-phase scheme that consumes a HOST array sliced by a HOST +# dimension (gases(1:n_gases)) and produces a constituent (so the register +# call body is emitted). Mirrors rrtmgp_constituents_register, whose +# rad_climate(1:rad_climate_dimension) arg triggered the bug. +_REG_HOSTDIM_HOST = ''' +[ccpp-table-properties] + name = gasreg_host + type = host +[ccpp-arg-table] + name = gasreg_host + type = host +[ n_gases ] + standard_name = gas_list_dimension + units = count + dimensions = () + type = integer +[ gases ] + standard_name = list_of_gases + units = none + type = character | kind = len=256 + dimensions = (gas_list_dimension) +''' + +_REG_HOSTDIM_SCHEME = ''' +[ccpp-table-properties] + name = gas_register + type = scheme +[ccpp-arg-table] + name = gas_register_register + type = scheme +[ gases ] + standard_name = list_of_gases + units = none + type = character | kind = len=256 + dimensions = (gas_list_dimension) + intent = in +[ dyn_consts ] + standard_name = gasreg_dyn_consts + units = none + type = ccpp_constituent_properties_t + allocatable = True + dimensions = (:) + intent = out +[ errmsg ] + standard_name = ccpp_error_message + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[ errflg ] + standard_name = ccpp_error_code + units = 1 + dimensions = () + type = integer + intent = out +''' + +_REG_HOSTDIM_SUITE = ( + '\n' + '\n' + ' \n' + ' gas_register\n' + ' \n' + '\n' +) + + +class TestRegisterHostDimensionImported(unittest.TestCase): + """A register-phase scheme arg that is a host array sliced by a host + dimension must import BOTH the array and its dimension symbol into + _register. Regression for the rrtmgp + 'rad_climate_dimension has no IMPLICIT type' suite-cap error.""" + + @classmethod + def setUpClass(cls): + import logging + from generator.suite_xml import parse_suite_xml + host_tbls = _parse(_REG_HOSTDIM_HOST, 'gasreg_host.meta') + ctrl_tbls = parse_metadata_file(_sf('control_full.meta')) + hd = build_flat_host_dict(host_tbls, ctrl_tbls, []) + store = SchemeStore.build_from( + _parse(_REG_HOSTDIM_SCHEME, 'gas_register.meta')) + with tempfile.TemporaryDirectory() as tmp: + sx = os.path.join(tmp, 'suite_gasreg.xml') + with open(sx, 'w') as fh: + fh.write(_REG_HOSTDIM_SUITE) + suite = parse_suite_xml(sx, tmp, logging.getLogger('test'), + skip_validation=True) + sr = resolve_suite(suite, store, hd) + cls.text = '\n'.join(_generate_suite_cap('gasreg', sr, store, hd)) + cls.reg = cls.text.split('subroutine gasreg_register')[1].split( + 'end subroutine gasreg_register')[0] + + def test_call_slices_array_by_dimension(self): + # The call subscript references the host dimension's local name. + self.assertIn('gases(1:n_gases)', self.reg) + + def test_dimension_symbol_imported(self): + # Both the array and the dimension must be USE'd from the host module. + self.assertRegex(self.reg, r'use gasreg_host, only:[^\n]*\bn_gases\b') + self.assertRegex(self.reg, r'use gasreg_host, only:[^\n]*\bgases\b') + + def load_tests(loader, tests, ignore): import generator.suite_cap as subcycle tests.addTests(doctest.DocTestSuite(subcycle)) From 007a0ea3dbae750a2ff4e2a39de550fbede505c4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 4 Jun 2026 17:50:05 -0600 Subject: [PATCH 57/74] Bug fix and improvement: catch non-allocatable variables that depend on dimensions that are updated during the run --- capgen-ng/ccpp_capgen_ng.py | 8 +++- capgen-ng/generator/suite_resolver.py | 57 ++++++++++++++++++++++ unit-tests/test_suite_resolver.py | 68 +++++++++++++++++++++++++++ 3 files changed, 132 insertions(+), 1 deletion(-) diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index 5b112ed3..cc02dc3f 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -90,7 +90,9 @@ ) from generator.kinds_writer import write_ccpp_kinds from generator.suite_xml import parse_suite_xml_files -from generator.suite_resolver import resolve_suite, iter_phase_calls +from generator.suite_resolver import ( + resolve_suite, iter_phase_calls, validate_init_dimensions, +) from generator.group_cap import write_group_cap from generator.suite_data import write_suite_data, write_suite_meta from generator.suite_cap import write_suite_cap @@ -954,6 +956,10 @@ def capgen( for suite in suites: log.info("Resolving suite '%s'", suite.name) suite_res = resolve_suite(suite, scheme_store, host_dict) + # Fail early (before any cap is written) if a non-allocatable + # suite-owned variable is dimensioned by a scheme-updated quantity + # that isn't known when suite_data_init_fields allocates it. + validate_init_dimensions(suite_res) suite_names.append(suite.name) suite_resolutions.append(suite_res) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index ae9e8728..a244c562 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -2485,6 +2485,63 @@ def resolve_suite( ) +def validate_init_dimensions(suite_res: SuiteResolution) -> None: + """Reject suite-owned vars whose suite-time allocation can't be sized. + + capgen-ng allocates every non-allocatable, dimensioned suite-owned + variable once in ``suite_data_init_fields``, which runs at the very + start of ``_init`` -- before any ``init`` / ``timestep_init`` / + ``run`` scheme code. Only the ``register`` phase completes earlier, so + a dimension whose value is written by a scheme in any later phase is not + yet set when the allocation happens (the size would be uninitialised + memory). Such a variable must instead be declared ``allocatable`` + (Fortran ``allocatable, intent(out)`` + metadata ``allocatable = True``) + so its producing scheme allocates it once the size is known. + + This is sound (no false positives) but intentionally partial: it sees + only dimensions a *scheme* writes. A host that recomputes a host-owned + dimension in its own driver each step is invisible here -- there is no + metadata signal for it. + + Raises + ------ + CCPPError + If a non-allocatable suite-owned variable is dimensioned by a + standard name written by a scheme in a phase after ``register``. + """ + written_after_register: Set[str] = set() + for resolved_group in suite_res.groups: + for phase, items in resolved_group.phase_calls.items(): + if phase == 'register': + continue + for resolved_call in iter_phase_calls(items): + for arg in resolved_call.args: + if (arg.intent or 'in') in ('out', 'inout'): + written_after_register.add(arg.standard_name) + + for suite_var in suite_res.suite_vars.values(): + if suite_var.allocatable or not suite_var.dimensions: + continue + for dim in suite_var.dimensions: + for token in dim.split(':'): + token = token.strip() + if token in written_after_register: + raise CCPPError( + "Suite-owned variable '{var}' is allocated by the " + "suite at init time (in suite_data_init_fields), but " + "its dimension '{dim}' is written by a scheme in a " + "phase after 'register', so its size is not yet known " + "when the allocation runs (the allocation would use " + "uninitialised memory).\n" + " Declare '{var}' allocatable -- 'allocatable, " + "intent(out)' in the producing scheme's Fortran and " + "'allocatable = True' in its metadata -- so the scheme " + "allocates it once '{dim}' is set.".format( + var=suite_var.standard_name, dim=token, + ) + ) + + # auto-clone-constituents: BEGIN legacy-shim helpers. Delete this # block together with the rest of the auto-clone-constituents # touchpoints; nothing else in the resolver references these. diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 5e701861..d8d0ac0b 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -11,6 +11,7 @@ import os import sys import tempfile +import types import unittest from metadata.metadata_table import _parse_lines, parse_metadata_file @@ -33,6 +34,7 @@ _dedup_scheme_names, resolve_suite, iter_phase_calls, + validate_init_dimensions, SuiteVar, ResolvedArg, ResolvedCall, @@ -4967,6 +4969,72 @@ def test_end_to_end_ddt_dim_in_group_cap(self): # Doctest loader ######################################################################## +######################################################################## +# validate_init_dimensions: a non-allocatable suite var dimensioned by a +# scheme-updated-after-register quantity must be allocatable. +######################################################################## + +class TestValidateInitDimensions(unittest.TestCase): + """Reject a non-allocatable suite var whose dimension is written by a + scheme in a phase after register (capgen can't size it at init). + Regression for the rrtmgp pint_day OOM.""" + + @staticmethod + def _arg(std, intent): + return types.SimpleNamespace(standard_name=std, intent=intent) + + @classmethod + def _sr(cls, phase_writes, suite_vars): + # phase_writes: {phase: [(std, intent), ...]}; suite_vars: [(std, alloc, dims)] + calls = [] + pc = {} + for phase, writes in phase_writes.items(): + pc[phase] = [ResolvedCall(scheme_name='s', phase=phase, + args=[cls._arg(s, i) for s, i in writes])] + grp = types.SimpleNamespace(phase_calls=pc) + svs = {s: types.SimpleNamespace(standard_name=s, allocatable=a, dimensions=d) + for s, a, d in suite_vars} + return types.SimpleNamespace(groups=[grp], suite_vars=svs) + + def test_late_written_dim_nonalloc_raises(self): + sr = self._sr( + {'timestep_init': [('mydim', 'out')], 'run': [('myarr', 'out')]}, + [('mydim', False, []), ('myarr', False, ['mydim'])], + ) + with self.assertRaisesRegex(CCPPError, "myarr.*mydim|allocatable"): + validate_init_dimensions(sr) + + def test_register_written_dim_ok(self): + sr = self._sr( + {'register': [('mydim', 'out')], 'run': [('myarr', 'out')]}, + [('mydim', False, []), ('myarr', False, ['mydim'])], + ) + validate_init_dimensions(sr) # no raise + + def test_allocatable_var_skipped(self): + sr = self._sr( + {'timestep_init': [('mydim', 'out')], 'run': [('myarr', 'out')]}, + [('mydim', False, []), ('myarr', True, ['mydim'])], + ) + validate_init_dimensions(sr) # allocatable -> not capgen's to size + + def test_host_static_dim_ok(self): + # 'hostdim' is never written by a scheme -> assumed available at init. + sr = self._sr( + {'run': [('myarr', 'out')]}, + [('myarr', False, ['hostdim'])], + ) + validate_init_dimensions(sr) # no raise + + def test_range_dimension_token_checked(self): + sr = self._sr( + {'timestep_init': [('mydim', 'inout')], 'run': [('myarr', 'out')]}, + [('mydim', False, []), ('myarr', False, ['ccpp_constant_one:mydim'])], + ) + with self.assertRaises(CCPPError): + validate_init_dimensions(sr) + + def load_tests(loader, tests, ignore): import generator.suite_resolver as suite_resolution import generator.group_cap as gc From 2424718161400c5eca32f9bf17c46bd05c70b882 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 5 Jun 2026 07:50:52 -0600 Subject: [PATCH 58/74] BUg fixes for num_ccpp_constituents dimensioned objects; end-to-end tests for constituents dims and suite allocatable arrays --- capgen-ng/generator/suite_data.py | 23 ++++ capgen-ng/generator/suite_resolver.py | 13 ++ doc/migration.md | 59 +++++++++ end-to-end-tests/CMakeLists.txt | 2 + .../constituents_dim/CMakeLists.txt | 70 ++++++++++ end-to-end-tests/constituents_dim/README.md | 29 +++++ .../constituents_dim/const_dim_consumer.F90 | 52 ++++++++ .../constituents_dim/const_dim_consumer.meta | 39 ++++++ .../constituents_dim/const_dim_producer.F90 | 64 +++++++++ .../constituents_dim/const_dim_producer.meta | 55 ++++++++ .../constituents_dim_suite.xml | 9 ++ .../constituents_dim/host_data.F90 | 23 ++++ .../constituents_dim/host_data.meta | 26 ++++ end-to-end-tests/constituents_dim/main.F90 | 123 ++++++++++++++++++ end-to-end-tests/constituents_dim/main.meta | 65 +++++++++ .../constituents_dim/register_consts.F90 | 55 ++++++++ .../constituents_dim/register_consts.meta | 31 +++++ .../suite_allocate/CMakeLists.txt | 69 ++++++++++ end-to-end-tests/suite_allocate/README.md | 33 +++++ end-to-end-tests/suite_allocate/data.F90 | 17 +++ end-to-end-tests/suite_allocate/data.meta | 14 ++ end-to-end-tests/suite_allocate/main.F90 | 114 ++++++++++++++++ end-to-end-tests/suite_allocate/main.meta | 65 +++++++++ .../suite_allocate/make_workspace.F90 | 41 ++++++ .../suite_allocate/make_workspace.meta | 39 ++++++ .../suite_allocate/suite_allocate_suite.xml | 8 ++ .../suite_allocate/use_workspace.F90 | 52 ++++++++ .../suite_allocate/use_workspace.meta | 65 +++++++++ unit-tests/test_suite_data.py | 42 +++++- unit-tests/test_suite_resolver.py | 23 ++++ 30 files changed, 1319 insertions(+), 1 deletion(-) create mode 100644 end-to-end-tests/constituents_dim/CMakeLists.txt create mode 100644 end-to-end-tests/constituents_dim/README.md create mode 100644 end-to-end-tests/constituents_dim/const_dim_consumer.F90 create mode 100644 end-to-end-tests/constituents_dim/const_dim_consumer.meta create mode 100644 end-to-end-tests/constituents_dim/const_dim_producer.F90 create mode 100644 end-to-end-tests/constituents_dim/const_dim_producer.meta create mode 100644 end-to-end-tests/constituents_dim/constituents_dim_suite.xml create mode 100644 end-to-end-tests/constituents_dim/host_data.F90 create mode 100644 end-to-end-tests/constituents_dim/host_data.meta create mode 100644 end-to-end-tests/constituents_dim/main.F90 create mode 100644 end-to-end-tests/constituents_dim/main.meta create mode 100644 end-to-end-tests/constituents_dim/register_consts.F90 create mode 100644 end-to-end-tests/constituents_dim/register_consts.meta create mode 100644 end-to-end-tests/suite_allocate/CMakeLists.txt create mode 100644 end-to-end-tests/suite_allocate/README.md create mode 100644 end-to-end-tests/suite_allocate/data.F90 create mode 100644 end-to-end-tests/suite_allocate/data.meta create mode 100644 end-to-end-tests/suite_allocate/main.F90 create mode 100644 end-to-end-tests/suite_allocate/main.meta create mode 100644 end-to-end-tests/suite_allocate/make_workspace.F90 create mode 100644 end-to-end-tests/suite_allocate/make_workspace.meta create mode 100644 end-to-end-tests/suite_allocate/suite_allocate_suite.xml create mode 100644 end-to-end-tests/suite_allocate/use_workspace.F90 create mode 100644 end-to-end-tests/suite_allocate/use_workspace.meta diff --git a/capgen-ng/generator/suite_data.py b/capgen-ng/generator/suite_data.py index 1ae09231..bc1d9134 100644 --- a/capgen-ng/generator/suite_data.py +++ b/capgen-ng/generator/suite_data.py @@ -15,6 +15,17 @@ 'real', 'integer', 'character', 'logical', 'complex', 'double precision' }) +# Framework-provided constituent count dimension. A suite-owned variable may be +# dimensioned by number_of_ccpp_constituents; unlike host/suite dims its extent +# is owned by the framework's per-instance constituent object +# (``ccpp_model_constituents_obj(i)%num_layer_vars``) declared in module +# ``ccpp_host_constituents``. Mirrors the constants of the same name in +# generator.suite_resolver. +_CONST_NUM_STD = 'number_of_ccpp_constituents' +_CONST_OBJ_VAR = 'ccpp_model_constituents_obj' +_CONST_OBJ_MODULE = 'ccpp_host_constituents' +_CONST_NUM_MEMBER = 'num_layer_vars' + def _type_str(type_: str, kind: str) -> str: """Return the Fortran type clause for a SuiteVar field. @@ -67,6 +78,14 @@ def _collect_dim_uses( # Suite-owned dim → no USE needed (same module access). if dim_std in suite_var_std_names: continue + # Framework constituent count → extent comes from the per-instance + # constituent object, not the host or a suite scalar. USE its + # module so init_fields can reference the count member. + if dim_std == _CONST_NUM_STD: + uses.setdefault(_CONST_OBJ_MODULE, []) + if _CONST_OBJ_VAR not in uses[_CONST_OBJ_MODULE]: + uses[_CONST_OBJ_MODULE].append(_CONST_OBJ_VAR) + continue if host_dict is None: raise CCPPError( "Suite-owned variable '{}' has dimension '{}' but no host " @@ -105,6 +124,10 @@ def _dim_local_expr(dim_std: str, suite_vars: Dict[str, SuiteVar], host_dict) -> """ if dim_std in suite_vars: return 'ccpp_suite_data(i)%{}'.format(suite_vars[dim_std].local_name) + # Framework constituent count: extent from the per-instance constituent + # object (``i`` is the instance index in the init_fields alloc context). + if dim_std == _CONST_NUM_STD: + return '{}(i)%{}'.format(_CONST_OBJ_VAR, _CONST_NUM_MEMBER) entry = host_dict.get(dim_std) if host_dict else None if entry is None: raise CCPPError( diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index a244c562..3da74be0 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -573,6 +573,19 @@ def _one_dim_part( lower_str = lower_str.strip() upper_str = upper_str.strip() + # Framework-provided constituent count dimension. Any variable -- host, + # suite-owned, or scheme -- may be dimensioned by + # ``number_of_ccpp_constituents``. The framework owns the extent via the + # per-instance constituent object, so a *call subscript* passes the whole + # constituent axis. This mirrors :func:`_const_dim_part` (which only fires + # for framework-constituent args) and generalises the recognition to every + # other variable dimensioned by the count. There is no host scalar to USE, + # so ``used`` is left untouched. (Allocating a suite-owned var sized by + # this dim is handled separately in generator.suite_data, which resolves the + # extent to ``ccpp_model_constituents_obj(i)%num_layer_vars``.) + if upper_str == _CONST_NUM_STD: + return ':', used + # Registered scalar-index dimension: collapse to the paired index # variable's local Fortran name regardless of lower bound. See # capgen-ng/metadata/registered_dimensions.py for the contract. diff --git a/doc/migration.md b/doc/migration.md index b2f48984..9fb60ef9 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -72,6 +72,16 @@ Example with multi-line dependencies (real CCPP physics pattern): dependencies = Radiation/RRTMG/radlw_main.F90,Radiation/RRTMG/radsw_main.F90 ``` +> **Standalone DDT files require `module_name`.** A `type = ddt` table +> in a `.meta` file with **no co-located** `scheme`/`host`/`control` +> table (a "wrapper object" like `ccpp_optical_props.meta` defining +> `ty_optical_props_1scl_ccpp`) cannot inherit its module from a sibling. +> If the defining Fortran module name differs from the DDT table (type) +> name — which it almost always does for these wrappers — you **must** +> declare `module_name` explicitly. capgen-ng does *not* guess (e.g. +> from the file name); a DDT it can't resolve raises a clear error at +> generation time naming the type and the `module_name` remedy. + ### 1.3 New per-variable attributes Inside a `[ var_name ]` section. All optional. @@ -83,6 +93,7 @@ Inside a `[ var_name ]` section. All optional. | `advected` | bool | `False` | Scheme metadata only. | | `molar_mass` | float | `0.0` | Scheme metadata only. | | `diagnostic_name` | str | (defaults to `local_name`) | Host-tooling hint; mutually exclusive with `diagnostic_name_fixed`. | +| `allocatable` | bool | `False` | Must match the Fortran `allocatable` attribute on the dummy. Required for any array the *scheme* allocates (see §1.3.3). | #### 1.3.1 Host `active` + scheme arg shape @@ -183,6 +194,54 @@ the same checks against the frozen fields. Error messages name the source as `host`, `control`, or `suite` so you know whose contract you're violating. +#### 1.3.3 `allocatable` and who owns suite-data allocation + +A **suite-owned variable** (an interstitial: first written by a scheme +with `intent=out`, then consumed by another) is stored as a component +of the generated `ccpp__data` DDT. capgen-ng allocates it for +you — **once**, in `suite_data_init_fields`, which runs at the very +start of `_init`. That works only when every dimension is known +that early, i.e. a **host variable** or a value set in the **`register`** +phase. + +When the size is *not* known at init — the array is dimensioned by a +quantity a scheme computes later (in `init` / `timestep_init` / `run`, +e.g. a per-timestep daylight-column count) — the suite cannot size it. +Such a variable must be declared **`allocatable`** and allocated by its +**producing scheme**: + +- metadata: `allocatable = True` on the arg; +- Fortran: `allocatable, intent(out)` on the dummy, and an explicit + `allocate(...)` in the scheme body (an `intent(out)` allocatable is + auto-deallocated on entry, so element assignment needs it allocated + first). + +For an `allocatable` arg capgen-ng then: (1) does **not** pre-allocate it +in `init_fields`; (2) passes the **whole** component at call sites +(`...%var`, no array section — an allocatable/assumed-shape mismatch is +otherwise a compile error); and (3) still frees it in +`suite_data_final_fields` under an `if (allocated(...))` guard. So the +**scheme allocates, the suite (or the scheme) deallocates** — the guard +makes either order safe (no leak, no double-free). + +Size it with the **authoritative dimension variable** — the standard +name in the arg's `dimensions` — not a look-alike local or a derived +expression. If that dimension is a scheme-set quantity, pass it in as a +scalar `intent=in` arg and `allocate` with it. (Real example: an array +declared `number_of_vertical_interfaces_in_RRTMGP` must be sized with +that value, **not** the host's `vertical_interface_dimension` nor +`nlay+1`, which differ when the scheme runs on a reduced vertical grid.) + +**Generation-time guard.** capgen-ng rejects a *non*-`allocatable` +suite-owned array whose dimension is written by a scheme in any phase +after `register` — it would otherwise be allocated from uninitialized +memory. The error names the variable, the offending dimension, and the +fix (declare it `allocatable`). The check is sound but partial: it sees +only dimensions a *scheme* writes, not a host scalar the host driver +re-computes each step — those remain the author's responsibility, and +the rule "anything the scheme allocates must be `allocatable` in Fortran +and metadata" is ultimately enforced by the compiler plus `ccpp_validator`. + ### 1.4 Sliced local names with long subscript indices Local names with array slices may carry CCPP standard names as subscript diff --git a/end-to-end-tests/CMakeLists.txt b/end-to-end-tests/CMakeLists.txt index 76109f2d..3e7232b9 100644 --- a/end-to-end-tests/CMakeLists.txt +++ b/end-to-end-tests/CMakeLists.txt @@ -79,6 +79,8 @@ add_subdirectory(instances) # Run most complex tests last add_subdirectory(capgen_ng) add_subdirectory(var_compat) +add_subdirectory(suite_allocate) +add_subdirectory(constituents_dim) add_subdirectory(advection) add_subdirectory(instances_advection) # auto-clone-constituents: Remove this test when the legacy switch to support diff --git a/end-to-end-tests/constituents_dim/CMakeLists.txt b/end-to-end-tests/constituents_dim/CMakeLists.txt new file mode 100644 index 00000000..b05b5b98 --- /dev/null +++ b/end-to-end-tests/constituents_dim/CMakeLists.txt @@ -0,0 +1,70 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "register_consts" "const_dim_producer" "const_dim_consumer") +set(HOST_FILES "host_data" "main") +set(SUITE_FILES "constituents_dim_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Run ccpp_capgen_ng +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files (incl. the constituent framework module +# and generated ccpp_host_constituents) and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_constituents_dim.x + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} +) +target_link_libraries(test_constituents_dim.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_constituents_dim.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_constituents_dim.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_constituents_dim + COMMAND test_constituents_dim.x) diff --git a/end-to-end-tests/constituents_dim/README.md b/end-to-end-tests/constituents_dim/README.md new file mode 100644 index 00000000..9f4cd8ff --- /dev/null +++ b/end-to-end-tests/constituents_dim/README.md @@ -0,0 +1,29 @@ +# constituents_dim test + +Covers variables dimensioned by the framework constituent count +number_of_ccpp_constituents (which the host never declares as a scalar — the +framework owns it). register_consts registers 3 dynamic constituents, so the +count is 3 for the rest of the suite. Three cases, each a distinct capgen path: + +- Case 1 — host var dimensioned by the count. surface_upward_test_constituent_flux + (horizontal_dimension, number_of_ccpp_constituents) is host-owned; the host + sizes it to the runtime count and capgen passes the whole constituent axis as + : to const_dim_producer. +- Case 2a — non-allocatable suite var, framework allocates in init_fields. + test_constituent_workspace(number_of_ccpp_constituents) is suite-owned and + not allocatable, so suite_data_init_fields allocates it via + ccpp_model_constituents_obj(i)%num_layer_vars. +- Case 2b — allocatable suite var, the scheme allocates in _run. + test_allocatable_constituent_workspace(number_of_ccpp_constituents) is + allocatable = True; init_fields skips it and const_dim_producer allocates + it using the count received as a scalar (number_of_ccpp_constituents --> + ccpp_model_constituents_obj(inst)%num_layer_vars). Uses the existing + suite-owned-allocatable path. final_fields frees both suite workspaces. + +The producer fills the workspaces and verifies Case 1; the consumer verifies +Cases 2a/2b. Any mismatch sets errcode, failing the run. Built with +-fcheck=all, so allocation/teardown errors fail the test. + +The 2a vs 2b contrast is the same ownership rule as suite_allocate: a +non-allocatable suite var is framework-owned (allocated in init_fields); an +allocatable = True one is scheme-owned (allocated in _run). diff --git a/end-to-end-tests/constituents_dim/const_dim_consumer.F90 b/end-to-end-tests/constituents_dim/const_dim_consumer.F90 new file mode 100644 index 00000000..f81690b6 --- /dev/null +++ b/end-to-end-tests/constituents_dim/const_dim_consumer.F90 @@ -0,0 +1,52 @@ +!>\file const_dim_consumer.F90 +!! Consumes the two suite-owned workspaces produced by const_dim_producer and +!! verifies their contents. Both are dimensioned by number_of_ccpp_constituents; +!! cwork was allocated by the framework (init_fields, Case 2a) and awork by the +!! producing scheme (_run, Case 2b). Receiving them through plain (non-allocatable) +!! dummies exercises capgen passing the allocated components to a consumer. + +module const_dim_consumer + + use ccpp_kinds, only: kind_phys + + implicit none + + private + public :: const_dim_consumer_run + +contains + + !! \section arg_table_const_dim_consumer_run Argument Table + !! \htmlinclude const_dim_consumer_run.html + !! + subroutine const_dim_consumer_run(cwork, awork, errmsg, errcode) + real(kind=kind_phys), intent(in) :: cwork(:) + real(kind=kind_phys), intent(in) :: awork(:) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errcode + + integer :: m + + errmsg = '' + errcode = 0 + + ! Case 2a: framework-allocated suite workspace, filled by the producer. + do m = 1, size(cwork) + if (cwork(m) /= real(10 * m, kind_phys)) then + errcode = 1 + errmsg = 'Case 2a: framework-allocated suite workspace has wrong value' + return + end if + end do + + ! Case 2b: scheme-allocated suite workspace, filled by the producer. + do m = 1, size(awork) + if (awork(m) /= real(100 * m, kind_phys)) then + errcode = 1 + errmsg = 'Case 2b: scheme-allocated suite workspace has wrong value' + return + end if + end do + end subroutine const_dim_consumer_run + +end module const_dim_consumer diff --git a/end-to-end-tests/constituents_dim/const_dim_consumer.meta b/end-to-end-tests/constituents_dim/const_dim_consumer.meta new file mode 100644 index 00000000..fe52cb0b --- /dev/null +++ b/end-to-end-tests/constituents_dim/const_dim_consumer.meta @@ -0,0 +1,39 @@ +[ccpp-table-properties] + name = const_dim_consumer + type = scheme + dependencies = + +[ccpp-arg-table] + name = const_dim_consumer_run + type = scheme +[cwork] + standard_name = test_constituent_workspace + long_name = non-allocatable suite workspace dimensioned by the count (Case 2a) + units = 1 + dimensions = (number_of_ccpp_constituents) + type = real + kind = kind_phys + intent = in +[awork] + standard_name = test_allocatable_constituent_workspace + long_name = allocatable suite workspace dimensioned by the count (Case 2b) + units = 1 + dimensions = (number_of_ccpp_constituents) + type = real + kind = kind_phys + intent = in +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[errcode] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/constituents_dim/const_dim_producer.F90 b/end-to-end-tests/constituents_dim/const_dim_producer.F90 new file mode 100644 index 00000000..c248f7e3 --- /dev/null +++ b/end-to-end-tests/constituents_dim/const_dim_producer.F90 @@ -0,0 +1,64 @@ +!>\file const_dim_producer.F90 +!! Exercises three ways a variable can be dimensioned by the framework +!! constituent count number_of_ccpp_constituents: +!! Case 1 - consume a HOST array dimensioned by the count (passed as ':'). +!! Case 2a - fill a non-allocatable SUITE workspace the framework allocated +!! in init_fields (sized to ccpp_model_constituents_obj(i)%num_layer_vars). +!! Case 2b - allocate an allocatable SUITE workspace here in _run, sized by the +!! count received as a scalar argument. + +module const_dim_producer + + use ccpp_kinds, only: kind_phys + + implicit none + + private + public :: const_dim_producer_run + +contains + + !! \section arg_table_const_dim_producer_run Argument Table + !! \htmlinclude const_dim_producer_run.html + !! + subroutine const_dim_producer_run(coupler_flux, cwork, n_const, awork, & + errmsg, errcode) + real(kind=kind_phys), intent(in) :: coupler_flux(:, :) + real(kind=kind_phys), intent(out) :: cwork(:) + integer, intent(in) :: n_const + real(kind=kind_phys), allocatable, intent(out) :: awork(:) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errcode + + integer :: m, i + + errmsg = '' + errcode = 0 + + ! Case 1: the host filled coupler_flux(i, m) = m; the constituent axis was + ! passed whole (':'), so size(coupler_flux, 2) is the constituent count. + do m = 1, size(coupler_flux, 2) + do i = 1, size(coupler_flux, 1) + if (coupler_flux(i, m) /= real(m, kind_phys)) then + errcode = 1 + errmsg = 'Case 1: coupler_flux dimensioned by ' // & + 'number_of_ccpp_constituents has the wrong value' + return + end if + end do + end do + + ! Case 2a: fill the framework-allocated (init_fields) suite workspace. + do m = 1, size(cwork) + cwork(m) = real(10 * m, kind_phys) + end do + + ! Case 2b: this scheme owns the allocation, sized by the scalar count which + ! the framework resolves to ccpp_model_constituents_obj(inst)%num_layer_vars. + allocate(awork(n_const)) + do m = 1, n_const + awork(m) = real(100 * m, kind_phys) + end do + end subroutine const_dim_producer_run + +end module const_dim_producer diff --git a/end-to-end-tests/constituents_dim/const_dim_producer.meta b/end-to-end-tests/constituents_dim/const_dim_producer.meta new file mode 100644 index 00000000..09f74c80 --- /dev/null +++ b/end-to-end-tests/constituents_dim/const_dim_producer.meta @@ -0,0 +1,55 @@ +[ccpp-table-properties] + name = const_dim_producer + type = scheme + dependencies = + +[ccpp-arg-table] + name = const_dim_producer_run + type = scheme +[coupler_flux] + standard_name = surface_upward_test_constituent_flux + long_name = host array dimensioned by the constituent count (Case 1) + units = kg m-2 s-1 + dimensions = (horizontal_dimension, number_of_ccpp_constituents) + type = real + kind = kind_phys + intent = in +[cwork] + standard_name = test_constituent_workspace + long_name = non-allocatable suite workspace dimensioned by the count (Case 2a) + units = 1 + dimensions = (number_of_ccpp_constituents) + type = real + kind = kind_phys + intent = out +[n_const] + standard_name = number_of_ccpp_constituents + long_name = number of CCPP constituents + units = count + dimensions = () + type = integer + intent = in +[awork] + standard_name = test_allocatable_constituent_workspace + long_name = allocatable suite workspace dimensioned by the count (Case 2b) + units = 1 + dimensions = (number_of_ccpp_constituents) + type = real + kind = kind_phys + intent = out + allocatable = True +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[errcode] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/constituents_dim/constituents_dim_suite.xml b/end-to-end-tests/constituents_dim/constituents_dim_suite.xml new file mode 100644 index 00000000..819c67bf --- /dev/null +++ b/end-to-end-tests/constituents_dim/constituents_dim_suite.xml @@ -0,0 +1,9 @@ + + + + + register_consts + const_dim_producer + const_dim_consumer + + diff --git a/end-to-end-tests/constituents_dim/host_data.F90 b/end-to-end-tests/constituents_dim/host_data.F90 new file mode 100644 index 00000000..c2726819 --- /dev/null +++ b/end-to-end-tests/constituents_dim/host_data.F90 @@ -0,0 +1,23 @@ +module host_data + + !! \section arg_table_host_data Argument Table + !! \htmlinclude host_data.html + !! + use ccpp_kinds, only: kind_phys + + implicit none + + private + + public :: ncols, pver, coupler_flux + + ! Small single-chunk domain. + integer, parameter :: ncols = 4 + integer, parameter :: pver = 4 + + ! Case 1: a host-owned array dimensioned by number_of_ccpp_constituents. + ! The host allocates it to the runtime constituent count; capgen passes the + ! whole constituent axis (':') to the consuming scheme. + real(kind=kind_phys), allocatable, target :: coupler_flux(:, :) + +end module host_data diff --git a/end-to-end-tests/constituents_dim/host_data.meta b/end-to-end-tests/constituents_dim/host_data.meta new file mode 100644 index 00000000..e82be909 --- /dev/null +++ b/end-to-end-tests/constituents_dim/host_data.meta @@ -0,0 +1,26 @@ +[ccpp-table-properties] + name = host_data + type = host + dependencies = +[ccpp-arg-table] + name = host_data + type = host +[ncols] + standard_name = horizontal_dimension + units = count + type = integer + protected = True + dimensions = () +[pver] + standard_name = vertical_layer_dimension + units = count + type = integer + protected = True + dimensions = () +[coupler_flux] + standard_name = surface_upward_test_constituent_flux + long_name = host-owned array dimensioned by the framework constituent count (Case 1) + units = kg m-2 s-1 + dimensions = (horizontal_dimension, number_of_ccpp_constituents) + type = real + kind = kind_phys diff --git a/end-to-end-tests/constituents_dim/main.F90 b/end-to-end-tests/constituents_dim/main.F90 new file mode 100644 index 00000000..da2062ff --- /dev/null +++ b/end-to-end-tests/constituents_dim/main.F90 @@ -0,0 +1,123 @@ +program test_constituents_dim + + use, intrinsic :: iso_fortran_env, only: output_unit, & + error_unit + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + use host_data, only: ncols, & + pver, & + coupler_flux + + use test_host_ccpp_cap, only: ccpp_register, & + ccpp_register_constituents, & + ccpp_number_constituents, & + ccpp_initialize_constituents, & + ccpp_init, & + ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_final, & + ccpp_physics_final, & + ccpp_final, & + ccpp_deallocate_dynamic_constituents + + implicit none + + character(len=*), parameter :: ccpp_suite = 'constituents_dim_suite' + character(len=512) :: errmsg + integer :: errcode + integer :: num_const + integer :: m, i + type(ccpp_constituent_properties_t), allocatable, target :: host_constituents(:) + + errcode = 0 + errmsg = '' + + ! Register phase: register_consts_register registers 3 dynamic constituents, + ! which is what gives the suite a non-trivial number_of_ccpp_constituents. + call ccpp_register(suite_name=trim(ccpp_suite), errmsg=errmsg, errcode=errcode) + call check('ccpp_register') + + ! This test registers all constituents on the scheme side; the host adds none. + allocate(host_constituents(0)) + call ccpp_register_constituents(host_constituents, errmsg=errmsg, errcode=errcode) + call check('ccpp_register_constituents') + + call ccpp_number_constituents(num_const, errmsg=errmsg, errcode=errcode) + call check('ccpp_number_constituents') + if (num_const < 1) then + write(error_unit, '(a,i0)') & + 'Error: expected at least one constituent, got ', num_const + stop 1 + end if + + call ccpp_initialize_constituents(ncols=ncols, num_layers=pver, & + errcode=errcode, errmsg=errmsg) + call check('ccpp_initialize_constituents') + + ! Case 1: the host owns coupler_flux and sizes it to the constituent count. + ! capgen passes the whole constituent axis (':') to const_dim_producer_run. + allocate(coupler_flux(ncols, num_const)) + do m = 1, num_const + do i = 1, ncols + coupler_flux(i, m) = real(m, kind_phys) + end do + end do + + ! ccpp_init -> suite_data_init_fields allocates the non-allocatable suite + ! workspace (Case 2a) using ccpp_model_constituents_obj(i)%num_layer_vars. + call ccpp_init(suite_name=trim(ccpp_suite), errmsg=errmsg, errcode=errcode) + call check('ccpp_init') + + call ccpp_physics_init(suite_name=trim(ccpp_suite), group_name='all', & + col_start=1, col_end=ncols, thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + call check('ccpp_physics_init') + + call ccpp_physics_timestep_init(suite_name=trim(ccpp_suite), group_name='all', & + col_start=1, col_end=ncols, thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + call check('ccpp_physics_timestep_init') + + ! Producer fills/allocates the workspaces and checks Case 1; consumer verifies + ! Cases 2a/2b. Any mismatch sets errcode inside the schemes. + call ccpp_physics_run(suite_name=trim(ccpp_suite), group_name='all', & + col_start=1, col_end=ncols, thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + call check('ccpp_physics_run') + + call ccpp_physics_timestep_final(suite_name=trim(ccpp_suite), group_name='all', & + col_start=1, col_end=ncols, thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + call check('ccpp_physics_timestep_final') + + call ccpp_physics_final(suite_name=trim(ccpp_suite), group_name='all', & + col_start=1, col_end=ncols, thread_num=1, nthreads=1, nphys_threads=1, & + errmsg=errmsg, errcode=errcode) + call check('ccpp_physics_final') + + ! ccpp_final -> suite_data_final_fields frees both suite workspaces (guarded). + call ccpp_final(suite_name=trim(ccpp_suite), errmsg=errmsg, errcode=errcode) + call check('ccpp_final') + + call ccpp_deallocate_dynamic_constituents() + deallocate(host_constituents) + if (allocated(coupler_flux)) deallocate(coupler_flux) + + write(output_unit, '(a,i0,a)') & + 'PASS: constituents_dim (number_of_ccpp_constituents = ', num_const, ')' + +contains + + subroutine check(phase) + character(len=*), intent(in) :: phase + if (errcode /= 0) then + write(error_unit, '(a)') 'An error occurred in ' // trim(phase) // ':' + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + end subroutine check + +end program test_constituents_dim diff --git a/end-to-end-tests/constituents_dim/main.meta b/end-to-end-tests/constituents_dim/main.meta new file mode 100644 index 00000000..fee18c58 --- /dev/null +++ b/end-to-end-tests/constituents_dim/main.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ col_start ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = count + dimensions = () + type = integer +[ col_end ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = count + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errcode ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer diff --git a/end-to-end-tests/constituents_dim/register_consts.F90 b/end-to-end-tests/constituents_dim/register_consts.F90 new file mode 100644 index 00000000..84a9a4d9 --- /dev/null +++ b/end-to-end-tests/constituents_dim/register_consts.F90 @@ -0,0 +1,55 @@ +!>\file register_consts.F90 +!! Register-phase scheme that registers three dynamic constituents. Declaring a +!! ccpp_constituent_properties_t(:) argument is what activates capgen-ng's +!! constituent machinery, giving the suite a meaningful +!! number_of_ccpp_constituents (= 3 here) for the rest of this test. + +module register_consts + + use ccpp_kinds, only: kind_phys + use ccpp_constituent_prop_mod, only: ccpp_constituent_properties_t + + implicit none + + private + public :: register_consts_register + +contains + + !! \section arg_table_register_consts_register Argument Table + !! \htmlinclude register_consts_register.html + !! + subroutine register_consts_register(dyn_const, errmsg, errcode) + type(ccpp_constituent_properties_t), allocatable, intent(out) :: dyn_const(:) + character(len=512), intent(out) :: errmsg + integer, intent(out) :: errcode + + errmsg = '' + errcode = 0 + + allocate(dyn_const(3), stat=errcode) + if (errcode /= 0) then + errmsg = 'Error allocating dyn_const in register_consts_register' + return + end if + + call dyn_const(1)%instantiate( & + std_name='test_constituent_one', long_name='test constituent one', & + diag_name='TEST_CONST_1', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + errcode=errcode, errmsg=errmsg) + if (errcode /= 0) return + call dyn_const(2)%instantiate( & + std_name='test_constituent_two', long_name='test constituent two', & + diag_name='TEST_CONST_2', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + errcode=errcode, errmsg=errmsg) + if (errcode /= 0) return + call dyn_const(3)%instantiate( & + std_name='test_constituent_three', long_name='test constituent three', & + diag_name='TEST_CONST_3', units='kg kg-1', default_value=0._kind_phys, & + vertical_dim='vertical_layer_dimension', advected=.true., & + errcode=errcode, errmsg=errmsg) + end subroutine register_consts_register + +end module register_consts diff --git a/end-to-end-tests/constituents_dim/register_consts.meta b/end-to-end-tests/constituents_dim/register_consts.meta new file mode 100644 index 00000000..4850032c --- /dev/null +++ b/end-to-end-tests/constituents_dim/register_consts.meta @@ -0,0 +1,31 @@ +[ccpp-table-properties] + name = register_consts + type = scheme + dependencies = + +[ccpp-arg-table] + name = register_consts_register + type = scheme +[dyn_const] + standard_name = dynamic_constituents_for_register_consts + long_name = dynamic constituents registered by this test scheme + units = none + dimensions = (:) + type = ccpp_constituent_properties_t + intent = out + allocatable = True +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=512 + intent = out +[errcode] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/suite_allocate/CMakeLists.txt b/end-to-end-tests/suite_allocate/CMakeLists.txt new file mode 100644 index 00000000..76a11d4b --- /dev/null +++ b/end-to-end-tests/suite_allocate/CMakeLists.txt @@ -0,0 +1,69 @@ +#------------------------------------------------------------------------------ +# +# Create list of SCHEME_FILES, HOST_FILES, and SUITE_FILES +# Paths should be relative to CMAKE_SOURCE_DIR (this file's directory) +# +#------------------------------------------------------------------------------ + +set(SCHEME_FILES "make_workspace" "use_workspace") +set(HOST_FILES "data" "main") +set(SUITE_FILES "suite_allocate_suite.xml") +set(HOST "test_host") +# By default, generated caps go in ccpp subdir +set(OUTPUT_ROOT "${CMAKE_CURRENT_BINARY_DIR}/ccpp") + +# Create lists for Fortran and meta data files from file names +list(TRANSFORM SCHEME_FILES APPEND ".F90" OUTPUT_VARIABLE SCHEME_FORTRAN_FILES) +list(TRANSFORM SCHEME_FILES APPEND ".meta" OUTPUT_VARIABLE SCHEME_METADATA_FILES) +list(TRANSFORM HOST_FILES APPEND ".F90" OUTPUT_VARIABLE HOST_FORTRAN_FILES) +list(TRANSFORM HOST_FILES APPEND ".meta" OUTPUT_VARIABLE HOST_METADATA_FILES) + +# Run ccpp_validator +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${SCHEME_FORTRAN_FILES} + METADATA_FILES ${SCHEME_METADATA_FILES} + TYPE "SCHEME") +ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} + SOURCE_FILES ${HOST_FORTRAN_FILES} + METADATA_FILES ${HOST_METADATA_FILES} + TYPE "HOST") + +# Run ccpp_capgen_ng +ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} + HOSTFILES ${HOST_METADATA_FILES} + SCHEMEFILES ${SCHEME_METADATA_FILES} + SUITES ${SUITE_FILES} + HOST_NAME ${HOST} + OUTPUT_ROOT "${OUTPUT_ROOT}") + +# Retrieve the list of Fortran files required for test host from datatable.xml; +# this includes capgen-generated files and dependencies inferred from metadata +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES_FILTERED ${CCPP_FILES}) +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" + REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) + +message(STATUS "List of capgen dependencies: ${CAPGEN_DEPENDENCIES}") +message(STATUS "List of filtered scheme files: ${SCHEME_FORTRAN_FILES_FILTERED}") +message(STATUS "List of capgen-generated files: ${CAPGEN_FILES}") + +add_executable(test_suite_allocate.x + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES_FILTERED} + ${HOST_FORTRAN_FILES} + ${CAPGEN_FILES} +) +target_link_libraries(test_suite_allocate.x PRIVATE MPI::MPI_Fortran) +if(OPENMP) + target_link_libraries(test_suite_allocate.x PRIVATE OpenMP::OpenMP_Fortran) +endif() +set_target_properties(test_suite_allocate.x PROPERTIES LINKER_LANGUAGE Fortran) + +# Add executable to be called with ctest +add_test(NAME test_suite_allocate + COMMAND test_suite_allocate.x) diff --git a/end-to-end-tests/suite_allocate/README.md b/end-to-end-tests/suite_allocate/README.md new file mode 100644 index 00000000..7add35d1 --- /dev/null +++ b/end-to-end-tests/suite_allocate/README.md @@ -0,0 +1,33 @@ +# suite_allocate test + +Covers the one suite-data feature the rest of the tests do not: a +suite-owned, scheme-allocated variable (allocatable = True). + +scratch_workspace_field is produced by make_workspace (intent=out, +allocatable = True) and consumed by use_workspace. No host table declares +it, so capgen-ng promotes it to a suite-owned variable stored in +ccpp__data. + +Crucially, its dimension workspace_dimension is also suite-owned: it is set +by use_workspace in the timestep_init phase (which runs after +ccpp_init/suite_data_init_fields) — even though use_workspace is listed +after make_workspace in the suite. Phases run suite-wide in order, so the +dimension set in timestep_init is available to every scheme's run. Because +the size is unknown at init, init_fields cannot allocate the array; the +producing scheme must, in the run phase. A non-allocatable version of this is +exactly what validate_init_dimensions rejects. Because it is allocatable: + +- suite_data_init_fields must skip its allocation (the scheme owns it), +- the producing scheme allocates the suite-data component at run time, +- the whole allocated component is passed to the (non-allocatable) consumer dummy, +- suite_data_final_fields frees it (guarded; suite owns teardown). + +The driver asserts the consumer's reduction (workspace_checksum == nw*(nw+1)/2) +and a clean error code. Built with -fcheck=all, so any double-free or +use-after-free in the scheme-allocates / suite-frees ownership split fails the +test. + +This is distinct from: +- capgen_ng — suite-owned vars allocated at init from a register-set dim + (non-allocatable path), and a host-owned allocatable var (model_times). +- nested_suite / var_compat — standalone DDTs with module_name. diff --git a/end-to-end-tests/suite_allocate/data.F90 b/end-to-end-tests/suite_allocate/data.F90 new file mode 100644 index 00000000..fed06847 --- /dev/null +++ b/end-to-end-tests/suite_allocate/data.F90 @@ -0,0 +1,17 @@ +module data + + !! \section arg_table_data Argument Table + !! \htmlinclude data.html + !! + use ccpp_kinds, only: kind_phys + + implicit none + + private + + public :: checksum + + ! Host-owned scalar the consuming scheme fills from the suite-owned workspace. + real(kind=kind_phys) :: checksum + +end module data diff --git a/end-to-end-tests/suite_allocate/data.meta b/end-to-end-tests/suite_allocate/data.meta new file mode 100644 index 00000000..aa051e82 --- /dev/null +++ b/end-to-end-tests/suite_allocate/data.meta @@ -0,0 +1,14 @@ +[ccpp-table-properties] + name = data + type = host + dependencies = +[ccpp-arg-table] + name = data + type = host +[checksum] + standard_name = workspace_checksum + long_name = sum of the suite-owned scratch workspace + units = 1 + dimensions = () + type = real + kind = kind_phys diff --git a/end-to-end-tests/suite_allocate/main.F90 b/end-to-end-tests/suite_allocate/main.F90 new file mode 100644 index 00000000..c2afd946 --- /dev/null +++ b/end-to-end-tests/suite_allocate/main.F90 @@ -0,0 +1,114 @@ +program test_suite_allocate + + use, intrinsic :: iso_fortran_env, only: output_unit, & + error_unit + + use ccpp_kinds, only: kind_phys + + use data, only: checksum + + use test_host_ccpp_cap, only: ccpp_register, & + ccpp_init, & + ccpp_physics_init, & + ccpp_physics_timestep_init, & + ccpp_physics_run, & + ccpp_physics_timestep_final, & + ccpp_physics_final, & + ccpp_final + + implicit none + + character(len=*), parameter :: ccpp_suite = 'suite_allocate_suite' + character(len=512) :: errmsg + integer :: errflg + ! Must match the value use_workspace sets in its timestep_init phase. + integer, parameter :: expected_size = 4 + real(kind=kind_phys) :: expected + real(kind=kind_phys), parameter :: tol = 1.0e-6_kind_phys + + ! use_workspace sets workspace_dimension = expected_size in timestep_init; + ! make_workspace fills work(i) = i in run, so the consumer's sum is N*(N+1)/2. + expected = real(expected_size * (expected_size + 1) / 2, kind_phys) + checksum = -1.0_kind_phys + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP register step ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_register(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + call check_err('ccpp_register', errflg, errmsg) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP init step (suite_data_init_fields runs; ! + ! it must SKIP the allocatable suite var) ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_init(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + call check_err('ccpp_init', errflg, errmsg) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics init / timestep init steps ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_init(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, & + suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + call check_err('ccpp_physics_init', errflg, errmsg) + + call ccpp_physics_timestep_init(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, & + suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + call check_err('ccpp_physics_timestep_init', errflg, errmsg) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics run step: producer allocates the ! + ! suite-owned workspace, consumer reduces it ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_run(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, & + suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + call check_err('ccpp_physics_run', errflg, errmsg) + + if (abs(checksum - expected) > tol) then + write(error_unit, '(a,f0.6,a,f0.6)') & + "Error after ccpp_physics_run: workspace_checksum=", checksum, & + " expected ", expected + stop 1 + end if + write(output_unit, '(a,f0.6)') & + "PASS: After ccpp_physics_run: suite-owned allocatable workspace summed to ", checksum + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP physics timestep final / final steps ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_physics_timestep_final(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, & + suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + call check_err('ccpp_physics_timestep_final', errflg, errmsg) + + call ccpp_physics_final(lb=1, ub=1, nthreads=1, nphys_threads=1, thread_num=1, & + suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + call check_err('ccpp_physics_final', errflg, errmsg) + + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ! CCPP finalize step (final_fields frees the ! + ! suite-owned allocatable var; guarded) ! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + call ccpp_final(suite_name=trim(ccpp_suite), errmsg=errmsg, errflg=errflg) + call check_err('ccpp_final', errflg, errmsg) + + write(output_unit, '(a)') "PASS: suite_allocate test completed" + +contains + + subroutine check_err(phase, errflg, errmsg) + character(len=*), intent(in) :: phase + integer, intent(in) :: errflg + character(len=*), intent(in) :: errmsg + if (errflg /= 0) then + write(error_unit, '(a)') "An error occurred in " // trim(phase) // ":" + write(error_unit, '(a)') trim(errmsg) + stop 1 + end if + end subroutine check_err + +end program test_suite_allocate diff --git a/end-to-end-tests/suite_allocate/main.meta b/end-to-end-tests/suite_allocate/main.meta new file mode 100644 index 00000000..b90de81e --- /dev/null +++ b/end-to-end-tests/suite_allocate/main.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = main + type = control + dependencies = + +[ccpp-arg-table] + name = main + type = control +[ suite_name ] + standard_name = suite_name + long_name = name of the CCPP suite to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ group_name ] + standard_name = group_name + long_name = name of the CCPP group to dispatch to + units = none + dimensions = () + type = character + kind = len=256 +[ lb ] + standard_name = horizontal_loop_begin + long_name = start of horizontal range for this phase + units = index + dimensions = () + type = integer +[ ub ] + standard_name = horizontal_loop_end + long_name = end of horizontal range for this phase + units = index + dimensions = () + type = integer +[ thread_num ] + standard_name = thread_number + long_name = current thread number + units = index + dimensions = () + type = integer +[ nthreads ] + standard_name = number_of_threads + long_name = total number of OpenMP threads + units = count + dimensions = () + type = integer +[ nphys_threads ] + standard_name = number_of_physics_threads + long_name = thread budget for physics-internal OpenMP + units = count + dimensions = () + type = integer +[ errmsg ] + standard_name = ccpp_error_message + long_name = error message for CCPP error handling + units = none + dimensions = () + type = character + kind = len=512 +[ errflg ] + standard_name = ccpp_error_code + long_name = error flag for CCPP error handling + units = 1 + dimensions = () + type = integer diff --git a/end-to-end-tests/suite_allocate/make_workspace.F90 b/end-to-end-tests/suite_allocate/make_workspace.F90 new file mode 100644 index 00000000..ab371995 --- /dev/null +++ b/end-to-end-tests/suite_allocate/make_workspace.F90 @@ -0,0 +1,41 @@ +!>\file make_workspace.F90 +!! Producer scheme: allocates and fills a suite-owned scratch workspace. +!! The workspace standard name is not provided by the host, so capgen-ng +!! promotes it to a suite-owned, scheme-allocated (allocatable=True) variable +!! stored in ccpp__data. suite_data_init_fields skips its allocation; +!! this scheme owns it; final_fields frees it. + +module make_workspace + + use ccpp_kinds, only: kind_phys + + implicit none + + private + public :: make_workspace_run + +contains + + !! \section arg_table_make_workspace_run Argument Table + !! \htmlinclude make_workspace_run.html + !! + subroutine make_workspace_run(nw, work, errmsg, errflg) + integer, intent(in) :: nw + real(kind=kind_phys), allocatable, intent(out) :: work(:) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + + integer :: i + + errmsg = '' + errflg = 0 + + ! intent(out) on an allocatable dummy auto-deallocates on entry, so this is + ! safe to call repeatedly (the persistent suite-data component is reset here). + allocate(work(nw)) + do i = 1, nw + work(i) = real(i, kind_phys) + end do + end subroutine make_workspace_run + +end module make_workspace diff --git a/end-to-end-tests/suite_allocate/make_workspace.meta b/end-to-end-tests/suite_allocate/make_workspace.meta new file mode 100644 index 00000000..39dde2a7 --- /dev/null +++ b/end-to-end-tests/suite_allocate/make_workspace.meta @@ -0,0 +1,39 @@ +[ccpp-table-properties] + name = make_workspace + type = scheme + dependencies = + +[ccpp-arg-table] + name = make_workspace_run + type = scheme +[nw] + standard_name = workspace_dimension + long_name = size of the scratch workspace + units = count + dimensions = () + type = integer + intent = in +[work] + standard_name = scratch_workspace_field + long_name = suite-owned scratch workspace allocated by the producing scheme + units = 1 + dimensions = (workspace_dimension) + type = real + kind = kind_phys + intent = out + allocatable = True +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/end-to-end-tests/suite_allocate/suite_allocate_suite.xml b/end-to-end-tests/suite_allocate/suite_allocate_suite.xml new file mode 100644 index 00000000..4f3bc223 --- /dev/null +++ b/end-to-end-tests/suite_allocate/suite_allocate_suite.xml @@ -0,0 +1,8 @@ + + + + + make_workspace + use_workspace + + diff --git a/end-to-end-tests/suite_allocate/use_workspace.F90 b/end-to-end-tests/suite_allocate/use_workspace.F90 new file mode 100644 index 00000000..25e2fe81 --- /dev/null +++ b/end-to-end-tests/suite_allocate/use_workspace.F90 @@ -0,0 +1,52 @@ +!>\file use_workspace.F90 +!! Consumer scheme: reads the suite-owned scratch workspace allocated by +!! make_workspace and reduces it into a host-owned scalar. Receiving the +!! suite-owned allocatable component through a plain (non-allocatable) dummy +!! exercises capgen-ng passing the whole allocated component to a consumer. + +module use_workspace + + use ccpp_kinds, only: kind_phys + + implicit none + + private + public :: use_workspace_timestep_init, use_workspace_run + +contains + + !! \section arg_table_use_workspace_timestep_init Argument Table + !! \htmlinclude use_workspace_timestep_init.html + !! + subroutine use_workspace_timestep_init(nw, errmsg, errflg) + integer, intent(out) :: nw + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + ! Own the suite workspace dimension here, in timestep_init -- a phase that + ! runs AFTER ccpp_init/suite_data_init_fields. make_workspace (listed earlier + ! but executed in the later run phase) allocates work(nw) using this value, + ! which is exactly why scratch_workspace_field must be allocatable=True: + ! init_fields cannot size it. + nw = 4 + end subroutine use_workspace_timestep_init + + !! \section arg_table_use_workspace_run Argument Table + !! \htmlinclude use_workspace_run.html + !! + subroutine use_workspace_run(work, checksum, errmsg, errflg) + real(kind=kind_phys), intent(in) :: work(:) + real(kind=kind_phys), intent(out) :: checksum + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errflg + + errmsg = '' + errflg = 0 + + checksum = sum(work) + end subroutine use_workspace_run + +end module use_workspace diff --git a/end-to-end-tests/suite_allocate/use_workspace.meta b/end-to-end-tests/suite_allocate/use_workspace.meta new file mode 100644 index 00000000..daa9270a --- /dev/null +++ b/end-to-end-tests/suite_allocate/use_workspace.meta @@ -0,0 +1,65 @@ +[ccpp-table-properties] + name = use_workspace + type = scheme + dependencies = + +[ccpp-arg-table] + name = use_workspace_timestep_init + type = scheme +[nw] + standard_name = workspace_dimension + long_name = size of the scratch workspace + units = count + dimensions = () + type = integer + intent = out +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out + +[ccpp-arg-table] + name = use_workspace_run + type = scheme +[work] + standard_name = scratch_workspace_field + long_name = suite-owned scratch workspace allocated by the producing scheme + units = 1 + dimensions = (workspace_dimension) + type = real + kind = kind_phys + intent = in +[checksum] + standard_name = workspace_checksum + long_name = sum of the suite-owned scratch workspace + units = 1 + dimensions = () + type = real + kind = kind_phys + intent = out +[errmsg] + standard_name = ccpp_error_message + long_name = error message for error handling in CCPP + units = none + dimensions = () + type = character + kind = len=* + intent = out +[errflg] + standard_name = ccpp_error_code + long_name = error code for error handling in CCPP + units = 1 + dimensions = () + type = integer + intent = out diff --git a/unit-tests/test_suite_data.py b/unit-tests/test_suite_data.py index 4fabcf7e..6c92cff4 100644 --- a/unit-tests/test_suite_data.py +++ b/unit-tests/test_suite_data.py @@ -6,7 +6,12 @@ import unittest from metadata.parse_tools import CCPPError -from generator.suite_data import _generate_suite_data, write_suite_data +from generator.suite_data import ( + _generate_suite_data, + _collect_dim_uses, + _dim_local_expr, + write_suite_data, +) from generator.suite_resolver import SuiteVar @@ -209,6 +214,41 @@ def test_no_ddt_no_use(self): self.assertIn('use ccpp_kinds, only: kind_phys', text) +class TestConstituentCountDim(unittest.TestCase): + """Suite-owned var dimensioned by ``number_of_ccpp_constituents``. + + The framework owns the extent, so ``init_fields`` must allocate the field + via the per-instance constituent object's ``num_layer_vars`` member and USE + the object's module (``ccpp_host_constituents``). Regression for the + CAM-SIMA se_cslam allocate path (Fix B). + """ + + def test_dim_local_expr_resolves_to_constituent_count(self): + self.assertEqual( + _dim_local_expr('number_of_ccpp_constituents', {}, {}), + 'ccpp_model_constituents_obj(i)%num_layer_vars', + ) + + def test_collect_dim_uses_adds_constituent_module(self): + sv = {'workspace': _make_sv( + 'workspace', 'work', dims=['number_of_ccpp_constituents'])} + uses = _collect_dim_uses(sv, {}) + self.assertEqual(uses.get('ccpp_host_constituents'), + ['ccpp_model_constituents_obj']) + + def test_init_fields_allocates_with_constituent_count(self): + sv = {'workspace': _make_sv( + 'workspace', 'work', dims=['number_of_ccpp_constituents'])} + text = '\n'.join(_generate_suite_data('cdim', sv, host_dict={})) + self.assertIn( + 'use ccpp_host_constituents, only: ccpp_model_constituents_obj', + text) + self.assertIn( + 'allocate(ccpp_suite_data(i)%work(' + 'ccpp_model_constituents_obj(i)%num_layer_vars))', + text) + + class TestWriteSuiteData(unittest.TestCase): def test_writes_file(self): diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index d8d0ac0b..10af3e07 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -25,6 +25,7 @@ _apply_transform_formula, _build_call_subscript, _build_merged_subscript, + _one_dim_part, _resolve_single_bound, _substitute_instance_idx, _translate_active_expr, @@ -2533,6 +2534,28 @@ def test_unknown_dim_falls_back_to_std_name(self): ) +class TestConstituentCountDimSubscript(unittest.TestCase): + """``number_of_ccpp_constituents`` as a *call subscript* axis. + + Any variable (host, suite-owned, or scheme) may be dimensioned by the + framework constituent count; ``_one_dim_part`` must emit a whole-axis + ``:`` for it -- not only framework-constituent args (which go through + ``_const_dim_part``). Regression for the CAM-SIMA se_cslam failure where + host vars (cflx/qbot/fracis) are dimensioned by number_of_ccpp_constituents. + """ + + def test_bare_count_is_whole_axis(self): + part, used = _one_dim_part('number_of_ccpp_constituents', 'run', {}) + self.assertEqual(part, ':') + self.assertEqual(used, set()) + + def test_explicit_lower_bound_count_is_whole_axis(self): + part, used = _one_dim_part( + 'ccpp_constant_one:number_of_ccpp_constituents', 'run', {}) + self.assertEqual(part, ':') + self.assertEqual(used, set()) + + class TestCollectKindsUsed(unittest.TestCase): """``_collect_kinds_used`` must collect only kind *symbols* — integer literals and ``len=...`` specifiers are NOT module symbols and must From 04acc29bccf7ce0db349af89f97ec54f614ba2aa Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 5 Jun 2026 09:57:41 -0600 Subject: [PATCH 59/74] Bug fixes for constituents dimensions and variables being used in other schemes --- capgen-ng/generator/suite_resolver.py | 45 ++++- capgen-ng/metadata/variable_resolver.py | 33 ++++ end-to-end-tests/constituents_dim/README.md | 38 ++-- .../constituents_dim/const_dim_consumer.F90 | 18 +- .../constituents_dim/const_dim_consumer.meta | 16 ++ .../constituents_dim/const_dim_producer.F90 | 22 ++- .../constituents_dim/const_dim_producer.meta | 18 ++ unit-tests/test_suite_resolver.py | 166 +++++++++++++++++- 8 files changed, 321 insertions(+), 35 deletions(-) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 3da74be0..8313cb5e 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -1455,6 +1455,7 @@ def _resolve_one_arg( used_local_names: Set[str], suite_name: str = '', loop_context: Optional[List[Tuple[str, Optional[str]]]] = None, + const_stds: Set[str] = frozenset(), ) -> ResolvedArg: """Resolve one scheme argument against host/control/suite dictionaries. @@ -1618,6 +1619,7 @@ def _resolve_one_arg( # Returns ``None`` if the arg is not constituent-related. const_arg = _resolve_constituent_arg( scheme_var, phase, host_dict, suite_vars, scheme_name, suite_name, + const_stds=const_stds, ) if const_arg is not None: return const_arg @@ -2026,6 +2028,7 @@ def _resolve_constituent_arg( suite_vars: Dict[str, 'SuiteVar'], scheme_name: str, suite_name: str, + const_stds: Set[str] = frozenset(), ) -> Optional[ResolvedArg]: """Synthesise a ``source='constituent'`` ResolvedArg, or return ``None``. @@ -2077,6 +2080,26 @@ def _resolve_constituent_arg( is_index_name = std_name.startswith(_INDEX_PREFIX) is_framework_name = std_name in _FRAMEWORK_CONST_STDS or is_index_name + # Rule (b): an UNFLAGGED consumer of a name that some scheme declares as a + # constituent -- a base constituent (``advected``, read via vars_layer) or a + # constituent tendency (``constituent`` on a ``tendency_of_*`` producer, read + # via vars_layer_tend) -- resolves to the SAME framework column the + # producer/registration backs. Consumers must not re-flag it: whether a + # given standard name is a constituent or an ordinary variable is the host's + # decision (CAM-SIMA vs CCPP-SCM), so we infer it from the + # scheme-metadata-wide set instead of the consumer's own metadata. + # Host/earlier-suite provision WINS: if this host declares the name, or a + # prior scheme already produced it as an ordinary variable, defer to normal + # resolution (a genuine constituent is in neither host_dict nor suite_vars). + is_known_constituent = std_name in const_stds + inferred_constituent_consumer = ( + is_known_constituent + and not scheme_var.is_constituent + and intent in ('in', 'inout') + and not (host_dict and std_name in host_dict) + and std_name not in suite_vars + ) + # Host/suite provides it -> not a framework auto-provision. Defer to # normal host/suite resolution when: # @@ -2187,7 +2210,7 @@ def _common_kwargs(base_expr, subscript, call_expr, used_const_dim_std=used_const_dim_std, )) - if not scheme_var.is_constituent: + if not (scheme_var.is_constituent or inferred_constituent_consumer): return None # not constituent-related # ---- Provider gate (mirrors original-capgen ConstituentVarDict.find_variable) @@ -2231,14 +2254,16 @@ def _common_kwargs(base_expr, subscript, call_expr, member = 'vars_layer_tend' else: # in / inout if is_tendency_name: - raise CCPPError( - "Constituent tendency arg '{}' (standard_name='{}', " - "scheme='{}', phase='{}') must be declared with intent=out; " - "physics phases only produce tendencies, never consume " - "them.".format(local, std_name, scheme_name, phase) - ) - base_std = std_name - member = 'vars_layer' + # Consumer of a constituent tendency (rule b): read the SAME column + # the producer wrote -- ccpp_constituent_tendencies(, + # index_of_). Reaching here means it is a recognised + # constituent tendency (flagged, or inferred via const_stds) + # that neither the host nor an earlier suite var provides. + base_std = std_name[len(_TEND_PREFIX):] + member = 'vars_layer_tend' + else: + base_std = std_name + member = 'vars_layer' leading_sub, used_host_std = _build_call_subscript( scheme_dims, phase, host_dict, suite_vars=suite_vars, @@ -2772,6 +2797,7 @@ def _resolve_one_call( vars_list = scheme_store.variables_for(scheme_name, phase) if vars_list is None: return None + const_stds = scheme_store.constituent_stdnames() resolved_call = ResolvedCall( scheme_name=scheme_name, phase=phase, scheme_module=scheme_store.module_for(scheme_name), @@ -2780,6 +2806,7 @@ def _resolve_one_call( arg = _resolve_one_arg( scheme_var, phase, host_dict, suite_vars, scheme_name, used_local_names, suite_name=suite_name, loop_context=loop_context, + const_stds=const_stds, ) resolved_call.args.append(arg) return resolved_call diff --git a/capgen-ng/metadata/variable_resolver.py b/capgen-ng/metadata/variable_resolver.py index dcde25e4..77fd7960 100644 --- a/capgen-ng/metadata/variable_resolver.py +++ b/capgen-ng/metadata/variable_resolver.py @@ -782,6 +782,9 @@ def __init__(self) -> None: # duplicate-phase error path so the message can name both the # original registration site and the duplicate. self._source_paths: Dict[str, Dict[str, str]] = {} + # Memoised set of constituent standard names (base + tendency; + # see constituent_stdnames); None until first computed. + self._const_stds: Optional[frozenset] = None @classmethod def build_from(cls, scheme_tables: List[MetadataTable]) -> 'SchemeStore': @@ -845,6 +848,36 @@ def scheme_names(self) -> List[str]: """Return sorted list of all known scheme names.""" return sorted(self._data.keys()) + def constituent_stdnames(self) -> frozenset: + """Standard names that some scheme declares as a CONSTITUENT. + + A variable is a constituent when any scheme argument flags it + ``is_constituent`` (``advected`` / ``constituent`` / ``molar_mass``): + + * a base constituent -- e.g. a mixing ratio flagged ``advected = true`` + (read via ``vars_layer``), or + * a constituent tendency -- a ``tendency_of_*`` arg flagged + ``constituent = true`` (read via ``vars_layer_tend``). + + A *consumer* of either (a scheme reading a mixing ratio, or a diagnostics + scheme reading a tendency) must NOT re-flag it: whether a given standard + name is a constituent or an ordinary variable is the host's decision + (CAM-SIMA registers it as a constituent; CCPP-SCM may expose the same + name as an ordinary host variable). The suite resolver therefore infers + constituent-ness from this scheme-metadata-wide set rather than from the + consumer's own metadata. Memoised; covers every loaded scheme because + standard-name semantics are global. + """ + if self._const_stds is None: + result = set() + for phases in self._data.values(): + for varlist in phases.values(): + for var in varlist: + if getattr(var, 'is_constituent', False): + result.add(var.standard_name) + self._const_stds = frozenset(result) + return self._const_stds + def module_for(self, name: str) -> str: """Return the Fortran module name that exports scheme *name*. diff --git a/end-to-end-tests/constituents_dim/README.md b/end-to-end-tests/constituents_dim/README.md index 9f4cd8ff..b02afc7c 100644 --- a/end-to-end-tests/constituents_dim/README.md +++ b/end-to-end-tests/constituents_dim/README.md @@ -2,27 +2,45 @@ Covers variables dimensioned by the framework constituent count number_of_ccpp_constituents (which the host never declares as a scalar — the -framework owns it). register_consts registers 3 dynamic constituents, so the -count is 3 for the rest of the suite. Three cases, each a distinct capgen path: +framework owns it), plus consuming constituents WITHOUT a flag (rule b). +register_consts registers 3 dynamic constituents, so the count is 3 for the rest +of the suite. + +Count-as-dimension cases (each a distinct capgen path): - Case 1 — host var dimensioned by the count. surface_upward_test_constituent_flux (horizontal_dimension, number_of_ccpp_constituents) is host-owned; the host sizes it to the runtime count and capgen passes the whole constituent axis as : to const_dim_producer. - Case 2a — non-allocatable suite var, framework allocates in init_fields. - test_constituent_workspace(number_of_ccpp_constituents) is suite-owned and - not allocatable, so suite_data_init_fields allocates it via + test_constituent_workspace(number_of_ccpp_constituents) is suite-owned and not + allocatable, so suite_data_init_fields allocates it via ccpp_model_constituents_obj(i)%num_layer_vars. - Case 2b — allocatable suite var, the scheme allocates in _run. test_allocatable_constituent_workspace(number_of_ccpp_constituents) is - allocatable = True; init_fields skips it and const_dim_producer allocates - it using the count received as a scalar (number_of_ccpp_constituents --> - ccpp_model_constituents_obj(inst)%num_layer_vars). Uses the existing - suite-owned-allocatable path. final_fields frees both suite workspaces. + allocatable = True; init_fields skips it and const_dim_producer allocates it + using the count received as a scalar (number_of_ccpp_constituents --> + ccpp_model_constituents_obj(inst)%num_layer_vars). final_fields frees both + suite workspaces. + +Rule (b) cases — consuming a constituent without re-flagging it. const_dim_producer +flags and writes them; const_dim_consumer reads them with NO constituent flag, and +capgen infers them from the producer's flags: + +- Base constituent: qbase (test_constituent_one) is flagged advected = True on the + producer (intent=inout) and read unflagged by the consumer; both resolve to + ccpp_model_constituents_obj(inst)%vars_layer(:, index_of_test_constituent_one). +- Constituent tendency: qtend (tendency_of_test_constituent_one) is flagged + constituent = True on the producer (intent=out) and read unflagged by the + consumer; both resolve to ...%vars_layer_tend(:, index_of_test_constituent_one). + +Whether a name is a constituent is the host's decision (CAM-SIMA vs CCPP-SCM), so +the consumer never carries the flag; capgen infers it from the scheme-wide flag +set, and host/earlier-suite provision wins. The producer fills the workspaces and verifies Case 1; the consumer verifies -Cases 2a/2b. Any mismatch sets errcode, failing the run. Built with --fcheck=all, so allocation/teardown errors fail the test. +Cases 2a/2b and the rule (b) reads. Any mismatch sets errcode, failing the run. +Built with -fcheck=all, so allocation/teardown errors fail the test. The 2a vs 2b contrast is the same ownership rule as suite_allocate: a non-allocatable suite var is framework-owned (allocated in init_fields); an diff --git a/end-to-end-tests/constituents_dim/const_dim_consumer.F90 b/end-to-end-tests/constituents_dim/const_dim_consumer.F90 index f81690b6..d96b6444 100644 --- a/end-to-end-tests/constituents_dim/const_dim_consumer.F90 +++ b/end-to-end-tests/constituents_dim/const_dim_consumer.F90 @@ -19,9 +19,11 @@ module const_dim_consumer !! \section arg_table_const_dim_consumer_run Argument Table !! \htmlinclude const_dim_consumer_run.html !! - subroutine const_dim_consumer_run(cwork, awork, errmsg, errcode) + subroutine const_dim_consumer_run(cwork, awork, qbase, qtend, errmsg, errcode) real(kind=kind_phys), intent(in) :: cwork(:) real(kind=kind_phys), intent(in) :: awork(:) + real(kind=kind_phys), intent(in) :: qbase(:, :) + real(kind=kind_phys), intent(in) :: qtend(:, :) character(len=*), intent(out) :: errmsg integer, intent(out) :: errcode @@ -47,6 +49,20 @@ subroutine const_dim_consumer_run(cwork, awork, errmsg, errcode) return end if end do + + ! Rule (b): qbase (base constituent) and qtend (constituent tendency) carry + ! NO constituent flag here; capgen infers them from the producer's flags and + ! reads the same framework columns (vars_layer / vars_layer_tend). + if (any(qbase /= 42.0_kind_phys)) then + errcode = 1 + errmsg = 'rule b: unflagged base-constituent consumer read the wrong value' + return + end if + if (any(qtend /= 7.0_kind_phys)) then + errcode = 1 + errmsg = 'rule b: unflagged constituent-tendency consumer read the wrong value' + return + end if end subroutine const_dim_consumer_run end module const_dim_consumer diff --git a/end-to-end-tests/constituents_dim/const_dim_consumer.meta b/end-to-end-tests/constituents_dim/const_dim_consumer.meta index fe52cb0b..ef2622b8 100644 --- a/end-to-end-tests/constituents_dim/const_dim_consumer.meta +++ b/end-to-end-tests/constituents_dim/const_dim_consumer.meta @@ -22,6 +22,22 @@ type = real kind = kind_phys intent = in +[qbase] + standard_name = test_constituent_one + long_name = base constituent read WITHOUT a constituent flag (rule b consumer) + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = in +[qtend] + standard_name = tendency_of_test_constituent_one + long_name = constituent tendency read WITHOUT a constituent flag (rule b consumer) + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = in [errmsg] standard_name = ccpp_error_message long_name = error message for error handling in CCPP diff --git a/end-to-end-tests/constituents_dim/const_dim_producer.F90 b/end-to-end-tests/constituents_dim/const_dim_producer.F90 index c248f7e3..60a8828c 100644 --- a/end-to-end-tests/constituents_dim/const_dim_producer.F90 +++ b/end-to-end-tests/constituents_dim/const_dim_producer.F90 @@ -22,13 +22,15 @@ module const_dim_producer !! \htmlinclude const_dim_producer_run.html !! subroutine const_dim_producer_run(coupler_flux, cwork, n_const, awork, & - errmsg, errcode) - real(kind=kind_phys), intent(in) :: coupler_flux(:, :) - real(kind=kind_phys), intent(out) :: cwork(:) - integer, intent(in) :: n_const - real(kind=kind_phys), allocatable, intent(out) :: awork(:) - character(len=*), intent(out) :: errmsg - integer, intent(out) :: errcode + qbase, qtend, errmsg, errcode) + real(kind=kind_phys), intent(in) :: coupler_flux(:, :) + real(kind=kind_phys), intent(out) :: cwork(:) + integer, intent(in) :: n_const + real(kind=kind_phys), allocatable, intent(out) :: awork(:) + real(kind=kind_phys), intent(inout) :: qbase(:, :) + real(kind=kind_phys), intent(out) :: qtend(:, :) + character(len=*), intent(out) :: errmsg + integer, intent(out) :: errcode integer :: m, i @@ -59,6 +61,12 @@ subroutine const_dim_producer_run(coupler_flux, cwork, n_const, awork, & do m = 1, n_const awork(m) = real(100 * m, kind_phys) end do + + ! Rule (b), producer side: flag a base constituent (advected) and a + ! constituent tendency (constituent=true) and write known values into their + ! framework columns. const_dim_consumer reads both with NO flag (inference). + qbase = 42.0_kind_phys + qtend = 7.0_kind_phys end subroutine const_dim_producer_run end module const_dim_producer diff --git a/end-to-end-tests/constituents_dim/const_dim_producer.meta b/end-to-end-tests/constituents_dim/const_dim_producer.meta index 09f74c80..f56f3a1c 100644 --- a/end-to-end-tests/constituents_dim/const_dim_producer.meta +++ b/end-to-end-tests/constituents_dim/const_dim_producer.meta @@ -38,6 +38,24 @@ kind = kind_phys intent = out allocatable = True +[qbase] + standard_name = test_constituent_one + long_name = base constituent modified in place (rule b producer; advected) + units = kg kg-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = inout + advected = True +[qtend] + standard_name = tendency_of_test_constituent_one + long_name = constituent tendency produced into the framework tendency array + units = kg kg-1 s-1 + dimensions = (horizontal_dimension, vertical_layer_dimension) + type = real + kind = kind_phys + intent = out + constituent = True [errmsg] standard_name = ccpp_error_message long_name = error message for error handling in CCPP diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 10af3e07..24cdc4e4 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -4522,16 +4522,166 @@ def test_base_constituent_intent_out_raises(self): self._resolve('air_temperature_extra', 'out', 'advected') self.assertIn("'tendency_of_'", str(ctx.exception)) - def test_tendency_intent_in_raises(self): - # constituent flag + intent=in on a tendency_of_* std name → reject. - with self.assertRaises(CCPPError) as ctx: - self._resolve('tendency_of_air_temperature', 'in', 'constituent') - self.assertIn('intent=out', str(ctx.exception)) + def test_tendency_intent_in_resolves_to_tendency_array(self): + # Rule (b): a constituent tendency may be CONSUMED (e.g. a diagnostics + # scheme). intent=in on a tendency_of_* std name now reads the framework + # tendency column instead of erroring. + res = self._resolve('tendency_of_air_temperature', 'in', 'constituent') + arg = list(iter_phase_calls(res.groups[0].phase_calls['run']))[0].args[0] + self.assertEqual(arg.source, 'constituent') + self.assertIn('vars_layer_tend', arg.call_expr) + + def test_tendency_intent_inout_resolves_to_tendency_array(self): + res = self._resolve('tendency_of_air_temperature', 'inout', 'constituent') + arg = list(iter_phase_calls(res.groups[0].phase_calls['run']))[0].args[0] + self.assertEqual(arg.source, 'constituent') + self.assertIn('vars_layer_tend', arg.call_expr) + + +class TestConstituentConsumerInferenceRuleB(unittest.TestCase): + """Rule (b): an UNFLAGGED scheme may CONSUME a constituent that another + scheme flags -- a base constituent (``advected``) read via ``vars_layer``, or + a constituent tendency (``constituent`` on a ``tendency_of_*`` producer) read + via ``vars_layer_tend``. The consumer infers it from the scheme-wide flag + set and never re-flags. Whether a name is a constituent is the host's call, + so a name NO scheme flags stays an ordinary variable. + """ + + _PRODUCER_CONSUMER = ( + '[ccpp-table-properties]\n' + ' name = tend_producer\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = tend_producer_run\n' + ' type = scheme\n' + '[ qt ]\n' + ' standard_name = tendency_of_air_temperature\n' + ' units = K s-1\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = out\n' + ' constituent = .true.\n' + '\n' + '[ccpp-table-properties]\n' + ' name = tend_consumer\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = tend_consumer_run\n' + ' type = scheme\n' + '[ qt ]\n' + ' standard_name = tendency_of_air_temperature\n' + ' units = K s-1\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = in\n' + ) + + # Consumer only (no scheme flags the tendency as a constituent). + _CONSUMER_ONLY = ( + '[ccpp-table-properties]\n' + ' name = tend_consumer\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = tend_consumer_run\n' + ' type = scheme\n' + '[ qt ]\n' + ' standard_name = tendency_of_air_temperature\n' + ' units = K s-1\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = in\n' + ) + + # A base constituent (advected) flagged by one scheme and consumed unflagged + # by another; nothing else provides it. + _BASE_FLAGGED_AND_UNFLAGGED = ( + '[ccpp-table-properties]\n' + ' name = base_flagged\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = base_flagged_run\n' + ' type = scheme\n' + '[ q ]\n' + ' standard_name = made_up_dry_mixing_ratio\n' + ' units = kg kg-1\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = in\n' + ' advected = .true.\n' + '\n' + '[ccpp-table-properties]\n' + ' name = base_unflagged\n' + ' type = scheme\n' + '[ccpp-arg-table]\n' + ' name = base_unflagged_run\n' + ' type = scheme\n' + '[ q ]\n' + ' standard_name = made_up_dry_mixing_ratio\n' + ' units = kg kg-1\n' + ' dimensions = (horizontal_dimension, vertical_layer_dimension)\n' + ' type = real | kind = kind_phys\n' + ' intent = in\n' + ) + + def _resolve(self, meta, schemes): + hd = _load_constituent_host_dict() + with tempfile.NamedTemporaryFile('w', suffix='.meta', delete=False) as fh: + fh.write(meta) + path = fh.name + try: + store = SchemeStore.build_from(parse_metadata_file(path)) + finally: + os.unlink(path) + xml = ('\n' + '\n' + ' \n' + + ''.join(' {}\n'.format(s) for s in schemes) + + ' \n\n') + with tempfile.TemporaryDirectory() as tmpdir: + xpath = os.path.join(tmpdir, 's.xml') + with open(xpath, 'w') as f: + f.write(xml) + from generator.suite_xml import parse_suite_xml + import logging + suite = parse_suite_xml(xpath, tmpdir, logging.getLogger('t'), + skip_validation=True) + return resolve_suite(suite, store, hd) - def test_tendency_intent_inout_raises(self): + def test_unflagged_consumer_reads_tendency_array(self): + res = self._resolve(self._PRODUCER_CONSUMER, + ['tend_producer', 'tend_consumer']) + calls = list(iter_phase_calls(res.groups[0].phase_calls['run'])) + producer_arg = calls[0].args[0] + consumer_arg = calls[1].args[0] + self.assertEqual(producer_arg.source, 'constituent') + self.assertIn('vars_layer_tend', producer_arg.call_expr) + # The consumer carries NO constituent flag, yet routes to the SAME + # framework tendency column (rule b inference from the producer). + self.assertEqual(consumer_arg.source, 'constituent') + self.assertIn('vars_layer_tend', consumer_arg.call_expr) + + def test_unflagged_tendency_without_producer_is_not_constituent(self): + # Nothing flags the tendency, so it stays an ordinary variable: an + # unprovided consumer is a normal "not provided" error, never silently + # routed to the constituent tendency array. with self.assertRaises(CCPPError) as ctx: - self._resolve('tendency_of_air_temperature', 'inout', 'constituent') - self.assertIn('intent=out', str(ctx.exception)) + self._resolve(self._CONSUMER_ONLY, ['tend_consumer']) + self.assertIn('not provided', str(ctx.exception)) + + def test_unflagged_base_consumer_reads_vars_layer(self): + # Symmetric to the tendency case: a base constituent flagged advected by + # one scheme is read by an unflagged consumer via the SAME base column + # (vars_layer, NOT the tendency array). + res = self._resolve(self._BASE_FLAGGED_AND_UNFLAGGED, + ['base_flagged', 'base_unflagged']) + calls = list(iter_phase_calls(res.groups[0].phase_calls['run'])) + flagged = calls[0].args[0] + unflagged = calls[1].args[0] + self.assertEqual(flagged.source, 'constituent') + self.assertIn('%vars_layer(', flagged.call_expr) + self.assertEqual(unflagged.source, 'constituent') + self.assertIn('%vars_layer(', unflagged.call_expr) + self.assertNotIn('vars_layer_tend', unflagged.call_expr) class TestHostDeclaredIndexOfWinsOverConstituents(unittest.TestCase): From aaccbffc1fbe264f0de0b36ccb4242c3fb24c4b2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 5 Jun 2026 13:37:27 -0600 Subject: [PATCH 60/74] Update docs --- doc/briefing.md | 51 ++++++++----- doc/briefing_pm.md | 40 +++++++---- doc/constituents.md | 36 ++++++++-- doc/constituents_overhaul.md | 90 ++++++++++++++++++++--- doc/migration.md | 134 ++++++++++++++++++++++++++++++++--- 5 files changed, 292 insertions(+), 59 deletions(-) diff --git a/doc/briefing.md b/doc/briefing.md index 62132c8d..fe7a307a 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -1,6 +1,6 @@ # capgen-ng — Briefing for CCPP Framework Developers & Power Users -*Prepared for the 2026-05-14 walk-through; last revised 2026-06-01. +*Prepared for the 2026-05-14 walk-through; last revised 2026-06-05. Companion document to `doc/migration.md` (the detailed migration guide) and `doc/redesign_prompt.md` (the implementation spec).* @@ -244,9 +244,9 @@ control-variable arguments to the public entry points. meeting. Pieces involved: framework setter additions (`set_advected`, `set_diagnostic_name`, `set_default_value`), `is_match` relaxation, Class A vs Class B property classification. -- **Validator host-metadata check.** `ccpp_validator.py` currently - validates scheme metadata only; host-metadata-vs-Fortran is on - hold until the e2e test suite settles. +- ~~**Validator host-metadata check.**~~ **Landed 2026-06-01**: + `ccpp_validator.py --host-files` validates `type = host` and + `type = ddt` tables against the Fortran (`doc/migration.md` §7.4). - **Codegen-time scheme-registration cross-check.** Today's registration check is at runtime (`ccpp_initialize_constituents`). Stronger options: new metadata @@ -356,14 +356,16 @@ don't rebuild downstream objects unless something actually moved. ## 10. Where things stand right now -- **Unit tests**: 1426 passing on `feature/capgen-ng` (1438 with - doctests; as of 2026-06-01). -- **End-to-end tests passing** (10): `advection`, - `advection_auto_clone`, `capgen_ng`, `chunked_data`, `ddthost`, - `instances`, `instances_advection`, `nested_suite`, `opt_arg`, - `var_compat`. `advection_auto_clone` is the newest — a port of - CAM-SIMA's `advection_test` exercising the auto-clone legacy - registration path under `--legacy-auto-clone-constituents`. +- **Unit tests**: 1516 passing on `feature/capgen-ng` (as of + 2026-06-05). +- **End-to-end tests passing** (12): `advection`, + `advection_auto_clone`, `capgen_ng`, `chunked_data`, + `constituents_dim`, `ddthost`, `instances`, `instances_advection`, + `nested_suite`, `opt_arg`, `suite_allocate`, `var_compat`. The two + newest — `constituents_dim` (a variable dimensioned by + `number_of_ccpp_constituents`) and `suite_allocate` (suite-owned + allocatable interstitials sized by a scheme-written dimension) — were + added while hardening the CAM-SIMA HPC build. - **Code size**: ~17.8k LOC of Python under `capgen-ng/` (includes docstrings, inline comments, and the three transient shim modules) + ~18k LOC of unit/doctest under `unit-tests/`. Still procedural, @@ -412,12 +414,25 @@ don't rebuild downstream objects unless something actually moved. - **UFS Weather Model**: not yet attempted; SCM is the proving ground first. An anticipated complication is the "fast physics" called directly from the FV3 dynamical core as a separate group. -- **CAM-SIMA**: not yet reconnected; pending the constituents - overhaul decision and CAM-SIMA developer availability. Note that - `--legacy-auto-clone-constituents` is the no-decision-needed bridge - that lets capgen-ng accept CAM-SIMA atmospheric_physics metadata - as-is — ~16 of the ~20 schemes that touch constituents rely on the - auto-clone path today. +- **CAM-SIMA**: **reconnected (2026-06-03 → 06-05).** capgen-ng now + drives the real CAM-SIMA build on Derecho via a thin compatibility + layer (`cime_config/capgen_compat/`, in the CAM-SIMA tree) that + re-implements original ccpp-capgen's Python API surface + (`cap_database`, `host_model_dict`, `call_list`, the per-variable + `Var` accessors) on top of capgen-ng's `datatable.xml` + + `ResolvedArg` / `HostVarEntry`. CAM-SIMA's `cam_autogen.py`, + `generate_registry_data.py`, and `write_init_files.py` are unchanged. + Three cases build **and run to completion** on Derecho (gnu): + `kessler`, `rrtmgp`, and `se_cslam` / CSLAM (the FCAM7 `cam7` suite — + the full convection + stratiform + radiation + gravity-wave physics). + `--legacy-auto-clone-constituents` is still the no-decision-needed + bridge for the ~16 schemes that rely on auto-clone. Bring-up this + week produced three reusable lessons baked into the docs: the + consume-without-re-flagging rule (`doc/migration.md` §6.5), the + adapter must key constituent handling on `ResolvedArg.source` + (`doc/constituents_overhaul.md` §4.15), and `module_name` overrides + are required wherever a Fortran module name differs from its + `.meta` table name (`doc/migration.md` §3.3). --- diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md index 06101d20..3e989f19 100644 --- a/doc/briefing_pm.md +++ b/doc/briefing_pm.md @@ -7,7 +7,7 @@ program managers; it summarises the case for `capgen-ng` in terms of product risk, schedule, and cross-organization impact rather than implementation detail.* -*Last revised: 2026-06-01.* +*Last revised: 2026-06-05.* --- @@ -217,10 +217,12 @@ generator itself sits at ~17.8k lines. The "who can fix this" pool is closer to "anyone with framework context". capgen-ng comes with ~1.4k docstring + unit tests (~18k lines of test code), plus an end-to-end test suite of -10 fixtures that covers all of prebuild's and capgen's existing +12 fixtures that covers all of prebuild's and capgen's existing end-to-end tests and adds new ones for multi-instance + constituents -(`instances_advection`) and the auto-clone-constituents shim -(`advection_auto_clone`). Including these tests and the rich inline +(`instances_advection`), the auto-clone-constituents shim +(`advection_auto_clone`), constituent-count dimensions +(`constituents_dim`), and suite-owned allocatable interstitials +(`suite_allocate`). Including these tests and the rich inline comments puts capgen-ng's full tree on the same order of magnitude as capgen — about half of which is test coverage and human-readable prose, not load-bearing logic. @@ -264,15 +266,16 @@ Features that exist only in capgen-ng (some exist in prebuild): --- -## 6. Where things stand right now (2026-06-01) +## 6. Where things stand right now (2026-06-05) -- **Unit tests**: 1426 passing (1438 with doctests). No known failures. -- **End-to-end tests**: 10 passing — `advection`, - `advection_auto_clone` (new 2026-05-21; CAM-SIMA advection_test - port exercising the auto-clone shim), `capgen_ng`, `chunked_data`, +- **Unit tests**: 1516 passing. No known failures. +- **End-to-end tests**: 12 passing — `advection`, + `advection_auto_clone` (CAM-SIMA advection_test port exercising the + auto-clone shim), `capgen_ng`, `chunked_data`, `constituents_dim`, `ddthost`, `instances`, `instances_advection` (multi-instance + constituents), `nested_suite`, `opt_arg`, - `var_compat`. + `suite_allocate`, `var_compat`. The two newest (`constituents_dim`, + `suite_allocate`) were added while hardening the CAM-SIMA HPC build. - **Code size**: ~17.8k lines of Python under `capgen-ng/` including inline comments and the three transient shim modules; ~18k lines of unit/doctest under `unit-tests/`. Still procedural; still flat @@ -302,11 +305,18 @@ Features that exist only in capgen-ng (some exist in prebuild): - **UFS Weather Model**: not yet attempted; SCM is the proving ground first. Expecting updates due to the "fast physics" called directly from the FV3 dynamical core as separate group. -- **CAM-SIMA**: not yet re-connected; the availability of CAM-SIMA - developers is now the primary gating item. The auto-clone shim - removes the metadata-edit blocker; the constituent overhaul - decision (see §7) is no longer on the critical path for getting - CAM-SIMA built. +- **CAM-SIMA**: **re-connected (2026-06-03 → 06-05).** capgen-ng now + drives the production CAM-SIMA build on the Derecho supercomputer + through a small compatibility layer that lets CAM-SIMA's existing + build scripts call capgen-ng without being rewritten. Three + configurations build **and run to completion**: `kessler`, `rrtmgp`, + and `se_cslam`/CSLAM — the last being the full CAM7 physics suite + (deep + shallow convection, stratiform microphysics, RRTMGP + radiation, gravity-wave drag) on a cubed-sphere/CSLAM-advection + configuration. This is the first time the redesigned generator has + produced a complete, running CAM-SIMA model. The constituent + overhaul decision (see §7) remains a separate track and was not on + the critical path for this milestone. --- diff --git a/doc/constituents.md b/doc/constituents.md index 0dc6e46f..b888fb6e 100644 --- a/doc/constituents.md +++ b/doc/constituents.md @@ -122,6 +122,24 @@ The resolver translates this scheme arg to in the generated group cap. No host metadata declaration is needed for the variable. +**Consumers need not re-flag (rule b, 2026-06-05).** Whether a standard +name is a constituent or an ordinary variable is the **host's** decision +(CAM-SIMA exposes water vapor as a constituent; CCPP-SCM may expose the +same name as an ordinary host variable), so a scheme that only **reads** +a constituent — the base species, or a `tendency_of_` — does **not** +repeat the `advected` / `constituent` flag. capgen-ng infers +constituent-ness for an unflagged `intent=in/inout` consumer from the +scheme-metadata-wide set of names *some* scheme flags +(`VariableResolver.constituent_stdnames()`): an unflagged read of the +base resolves to `%vars_layer(...)`, an unflagged read of +`tendency_of_` to `%vars_layer_tend(..., index_of_)` — the same +column a tendency producer (Rule 3) wrote. **Host / earlier-suite +provision wins**: if the host declares the name, or an earlier scheme +already produced it as an ordinary variable, normal host/suite +resolution takes over. (This is what lets the CAM-SIMA `cam7` +`sima_diagnostics` schemes read `tendency_of_water_vapor_…` that the +convection schemes produce.) + ### Rule 3 — Produce a tendency (any physics phase) A scheme that writes a constituent tendency declares the variable with @@ -146,12 +164,18 @@ same name. ### Rule 4 — Mismatched combinations are hard errors -Two combinations are explicitly rejected by the resolver at code-gen time: +One combination is rejected by the resolver at code-gen time: | Mismatch | Error | |---|---| | `is_constituent=True` + `intent=out` + std_name does NOT start with `tendency_of_` | *"Physics phases may only produce constituent tendencies; new base constituents must be declared via a `ccpp_constituent_properties_t` argument in a register-phase scheme."* | -| `is_constituent=True` + `intent in (in, inout)` + std_name starts with `tendency_of_` | *"Constituent tendency arg must be declared with intent=out; physics phases only produce tendencies, never consume them."* | + +> **Changed 2026-06-05:** consuming a constituent **tendency** +> (`intent=in/inout` on a `tendency_of_*` standard name) is **no longer** +> an error. It resolves to `%vars_layer_tend(..., index_of_)` — the +> same column a tendency producer writes — so a diagnostics scheme can +> read a tendency another scheme produced. See "Consumers need not +> re-flag (rule b)" under Rule 2. ### Direct framework-array access @@ -810,12 +834,14 @@ This means: ### Forbidden patterns recap -These are rejected at code-gen time (Rule 4 of [§2](#2-the-four-rules-scheme-author-conventions)): +This is rejected at code-gen time (Rule 4 of [§2](#2-the-four-rules-scheme-author-conventions)): - `is_constituent + intent=out + non-tendency std_name` — physics phases may only produce tendencies, not new base constituents. -- `is_constituent + intent=in/inout + tendency_of_*` — tendencies are - write-only. + +(As of 2026-06-05, `intent=in/inout + tendency_of_*` is **allowed** — a +constituent tendency may be *consumed*, resolving to `%vars_layer_tend`. +Only *producing* a tendency uses `intent=out`.) ### Subscript indices in sliced local_names must be standard names diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index 97cc3d75..730a1322 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -2,7 +2,7 @@ **Authors:** Dom Heinzeller (lead), Claude (assistant) **Date drafted:** 2026-05-12 -**Last revised:** 2026-05-18 +**Last revised:** 2026-06-05 **Intended audience:** CCPP framework team, CAM-SIMA team **Status:** Discussion document — no decisions are final. Proposals A/B/C below remain pending the upcoming meeting; the bug fix from @@ -11,7 +11,13 @@ internal cleanup from Proposal B (§4.8) have landed; the missing setters from Proposal A and the `is_match` relaxation from Proposal B have not. Independent of A/B/C, the per-suite dynamic_constituents buffer was made per-instance on 2026-05-18 to fix a multi-instance -mutation conflict — see §4.13. +mutation conflict — see §4.13. Since 2026-06-03 capgen-ng drives the +real CAM-SIMA build (via the `cime_config/capgen_compat/` facade): the +`kessler`, `rrtmgp`, and `se_cslam`/CSLAM (FCAM7 `cam7`) cases all build +and run on Derecho. That integration added the **rule-b** consumer path +(read a constituent or `tendency_of_` without re-flagging — §2.2.3) +and surfaced the host-adapter fix in §4.15; neither changes the A/B/C +decision surface. --- @@ -170,21 +176,43 @@ The resolver classifies each scheme arg into exactly one source. A `ccpp_model_constituents_obj()%vars_layer(:, :, index_of_)` (or `%vars_layer_tend(...)` for `tendency_of_` outputs). -### 2.2 The four scheme-author rules +### 2.2 The scheme-author rules (See `doc/constituents.md` for full details; this is the summary.) 1. **Register** — register-phase scheme args of type `ccpp_constituent_properties_t(:), intent=out, allocatable` declare new constituents the scheme contributes. -2. **Consume** — physics-phase scheme args with `advected=true` (or - `molar_mass=...` or `constituent=true`) and `intent=in/inout`, with - the constituent's standard name, read the base species. -3. **Produce a tendency** — physics-phase scheme args with - `constituent=true`, `intent=out`, and standard name - `tendency_of_`, write the tendency. -4. **Mismatched combinations are errors** — `intent=out` on a base - constituent, or `intent=in` on a tendency, are codegen-time errors. +2. **Flag what you own** — a physics-phase arg flagged `advected=true` + (or `molar_mass=...` or `constituent=true`) marks its standard name as + a constituent: a base species read via `%vars_layer` + (`intent=in/inout`), or — when the name is `tendency_of_` — a + constituent tendency *written* via `%vars_layer_tend` (`intent=out`). + A base constituent therefore uses `intent=inout` (read-modify-write the + shared column), never `intent=out`; `intent=out` is reserved for + tendencies. (This is why CAM-SIMA's `state_converters` dry→moist + converters declare the moist mixing ratios `intent=inout`.) +3. **Consume without re-flagging (rule b)** — a scheme that merely READS a + name some *other* scheme flags as a constituent does **not** repeat the + flag. Whether a standard name is a constituent or an ordinary variable + is the **host's** decision (CAM-SIMA exposes water vapor as a + constituent; CCPP-SCM may expose the same name as an ordinary host + variable), so capgen-ng infers it from the scheme-metadata-wide set of + flagged names (`VariableResolver.constituent_stdnames()`) rather than + from the consumer's own metadata. An unflagged `intent=in` read of the + base name resolves to `%vars_layer(...)`; an unflagged `intent=in` read + of `tendency_of_` resolves to `%vars_layer_tend(:, index_of_)` + — the same column a tendency *producer* wrote. **Host/earlier-suite + provision wins**: if the host declares the name, or an earlier scheme + already produced it as an ordinary variable, normal host/suite + resolution takes over. (Worked example: in the CAM-SIMA `cam7` suite + the convection/stratiform schemes write `tendency_of_water_vapor_...` + as a flagged constituent tendency, and the unflagged `sima_diagnostics` + schemes read it back via this rule — see §4.15.) +4. **Mismatched combinations are errors** — a constituent-FLAGGED + `intent=out` arg whose name is not a `tendency_of_*` is a codegen-time + error: physics phases may only PRODUCE tendencies; new base + constituents must be declared in the register phase. ### 2.3 Two registration sources (no auto-clone) @@ -690,6 +718,46 @@ shim. Remove the rewrite once known consumers are migrated. implementation cost; eliminates the cross-cutting confusion for any host whose `ccpp_error_code` local name is not `errcode`. +### 4.15 CAM-SIMA compat layer: `write_init_files` mis-flagged unflagged constituent-tendency consumers (FIXED 2026-06-05) + +- **Location**: `cime_config/capgen_compat/_var_wrapper.py` in CAM-SIMA + — the facade that lets capgen-ng drive CAM-SIMA's *unchanged* + `write_init_files.py` / `cam_autogen.py` — method + `_VarWrapper.from_resolved_arg`. +- **Symptom**: the `se_cslam` (FCAM7 `cam7`) build failed AFTER cap + generation, inside CAM-SIMA's own init-file generator: + `Error: Missing required host variables: + tendency_of_water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water`. +- **Mechanism**: in `cam7` the convection/stratiform schemes (`dadadj`, + `zm_conv_evap`, `rk_stratiform`, `zm_convr`, + `cloud_particle_sedimentation`) write that tendency as a FLAGGED + constituent tendency (`constituent=true intent=out`) → capgen-ng routes + them to `%vars_layer_tend` (`source='constituent'`, NOT recorded in + `suite_vars`) and the name enters `const_stds`. The four + `sima_diagnostics` schemes read it back `intent=in` UNFLAGGED → rule b + (§2.2.3) → `source='constituent'`, but `ResolvedArg.is_constituent` is + taken from the consumer's OWN flag = `False`. The compat wrapper + derived `advected`/`constituent` only from + `is_constituent`/`is_constituent_arg` → both `False`, and + `source='constituent'` was not in its suite-internal drop set, so + `write_init_files.gather_ccpp_req_vars` saw intent=in + not-constituent + + not-in-host-dict → "missing host variable". +- **Fix**: key the wrapper's `advected`/`constituent` on + `arg.source == 'constituent'` (a strict superset of the two old flags). + `write_init_files` skips constituents from BOTH USE-import and the + initial-conditions read — the constituents object supplies them at + runtime — so flagging the tendency consumer is correct, not a mask. + Verified 43/43 `capgen_compat` + 16/16 `test_write_init_files`, then + confirmed by a full `se_cslam` run to completion. +- **Takeaway**: `ResolvedArg.is_constituent` answers "did the SCHEME flag + it"; `source == 'constituent'` answers "is this supplied by the + constituents framework". Any host adapter (the CAM-SIMA compat layer + today, any future one) must key constituent handling on the *source*, + because rule-b inferred consumers legitimately carry + `is_constituent == False`. +- **Position relative to Proposals A/B/C**: orthogonal — a host-adapter + bug exposed by rule b, not a framework constituent-model change. + --- ## 5. Property classification (Class A vs Class B) diff --git a/doc/migration.md b/doc/migration.md index 9fb60ef9..53f6cf59 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -6,7 +6,7 @@ Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to **capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and `doc/redesign_analysis.md` (analysis of the old systems). -*Last revised: 2026-06-01.* Current unit-test suite: 1426 passing (1438 with doctests). +*Last revised: 2026-06-05.* Current unit-test suite: 1516 passing. **Repository layout** (post-2026-05-13 cleanup): tooling lives under `capgen-ng/` (top-level of this repo). Unit tests live at the top @@ -580,11 +580,13 @@ through lifecycle and physics-phase calls; the framework consumes `number_of_instances` only at register/init time but carries it elsewhere for API symmetry with `(thread_number, number_of_threads)`. -### 3.3 Host module convention +### 3.3 Module-name convention (host, scheme, and DDT tables) -The Fortran module that exports a host metadata table's variables is -typically named after the table. When that's not the case, use the -`module_name` table-property override (§1.2): +capgen-ng trusts metadata and does **not** parse Fortran, so it derives +the Fortran module name from the metadata: by default `module name = +table name`. When the Fortran `module` statement does not match the +`[ccpp-table-properties] name`, declare the real module name with the +`module_name` override (§1.2). This applies to **every** table type: ``` [ccpp-table-properties] @@ -593,6 +595,24 @@ typically named after the table. When that's not the case, use the module_name = mod_test_host_data ``` +The same rule bites **scheme** tables. If a scheme file's table is named +`gravity_wave_drag_common` but the Fortran is `module gw_common`, the +generated cap emits `use gravity_wave_drag_common` and the build fails +with `Cannot open module file 'gravity_wave_drag_common.mod'`. Fix it in +the scheme `.meta`: + +``` +[ccpp-table-properties] + name = gravity_wave_drag_common + type = scheme + module_name = gw_common +``` + +Standalone `type = ddt` tables **require** `module_name` explicitly +(there is no basename fallback). Porting a host like CAM-SIMA's +`atmospheric_physics` tree, where many scheme/DDT table names differ from +their module names, is largely a batch of `module_name` injections. + ### 3.4 Registered scalar-index dimensions A small set of CCPP standard-name dimensions are *registered*: each @@ -766,6 +786,54 @@ behavior of legacy `ccpp-prebuild` / `ccpp-capgen`. The staging temp file lives in the target's parent directory (always under `--output-root`), so no `/tmp` access is required. +### 4.5 Driving an existing capgen-based build: the CAM-SIMA compatibility layer + +A host whose build system was written against **original ccpp-capgen's +Python API** can adopt capgen-ng without rewriting that build system, by +inserting a thin facade. CAM-SIMA does exactly this with +`cime_config/capgen_compat/` (in the CAM-SIMA tree, not in capgen-ng). +CAM-SIMA's `cam_autogen.py`, `generate_registry_data.py`, and +`write_init_files.py` are unmodified; they import the facade instead of +original capgen and keep calling the same object surface +(`cap_database.host_model_dict()`, `cap_database.call_list(phase)`, +`Var.get_prop_value(...)`, `Var.source.ptype`, …). + +The facade re-implements that surface on top of capgen-ng's outputs: + +- `_runner.py` invokes `ccpp_capgen_ng.py` and returns the resolver + results plus the `datatable.xml`. +- `_cap_database.py` (`CapDatabase`) exposes `host_model_dict()` over the + flat `host_dict` and `call_list(phase)` over the per-(scheme, phase) + `ResolvedArg` lists, mapping original-capgen phase spellings + (`initialize`/`finalize`) onto capgen-ng's (`init`/`final`). +- `_var_wrapper.py` (`_VarWrapper`) reconstructs original capgen's + per-variable accessors over a `HostVarEntry` (host path) or a + `ResolvedArg` (call-list path). +- `metadata_table.py` / `parse_*` shim the metadata-parsing entry points + the registry generator expects. + +Two contracts matter when writing or maintaining such an adapter, both +learned from the CAM-SIMA bring-up: + +1. **Drop suite-internal args.** A `ResolvedArg` with + `source == 'suite'` is produced by one scheme and consumed by another + within the same suite (it lives in `_data`, never in the host + dict). The adapter must NOT surface it on the call list, or the host's + init-file generator will mis-flag it as a "missing required host + variable". +2. **Key constituent handling on the source.** Treat a `ResolvedArg` + with `source == 'constituent'` as supplied by the constituents object + (skip host USE-import and skip the initial-conditions read). Do **not** + key on `ResolvedArg.is_constituent`: that flag reflects whether the + *scheme* flagged the arg, and an unflagged rule-b consumer (§6.5) of a + constituent or `tendency_of_` carries `is_constituent = False` while + still being framework-supplied. Getting this wrong was the `se_cslam` + "Missing required host variables: tendency_of_water_vapor_…" failure + (`doc/constituents_overhaul.md` §4.15). + +This facade is how capgen-ng currently drives the `kessler`, `rrtmgp`, +and `se_cslam`/CSLAM (FCAM7 `cam7`) CAM-SIMA cases end-to-end on Derecho. + --- ## 5. Generated cap layout — what's new and what changed @@ -911,11 +979,13 @@ the buffer from creation — no ownership transfer call needed. - `ccpp_number_constituents(num_flds, advected, instance_number, ...)` - `ccpp_gather_constituents`, `ccpp_update_constituents` - `ccpp_is_scheme_constituent(var_name, ...)` (not per-instance) -- Scheme-side registration: four rules — register-phase - `ccpp_constituent_properties_t(:)` arg, consume base via - `advected=true intent=in/inout`, produce tendency via - `constituent=true intent=out` + `tendency_of_` std name, mismatches - are codegen errors. +- Scheme-side registration rules — register-phase + `ccpp_constituent_properties_t(:)` arg declares new constituents; + flag a base species with `advected=true intent=in/inout`; produce a + tendency with `constituent=true intent=out` + `tendency_of_` std + name; a constituent-flagged `intent=out` that is not a `tendency_of_*` + is a codegen error. A scheme that only READS a constituent or a + `tendency_of_` need not re-flag it — see §6.5. ### 6.3 Host metadata wins over auto-provisioning (2026-05-12) @@ -988,6 +1058,50 @@ Full reference: `doc/auto_clone_constituents.md`. E2e fixture: `end-to-end-tests/advection_auto_clone/` (a port of CAM-SIMA's `advection_test`). +### 6.5 Reading constituents and tendencies without re-flagging (rule b, 2026-06-05) + +Whether a given standard name is a constituent or an ordinary variable +is the **host's** decision: CAM-SIMA exposes water vapor as a +constituent; CCPP-SCM may expose the same name as an ordinary host +variable. A scheme that merely **reads** such a name therefore must +**not** repeat the `advected` / `constituent` flag — only the +declaring/producing scheme (or the host) does. + +capgen-ng infers constituent-ness for an unflagged consumer from the +scheme-metadata-wide set of names that *some* scheme flags +(`VariableResolver.constituent_stdnames()`): + +- an unflagged `intent=in/inout` read of a flagged base name resolves to + `…%vars_layer(:, …, index_of_)`; +- an unflagged `intent=in` read of `tendency_of_` resolves to + `…%vars_layer_tend(:, …, index_of_)` — the same column a constituent + tendency *producer* wrote. + +**Host / earlier-suite provision wins**: if the host declares the name, +or an earlier scheme already produced it as an ordinary variable, normal +host/suite resolution takes over and no constituent column is used. + +This is what lets the CAM-SIMA `cam7` suite work unchanged: the +convection/stratiform schemes write `tendency_of_water_vapor_…` as a +flagged constituent tendency and the `sima_diagnostics` schemes read it +back unflagged. Host adapters that post-process the resolver output +(e.g. CAM-SIMA's `write_init_files` via the compatibility layer, §4.5) +must key constituent handling on `ResolvedArg.source == 'constituent'`, +**not** on `ResolvedArg.is_constituent` — an inferred consumer carries +`is_constituent = False` by design. + +### 6.6 `number_of_ccpp_constituents` as a dimension + +A scheme (or a suite-owned interstitial) may be dimensioned by the +framework constituent count `number_of_ccpp_constituents`. capgen-ng +resolves that count for *any* variable: call-site subscripts emit `:` +for the constituent axis, and `_data` allocations size the axis +from the per-instance constituent object's `%num_layer_vars`. This is +what allows whole-buffer schemes (e.g. constituent advection) to declare +`dimensions = (horizontal_dimension, vertical_layer_dimension, +number_of_ccpp_constituents)`. E2e fixture: +`end-to-end-tests/constituents_dim/`. + --- ## 7. Validator From b576b357f49b8ff538ae530a3384e61f54499c5a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 5 Jun 2026 19:14:25 -0600 Subject: [PATCH 61/74] Add doc/capgen_compat_layer.md, update all other docs --- doc/briefing.md | 8 ++-- doc/briefing_pm.md | 5 ++- doc/capgen_compat_layer.md | 77 ++++++++++++++++++++++++++++++++++++ doc/constituents_overhaul.md | 3 +- doc/migration.md | 9 ++++- 5 files changed, 96 insertions(+), 6 deletions(-) create mode 100644 doc/capgen_compat_layer.md diff --git a/doc/briefing.md b/doc/briefing.md index fe7a307a..7de7a855 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -422,9 +422,11 @@ don't rebuild downstream objects unless something actually moved. `Var` accessors) on top of capgen-ng's `datatable.xml` + `ResolvedArg` / `HostVarEntry`. CAM-SIMA's `cam_autogen.py`, `generate_registry_data.py`, and `write_init_files.py` are unchanged. - Three cases build **and run to completion** on Derecho (gnu): - `kessler`, `rrtmgp`, and `se_cslam` / CSLAM (the FCAM7 `cam7` suite — - the full convection + stratiform + radiation + gravity-wave physics). + Three cases build **and run to completion** on Derecho under **both + gnu and intel** (bit-comparable results): `kessler`, `rrtmgp`, and + `se_cslam` / CSLAM (the FCAM7 `cam7` suite — the full convection + + stratiform + radiation + gravity-wave physics). A short shareable + brief on the compatibility layer is `doc/capgen_compat_layer.md`. `--legacy-auto-clone-constituents` is still the no-decision-needed bridge for the ~16 schemes that rely on auto-clone. Bring-up this week produced three reusable lessons baked into the docs: the diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md index 3e989f19..a78cb4d9 100644 --- a/doc/briefing_pm.md +++ b/doc/briefing_pm.md @@ -309,7 +309,8 @@ Features that exist only in capgen-ng (some exist in prebuild): drives the production CAM-SIMA build on the Derecho supercomputer through a small compatibility layer that lets CAM-SIMA's existing build scripts call capgen-ng without being rewritten. Three - configurations build **and run to completion**: `kessler`, `rrtmgp`, + configurations build **and run to completion under both the Intel and + GNU compilers**, with bit-comparable results: `kessler`, `rrtmgp`, and `se_cslam`/CSLAM — the last being the full CAM7 physics suite (deep + shallow convection, stratiform microphysics, RRTMGP radiation, gravity-wave drag) on a cubed-sphere/CSLAM-advection @@ -398,5 +399,7 @@ Three points worth raising explicitly: - `doc/migration.md` — host-author migration guide. - `doc/constituents_overhaul.md` — the constituent-reform discussion document. +- `doc/capgen_compat_layer.md` — short brief on the CAM-SIMA ↔ capgen-ng + compatibility layer (for the original ccpp-capgen author). - `end-to-end-tests/` — the working examples (`instances_advection` is the newest, exercises everything end-to-end). diff --git a/doc/capgen_compat_layer.md b/doc/capgen_compat_layer.md new file mode 100644 index 00000000..6e3a6c65 --- /dev/null +++ b/doc/capgen_compat_layer.md @@ -0,0 +1,77 @@ +# The CAM-SIMA ↔ capgen-ng compatibility layer — short brief + +*For the original ccpp-capgen author, to orient a feedback pass. +Full reference: `cime_config/capgen_compat/README.md` in the CAM-SIMA +tree. Last revised 2026-06-05.* + +## Why it exists + +capgen-ng is replacing original ccpp-capgen as CAM-SIMA's CCPP code +generator. Rather than rewrite CAM-SIMA's autogen pipeline up front, a +thin **compatibility layer** lets that pipeline keep calling original +capgen's Python surface while capgen-ng does the generation underneath. +`cam_autogen.py`, `generate_registry_data.py`, `write_init_files.py`, +and `hist_config.py` are **unmodified** — they import the facade instead +of original capgen. CAM-SIMA owns this directory; capgen-ng owns nothing +in it. It is **transient scaffolding** (see "Convergence goal"). + +## What the facade reconstructs + +| Original-capgen surface | Rebuilt over (capgen-ng) | File | +|---|---|---| +| `cap_database.host_model_dict()` / `.call_list(phase)` | flat `host_dict` + per-phase `ResolvedArg` lists | `_cap_database.py` | +| per-variable `Var` accessors (`get_prop_value`, `source.ptype`, `array_ref`, `intrinsic_elements`, `call_string`, …) | `HostVarEntry` (host path) / `ResolvedArg` (call-list path) | `_var_wrapper.py` | +| `MetaVar` / `MetadataSection` accessors used by the registry generator | capgen-ng `MetaVar` / `MetadataSection` via monkey-patch | `metadata_table.py` | +| `ParseObject`, `FortranWriter`, the richer `ParseContext` | **vendored verbatim from original capgen** (CAM-SIMA's own scripts use these; capgen-ng does not) | `parse_object.py`, `fortran_write.py`, `parse_source.py` | + +## Three design contracts worth your eyes + +capgen-ng classifies every scheme argument into exactly **one** source — +`control | host | suite | constituent` — on a flat `ResolvedArg` (there +is no `ConstituentVarDict`/scope-chain). The adapter keys on that: + +1. **`source='suite'` is dropped from `call_list`.** These are + interstitials produced and consumed within one suite (they live in + `_data`), so they are not host variables; surfacing them would + trip `write_init_files`' "missing required host variable" check. +2. **`source='constituent'` is mapped to `advected/constituent=True`** so + `write_init_files` routes it through the constituents object (skip + USE-import, skip the IC read). Key detail: the adapter keys on the + **source**, not on a per-arg `is_constituent` flag. capgen-ng now lets + an *unflagged* scheme consume a constituent — base *or* `tendency_of_*` + — because whether a name is a constituent is the **host's** decision; + such consumers carry `is_constituent=False` while still being + framework-supplied. +3. **`type = module` → `type = host`.** capgen-ng renamed your + `type = module`; the shim rewrites it at parse time and records which + tables were "module" so `write_init_files` still gets `ptype='module'` + (allocate + initialise) vs `ptype='host'` (passed via the arg list). + +## Convergence goal (the important framing) + +End state: this directory **does not exist**. CAM-SIMA talks to capgen-ng +through **three CLI utilities** — `ccpp_validator.py`, +`ccpp_capgen_ng.py`, `ccpp_datafile.py` — plus the on-disk +`datatable.xml` contract. A well-defined, feature-equivalent Python +API to these three utilities is also discussed in +`cime_config/capgen_compat/README.md`. No production CAM-SIMA path should +depend on capgen-ng's Python internals; today's `return_state=True` hook handing +back `(host_dict, suite_resolutions)` is scaffolding for this layer only. +The README has a phased retirement plan (A–G) with a measurable LOC drop +per phase. + +## Status + +`kessler`, `rrtmgp`, and `se_cslam` (the full `cam7` suite) build **and +run to completion** on Derecho under **both gnu and intel**, with +bit-comparable results. + +## Feedback we'd value + +1. Does the four-source model (`control/host/suite/constituent`) capture + everything `ConstituentVarDict` did for CAM-SIMA? +2. Are there `cap_database` / `Var` accessors that `write_init_files` or + `generate_registry_data` rely on that we've under- or mis-modeled? +3. Is **CLI + `datatable.xml`** a sufficient convergence interface for + everything CAM-SIMA currently reads out of original-capgen Python + objects — or is there state that has no on-disk equivalent yet? diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index 730a1322..90731663 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -748,7 +748,8 @@ shim. Remove the rewrite once known consumers are migrated. initial-conditions read — the constituents object supplies them at runtime — so flagging the tendency consumer is correct, not a mask. Verified 43/43 `capgen_compat` + 16/16 `test_write_init_files`, then - confirmed by a full `se_cslam` run to completion. + confirmed by full `se_cslam` runs to completion under both gnu and + intel (bit-comparable results). - **Takeaway**: `ResolvedArg.is_constituent` answers "did the SCHEME flag it"; `source == 'constituent'` answers "is this supplied by the constituents framework". Any host adapter (the CAM-SIMA compat layer diff --git a/doc/migration.md b/doc/migration.md index 53f6cf59..4ab99666 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -832,7 +832,11 @@ learned from the CAM-SIMA bring-up: (`doc/constituents_overhaul.md` §4.15). This facade is how capgen-ng currently drives the `kessler`, `rrtmgp`, -and `se_cslam`/CSLAM (FCAM7 `cam7`) CAM-SIMA cases end-to-end on Derecho. +and `se_cslam`/CSLAM (FCAM7 `cam7`) CAM-SIMA cases end-to-end on Derecho +— building and running to completion under both **gnu and intel**, with +bit-comparable results. A short shareable brief (for the original +ccpp-capgen author) is `doc/capgen_compat_layer.md`; the full developer +reference is `cime_config/capgen_compat/README.md` in the CAM-SIMA tree. --- @@ -1230,4 +1234,7 @@ Character `len=*` remains a wildcard against any concrete `len=N`. - `doc/constituents.md` — full constituents reference for capgen-ng. - `doc/constituents_overhaul.md` — architecture review and reform proposals for the next iteration. +- `doc/capgen_compat_layer.md` — short brief on the CAM-SIMA ↔ capgen-ng + compatibility layer (§4.5); full reference is + `cime_config/capgen_compat/README.md` in the CAM-SIMA tree. From 3b1fa57836609f84e904b3960f774d7062d9c4cb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 5 Jun 2026 19:28:36 -0600 Subject: [PATCH 62/74] Update doc/briefing_pm.md --- doc/briefing_pm.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md index a78cb4d9..2c2ce066 100644 --- a/doc/briefing_pm.md +++ b/doc/briefing_pm.md @@ -97,9 +97,8 @@ Three pressures converged in 2025/26: `ConstituentVarDict` scope-chain or the auto-clone path requires reading several modules together. Realistically, only one or two people on the framework team can change capgen without breaking - something downstream. One of them now lives overseas and opposes - simplification attempts from the others. This is an unacceptable - **bus-factor risk** that the redesign retires. + something downstream. One of them now lives overseas. This is an + unacceptable **bus-factor risk** that the redesign retires. --- From a1d9a007abe40bab7b241e441fd9bc0145911718 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 8 Jun 2026 09:58:58 -0600 Subject: [PATCH 63/74] Sharpening rules around character(len=...) so that first scheme/host/constituent that defines the variable must provide a valid length --- capgen-ng/ccpp_validator.py | 51 +++++++++++++++++-- capgen-ng/generator/suite_resolver.py | 22 ++++++++ capgen-ng/metadata/variable_resolver.py | 19 +++++++ doc/migration.md | 24 ++++++--- doc/redesign_analysis.md | 18 ++++--- unit-tests/test_suite_resolver.py | 54 ++++++++++++++------ unit-tests/test_validator.py | 64 +++++++++++++++++++++++ unit-tests/test_variable_resolver.py | 67 +++++++++++++++++++++++++ 8 files changed, 290 insertions(+), 29 deletions(-) diff --git a/capgen-ng/ccpp_validator.py b/capgen-ng/ccpp_validator.py index 53758520..38c6223c 100755 --- a/capgen-ng/ccpp_validator.py +++ b/capgen-ng/ccpp_validator.py @@ -11,8 +11,13 @@ (order-insensitive). 4. For every dummy argument present in both sides, the **per-arg attributes** agree: ``intent``, ``type``, ``kind``, and number of dimensions (rank). - ``character`` arguments treat ``len=*`` on either side as a wildcard - against any concrete ``len=N`` / ``len=:``. + A *scheme* ``character`` argument treats ``len=*`` on either side as a + wildcard against any concrete ``len=N`` / ``len=:`` — the storage is + supplied by the caller. In contrast, host / DDT metadata passed via + ``--host-files`` *defines* its character storage, so ``len=*`` is + rejected there (a concrete ``len=N`` is required); see + :func:`_check_definition_character_lengths`. Control tables are exempt + (their character vars are pass-through dummy arguments too). Asymmetric treatment of ``optional``: @@ -1354,6 +1359,38 @@ def _validate_ddt_table( return errors +def _check_definition_character_lengths(table) -> List[str]: + """Reject ``character ... kind = len=*`` in a definition-site table. + + Host and DDT metadata *define* the storage for their character + variables (a host module variable / a derived-type component), so an + assumed length (``len=*``) is illegal there: it is valid only on a + dummy argument (a scheme arg, or a control/lifecycle variable) where + the storage is supplied by the caller. Apply this only to ``host`` / + ``ddt`` tables -- ``control`` tables are exempt. This mirrors the + generator's ``build_flat_host_dict`` guard and is checked independently + of the Fortran-vs-metadata comparison (a ``character(len=*)`` host decl + is invalid Fortran in its own right and would never be found). + + Returns a list of error message strings (empty when none offend). + """ + errors: List[str] = [] + for mvar in table.variables(): + if ((mvar.type or '').strip().lower() == 'character' + and (mvar.kind or '').strip().lower() == 'len=*'): + errors.append( + "Character variable '{}' (standard_name '{}') in {} table " + "'{}' declares kind='len=*'; host and DDT metadata must give " + "character variables a concrete length (e.g. kind=len=512) " + "-- assumed length is valid only for dummy arguments (scheme " + "args and control/lifecycle variables).".format( + mvar.local_name, mvar.standard_name, + table.table_type, table.table_name, + ) + ) + return errors + + _FORTRAN_EXTENSIONS = ('.F90', '.f90', '.F', '.f') @@ -1576,13 +1613,21 @@ def validate( all_errors.extend( _validate_scheme(sname, scheme_store, subroutine_tree, log) ) - # Scheme-co-located DDTs validate the same way as host-side DDTs. + # Scheme-co-located DDTs validate the same way as host-side DDTs; + # their character components are definition sites too (a DDT component + # may not be assumed-length), so the len=* guard applies. for tbl in scheme_ddt_tables: + all_errors.extend(_check_definition_character_lengths(tbl)) all_errors.extend(_validate_ddt_table(tbl, ddt_index, log)) for tbl in host_tables: + # Host and DDT tables define their character storage: reject len=* + # regardless of the Fortran-comparison outcome. Control tables are + # exempt -- their character vars are pass-through dummy arguments. if tbl.table_type == 'host': + all_errors.extend(_check_definition_character_lengths(tbl)) all_errors.extend(_validate_host_table(tbl, modules_tree, log)) elif tbl.table_type == 'ddt': + all_errors.extend(_check_definition_character_lengths(tbl)) all_errors.extend(_validate_ddt_table(tbl, ddt_index, log)) elif tbl.table_type == 'control': log.info( diff --git a/capgen-ng/generator/suite_resolver.py b/capgen-ng/generator/suite_resolver.py index 8313cb5e..6113ad5b 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen-ng/generator/suite_resolver.py @@ -1646,6 +1646,28 @@ def _resolve_one_arg( elif host_entry is None and suite_var is None: # Case 2 or 3. if intent == 'out': + # This scheme is the first (in phase->scheme order) to provide + # this variable, so it DEFINES the suite-owned storage that the + # framework allocates in ``ccpp__data``. A character + # definer must specify a concrete length: ``len=*`` (assumed + # length) is only valid for a dummy argument, never for stored + # data, and a later ``len=*`` consumer/writer has no concrete + # length to inherit. Reject it here with a clear message rather + # than emitting an undeclarable ``character(len=*)`` component. + if ((scheme_var.type or '').strip().lower() == 'character' + and (scheme_var.kind or '').strip() == 'len=*'): + raise CCPPError( + "Suite-owned character variable '{}' (standard_name='{}') " + "is first defined as intent(out) by scheme '{}' (phase " + "'{}') with kind='len=*'; the defining scheme must declare " + "a concrete length (e.g. kind=len=512) because the " + "framework allocates storage for it in the suite data " + "module. Assumed length (len=*) is permitted only on " + "later schemes that consume or re-write the " + "variable.".format( + local, std_name, scheme_name, phase, + ) + ) inst_entry = host_dict.get('instance_number') inst_access = '({})'.format(inst_entry.local_name) if inst_entry else '(1)' suite_var = SuiteVar( diff --git a/capgen-ng/metadata/variable_resolver.py b/capgen-ng/metadata/variable_resolver.py index 77fd7960..676fb343 100644 --- a/capgen-ng/metadata/variable_resolver.py +++ b/capgen-ng/metadata/variable_resolver.py @@ -649,6 +649,25 @@ def build_flat_host_dict( result: Dict[str, HostVarEntry] = {} def _add(entry: HostVarEntry, source_label: str) -> None: + # A character variable whose storage is DEFINED by host or DDT + # metadata must be concrete: ``len=*`` (assumed length) is illegal + # for a host module variable or a derived-type component. Control + # variables are EXEMPT -- they are pass-through dummy arguments + # (suite_name, errmsg, ...) which the generated caps legitimately + # declare ``character(len=*)``. Reject it here so the error names + # the table rather than surfacing downstream as undeclarable Fortran. + if (not entry.is_control + and (entry.type or '').strip().lower() == 'character' + and (entry.kind or '').strip() == 'len=*'): + raise CCPPError( + "Character variable '{}' (standard_name='{}') in table '{}' " + "declares kind='len=*'; host and DDT metadata must give " + "character variables a concrete length (e.g. kind=len=512) " + "-- assumed length is valid only for dummy arguments (scheme " + "args and control/lifecycle variables).".format( + entry.local_name, entry.standard_name, source_label, + ) + ) prior = result.get(entry.standard_name) if prior is not None: prior_loc = ( diff --git a/doc/migration.md b/doc/migration.md index 4ab99666..72fe4587 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -185,14 +185,26 @@ is deliberate: real CCPP-physics schemes legitimately mix precisions and rely on the cap to handle the copy. Watch for unintended narrowing — there is no static guard. **Character `len=`** has its own block: matching `len=N` values pass, mismatched specific lengths -are an error unless the scheme uses `len=*` (wildcard). +are an error unless the *consuming* scheme uses `len=*` (wildcard). +`len=*` is only valid where a variable is *passed*, never where its +storage is *defined*: host and DDT metadata must give every character +variable a concrete length, and so must the first `intent=out` scheme +that defines a suite-owned character variable (see below). Both are +rejected with a clear error rather than emitting an undeclarable +`character(len=*)` component. Control variables are exempt — they are +pass-through dummy arguments (`suite_name`, `errmsg`, …) the caps +legitimately declare `character(len=*)`. **Suite-owned variables.** The first scheme to write a standard -name with `intent=out` freezes the var's type/kind/dimensions/units -on the SuiteVar; every later scheme that consumes it goes through -the same checks against the frozen fields. Error messages name the -source as `host`, `control`, or `suite` so you know whose contract -you're violating. +name with `intent=out` (in phase→scheme order) freezes the var's +type/kind/dimensions/units on the SuiteVar; every later scheme that +consumes it goes through the same checks against the frozen fields. +Because that first writer *defines* the storage the framework +allocates in `ccpp__data`, a character definer must declare a +concrete length (`kind = len=N`); `len=*` there is an error. Later +consumers/writers of the same variable may use `len=*` as a wildcard. +Error messages name the source as `host`, `control`, or `suite` so +you know whose contract you're violating. #### 1.3.3 `allocatable` and who owns suite-data allocation diff --git a/doc/redesign_analysis.md b/doc/redesign_analysis.md index c4e7469d..e7dc360b 100644 --- a/doc/redesign_analysis.md +++ b/doc/redesign_analysis.md @@ -1143,12 +1143,18 @@ the **host variable's** `active` attribute when the scheme itself specifies no ` **Character length (`len=N` / `len=*`) rules.** Character kind declarations follow specific compatibility rules enforced by the resolver: -- `len=*` in a **scheme** is always compatible with any host `len=` — assumed-length - dummy arguments accept any host-declared length. No transform is generated. -- Matching specific `len=N` in both host and scheme requires no transform (naturally equal). -- Mismatched specific lengths (`len=512` host vs `len=128` scheme) are a **metadata error**; - the scheme must declare `len=*` or match the defining metadata exactly. -- `len=*` in the **host** with a specific `len=N` in the scheme is also an error. +- `len=*` is valid only where a character variable is **passed**, never where its + storage is **defined**. Host and DDT metadata must give every character variable + a concrete `len=N`; so must the first `intent=out` scheme that defines a + suite-owned character variable (it freezes the storage the framework allocates in + `ccpp__data`). `len=*` in any of those positions is a **metadata error**. + Control variables are exempt — they are pass-through dummy arguments the caps + declare `character(len=*)`. +- `len=*` in a **consuming/later** scheme is always compatible with the defining + `len=` — assumed-length dummy arguments accept any declared length. No transform. +- Matching specific `len=N` on both sides requires no transform (naturally equal). +- Mismatched specific lengths (`len=512` definer vs `len=128` consumer) are a + **metadata error**; the consuming scheme must declare `len=*` or match exactly. The resolver raises `CCPPError` for the illegal cases. No kind transform is ever generated for character variables — lengths are a Fortran compatibility constraint, not a unit conversion. diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 24cdc4e4..73ef4ec0 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -1168,6 +1168,39 @@ def test_case2_suite_owned(self): self.assertIn('brand_new_standard_name', suite_vars) self.assertIsNotNone(arg.suite_var) + def test_case2_suite_owned_character_assumed_length_raises(self): + """A character variable first defined as intent(out) by a scheme with + kind=len=* is rejected: the defining scheme must give a concrete + length because the framework allocates suite-owned storage for it.""" + hd = self._host_dict() + suite_var = self._scheme_var('name', 'scheme_name', 'out', 'none', + '()', 'character', 'len=*') + with self.assertRaises(CCPPError) as cm: + _resolve_one_arg(suite_var, 'run', hd, {}, 'def_scheme', set()) + msg = str(cm.exception) + self.assertIn('scheme_name', msg) + self.assertIn('len=*', msg) + self.assertIn('def_scheme', msg) + + def test_case2_suite_owned_character_concrete_then_assumed_ok(self): + """A concrete-length definer followed by a len=* consumer/writer is + accepted: the suite var inherits the defining concrete length and the + later assumed-length declaration acts as a wildcard.""" + hd = self._host_dict() + definer = self._scheme_var('name', 'scheme_name', 'out', 'none', + '()', 'character', 'len=512') + suite_vars: dict = {} + _resolve_one_arg(definer, 'run', hd, suite_vars, 'def_scheme', set()) + self.assertEqual(suite_vars['scheme_name'].kind, 'len=512') + + consumer = self._scheme_var('nm', 'scheme_name', 'out', 'none', + '()', 'character', 'len=*') + arg = _resolve_one_arg(consumer, 'run', hd, suite_vars, 'use_scheme', + set()) + self.assertEqual(arg.source, 'suite') + # Storage length stays the defining concrete length. + self.assertEqual(suite_vars['scheme_name'].kind, 'len=512') + def test_case3_not_found_intent_in_raises(self): """Case 3: not in host, intent(in) → CCPPError.""" hd = self._host_dict() @@ -1857,12 +1890,13 @@ def test_len_match_compatible(self): arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) self.assertFalse(arg.needs_kind_transform) - def test_len_star_in_host_no_error(self): - """len=* in the host is also fine (assumed-length dummy everywhere).""" - hd = self._host_with_char('len=*') - suite_var = self._scheme_var_char('msg', 'my_message', 'len=*') - arg = _resolve_one_arg(suite_var, 'run', hd, {}, 'sch', set()) - self.assertFalse(arg.needs_kind_transform) + def test_len_star_in_host_raises(self): + """len=* in the host is rejected at host-dict construction: a host + character variable defines storage and must have a concrete length + (assumed length is valid only for a scheme dummy argument).""" + with self.assertRaises(CCPPError) as cm: + self._host_with_char('len=*') + self.assertIn('len=*', str(cm.exception)) def test_mismatched_specific_lengths_raises(self): """Specific len=128 vs len=512 is a metadata error.""" @@ -1873,14 +1907,6 @@ def test_mismatched_specific_lengths_raises(self): self.assertIn('len=512', str(cm.exception)) self.assertIn('len=128', str(cm.exception)) - def test_len_star_host_specific_scheme_raises(self): - """len=* in host but specific len=256 in scheme — error.""" - hd = self._host_with_char('len=*') - suite_var = self._scheme_var_char('msg', 'my_message', 'len=256') - with self.assertRaises(CCPPError) as cm: - _resolve_one_arg(suite_var, 'run', hd, {}, 'bad_scheme', set()) - self.assertIn('len=256', str(cm.exception)) - ######################################################################## # Tests: pure-kind transform (real-kind cast) diff --git a/unit-tests/test_validator.py b/unit-tests/test_validator.py index 2c7f57e2..99463c47 100644 --- a/unit-tests/test_validator.py +++ b/unit-tests/test_validator.py @@ -1274,6 +1274,70 @@ def test_kind_error(self): self.assertIn("kind mismatch", errs[0]) +class TestValidateHostCharacterAssumedLength(_HostValidationFixture): + """A host character variable declared ``kind = len=*`` is rejected even + when the Fortran side has a concrete length (the metadata defines the + storage there, so assumed length is illegal).""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ scheme_name ] + standard_name = scheme_name + long_name = scheme name + units = none + dimensions = () + type = character | kind = len=* + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + character(len=512) :: scheme_name + end module my_host + """) + + def test_assumed_length_error(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(len(errs), 1, errs) + self.assertIn("len=*", errs[0]) + self.assertIn("scheme_name", errs[0]) + + +class TestValidateHostCharacterConcreteLengthOK(_HostValidationFixture): + """A concrete host character length matching the Fortran passes.""" + + META = textwrap.dedent("""\ + [ccpp-table-properties] + name = my_host + type = host + [ccpp-arg-table] + name = my_host + type = host + [ scheme_name ] + standard_name = scheme_name + long_name = scheme name + units = none + dimensions = () + type = character | kind = len=512 + """) + + F90 = textwrap.dedent("""\ + module my_host + implicit none + character(len=512) :: scheme_name + end module my_host + """) + + def test_concrete_length_ok(self): + errs = validate([], [self.f90_path], host_files=[self.meta_path]) + self.assertEqual(errs, []) + + class TestValidateHostRankMismatch(_HostValidationFixture): META = textwrap.dedent("""\ diff --git a/unit-tests/test_variable_resolver.py b/unit-tests/test_variable_resolver.py index 2d2bf1e5..28767911 100644 --- a/unit-tests/test_variable_resolver.py +++ b/unit-tests/test_variable_resolver.py @@ -772,6 +772,73 @@ def test_missing_ddt_table_raises(self): build_flat_host_dict(host_tables, [], []) self.assertIn('gfs_statein_type', str(cm.exception)) + def test_host_character_assumed_length_raises(self): + """A host character variable with kind=len=* is rejected: host + metadata must give a concrete length (len=* is only valid for a + scheme dummy argument).""" + src = ''' +[ccpp-table-properties] + name = host_data + type = host +[ccpp-arg-table] + name = host_data + type = host +[ scheme_name ] + standard_name = scheme_name + units = none + dimensions = () + type = character + kind = len=* +''' + tables = _parse_lines(src.splitlines(keepends=True), 'h.meta') + with self.assertRaises(CCPPError) as cm: + build_flat_host_dict(tables, [], []) + msg = str(cm.exception) + self.assertIn('scheme_name', msg) + self.assertIn('len=*', msg) + + def test_control_character_assumed_length_ok(self): + """Control-table character variables are EXEMPT: they are pass-through + dummy arguments (suite_name, errmsg, ...) that the generated caps + declare ``character(len=*)``, so len=* is valid there.""" + src = ''' +[ccpp-table-properties] + name = ccpp_control + type = control +[ccpp-arg-table] + name = ccpp_control + type = control +[ label ] + standard_name = some_label + units = none + dimensions = () + type = character + kind = len=* +''' + tables = _parse_lines(src.splitlines(keepends=True), 'c.meta') + d = build_flat_host_dict([], tables, []) + self.assertEqual(d['some_label'].kind, 'len=*') + + def test_host_character_concrete_length_ok(self): + """A concrete host character length is accepted unchanged.""" + src = ''' +[ccpp-table-properties] + name = host_data + type = host +[ccpp-arg-table] + name = host_data + type = host +[ scheme_name ] + standard_name = scheme_name + units = none + dimensions = () + type = character + kind = len=512 +''' + tables = _parse_lines(src.splitlines(keepends=True), 'h.meta') + d = build_flat_host_dict(tables, [], []) + self.assertEqual(d['scheme_name'].kind, 'len=512') + def test_host_vars_not_control(self): # Host vars are is_control=False; loop bounds are now control vars (control table). host_tables = _parse_file('host_simple.meta') From f0bf6182a67f059d26dfbcbc5471a0f6296148da Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 8 Jun 2026 12:46:22 -0600 Subject: [PATCH 64/74] Bug fix in capgen-ng: if all groups are called, check for errflg /= 1 in between --- capgen-ng/generator/suite_cap.py | 9 +++++++ unit-tests/test_suite_cap.py | 45 +++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 8745aa85..6c40ffd2 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -1066,6 +1066,15 @@ def _emit_group_call(resolved_group, indent): lines.append('{} {}{}'.format(indent, lname, sep)) else: lines.append('{}call {}()'.format(indent, cap_sub)) + # Stop and propagate on the first group error. Each group phase + # subroutine resets ``errflg = 0`` on entry, so without this guard a + # ``group_name='all'`` dispatch would let a LATER group's success + # overwrite an EARLIER group's failure -- the error (and its message) + # would be silently masked and only resurface downstream as an + # "invalid group state" when ``run`` finds the failed group never + # reached ``IN_TIMESTEP``. Mirrors the per-scheme call guards. + if errflg_local: + lines.append('{}if ({} /= 0) return'.format(indent, errflg_local)) if has_group_name: grp_local = group_name_entry.local_name diff --git a/unit-tests/test_suite_cap.py b/unit-tests/test_suite_cap.py index 0d3699ac..b852d034 100644 --- a/unit-tests/test_suite_cap.py +++ b/unit-tests/test_suite_cap.py @@ -8,7 +8,7 @@ from metadata.metadata_table import parse_metadata_file from metadata.variable_resolver import build_flat_host_dict, SchemeStore -from generator.suite_resolver import resolve_suite +from generator.suite_resolver import resolve_suite, ResolvedGroup from generator.suite_cap import ( _all_suite_scheme_names, _schemes_with_register, @@ -287,6 +287,49 @@ def test_all_phases_have_default_case(self): "phase '{}' missing case default".format(phase)) +class TestGroupDispatchErrorPropagation(unittest.TestCase): + """A ``group_name='all'`` dispatch must stop and return on the FIRST + group's error. Each group phase subroutine resets ``errflg=0`` on entry, + so without a guard between group calls a later group's success would mask + an earlier group's failure -- which then resurfaces downstream only as an + "invalid group state" when ``run`` finds the failed group never reached + ``IN_TIMESTEP``. (Regression: CAM-SIMA cam4 physics_before_coupler.)""" + + def _two_group_run_all_block(self): + sr, store = _resolve() + g0 = sr.groups[0] + # Synthesize a second group that shares the first's phase calls so the + # case('', 'all') path emits two group calls. + sr.groups.append(ResolvedGroup( + group_name='physics_second', + phase_calls=g0.phase_calls, + dim_uses=g0.dim_uses, + )) + text = '\n'.join( + _generate_suite_cap('test_simple', sr, store, _load_full_host_dict()) + ) + sub = 'subroutine test_simple_physics_run' + s = text.index(sub) + e = text.index('end ' + sub, s) + block = text[s:e] + a = block.index("case('', 'all')") + nxt = block.index("case('physics", a + 1) # first individual group case + return block[a:nxt] + + def test_guard_between_group_calls(self): + all_block = self._two_group_run_all_block() + # Each of the two group calls is followed by an errflg guard. + self.assertEqual( + all_block.count('if (errflg /= 0) return'), 2, all_block + ) + # The guard after the first call must precede the second call so the + # second group is unreachable once the first has failed. + first_call = all_block.index('call physics_run(') + guard = all_block.index('if (errflg /= 0) return', first_call) + second_call = all_block.index('call physics_second_run(') + self.assertLess(guard, second_call, all_block) + + class TestWriteSuiteCap(unittest.TestCase): def test_writes_file(self): From 44eb383eafb5bba9f8c888da391ea1e75b0cbaec Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 8 Jun 2026 14:34:28 -0600 Subject: [PATCH 65/74] Bug fixes and updates for Fortran vs metadata validation: now validate old and new Fortran characters --- capgen-ng/ccpp_validator.py | 81 +++++++++++++++---- .../constituents_dim/const_dim_consumer.meta | 2 +- .../constituents_dim/const_dim_producer.meta | 2 +- .../nested_suite/suite_lifecycle.meta | 4 +- unit-tests/sample_files/scheme_multipart.meta | 6 +- unit-tests/test_validator.py | 53 +++++++++++- 6 files changed, 120 insertions(+), 28 deletions(-) diff --git a/capgen-ng/ccpp_validator.py b/capgen-ng/ccpp_validator.py index 38c6223c..7b42f71e 100755 --- a/capgen-ng/ccpp_validator.py +++ b/capgen-ng/ccpp_validator.py @@ -11,13 +11,14 @@ (order-insensitive). 4. For every dummy argument present in both sides, the **per-arg attributes** agree: ``intent``, ``type``, ``kind``, and number of dimensions (rank). - A *scheme* ``character`` argument treats ``len=*`` on either side as a - wildcard against any concrete ``len=N`` / ``len=:`` — the storage is - supplied by the caller. In contrast, host / DDT metadata passed via - ``--host-files`` *defines* its character storage, so ``len=*`` is - rejected there (a concrete ``len=N`` is required); see - :func:`_check_definition_character_lengths`. Control tables are exempt - (their character vars are pass-through dummy arguments too). + ``character`` length must be declared CONSISTENTLY — the metadata mirrors + the Fortran exactly, so ``len=*`` matches only ``len=*`` and ``len=N`` only + the identical ``len=N`` (no wildcarding). Old-style F77 forms + (``character*64``, ``character*(*)``, ``c*5``) are normalised to the + ``len=`` form before comparison. Additionally, host / DDT metadata passed + via ``--host-files`` *defines* its character storage, so ``len=*`` is + rejected there outright (a concrete ``len=N`` required); see + :func:`_check_definition_character_lengths`. Control tables are exempt. Asymmetric treatment of ``optional``: @@ -224,6 +225,12 @@ def _split_type_spec(spec: str) -> "Tuple[str, str]": ('character', 'len=*') >>> _split_type_spec('character') ('character', '') + >>> _split_type_spec('character*64') + ('character', 'len=64') + >>> _split_type_spec('character*(*)') + ('character', 'len=*') + >>> _split_type_spec('character*(80)') + ('character', 'len=80') >>> _split_type_spec('type(my_t)') ('type(my_t)', '') >>> _split_type_spec('double precision') @@ -231,7 +238,21 @@ def _split_type_spec(spec: str) -> "Tuple[str, str]": >>> _split_type_spec('not_a_type') ('', '') """ - m = _TYPE_SPEC_RE.match(spec.strip()) + spec = spec.strip() + # Old-style (F77) character length given with ``*`` instead of a modern + # ``(len=...)`` selector: + # character*64 -> len=64 character*(*) -> len=* + # character*(80) -> len=80 character*(CL) -> len=cl (named) + m_old = re.match(r'(?i)^character\s*\*\s*(.+)$', spec) + if m_old is not None: + length = m_old.group(1).strip() + paren = re.match(r'^\(\s*(.*?)\s*\)$', length) # peel one ( ) layer + if paren is not None: + length = paren.group(1).strip() + if length == '*': + return ('character', 'len=*') + return ('character', 'len={}'.format(length.lower())) + m = _TYPE_SPEC_RE.match(spec) if m is None: return ('', '') type_raw = m.group(1).lower() @@ -295,6 +316,10 @@ def _parse_decl_line(line: str) -> Dict[str, _ArgAttrs]: {} >>> _parse_decl_line('integer :: only_local') {'only_local': _ArgAttrs(type_='integer', kind_='', intent='', optional=False, rank=0)} + >>> _parse_decl_line('character*256, intent(out) :: scheme_name')['scheme_name'] + _ArgAttrs(type_='character', kind_='len=256', intent='out', optional=False, rank=0) + >>> sorted(_parse_decl_line('character :: c*5, d(10)*8').items()) + [('c', _ArgAttrs(type_='character', kind_='len=5', intent='', optional=False, rank=0)), ('d', _ArgAttrs(type_='character', kind_='len=8', intent='', optional=False, rank=1))] """ line = _COMMENT_RE.sub('', line) if '::' not in line: @@ -340,6 +365,21 @@ def _parse_decl_line(line: str) -> Dict[str, _ArgAttrs]: # parenthesised sub-expressions (e.g. ``::x = (a==b)``) live at # depth > 0 and are skipped. var_tok = _strip_initialiser(var_tok).rstrip() + # Old-style (F77) per-entity character length: ``c*5`` / ``d(10)*8`` + # / ``s*(*)``. A trailing ``*`` overrides the type-spec + # length for THIS entity only. + entity_kind = None + m_star = re.match( + r'(?i)^(.*?)\s*\*\s*(\(\s*\*\s*\)|\(\s*\w+\s*\)|\d+|\*|\w+)\s*$', + var_tok, + ) + if m_star is not None: + var_tok = m_star.group(1).strip() + length = m_star.group(2).strip() + paren = re.match(r'^\(\s*(.*?)\s*\)$', length) + if paren is not None: + length = paren.group(1).strip() + entity_kind = 'len=*' if length == '*' else 'len={}'.format(length.lower()) name_match = re.match(r'(\w+)\s*(\((.*)\))?\s*$', var_tok) if name_match is None: continue @@ -350,8 +390,9 @@ def _parse_decl_line(line: str) -> Dict[str, _ArgAttrs]: else: rank = line_rank result[name] = _ArgAttrs( - type_=type_, kind_=kind_, intent=intent, - optional=optional, rank=rank, + type_=type_, + kind_=entity_kind if entity_kind is not None else kind_, + intent=intent, optional=optional, rank=rank, ) return result @@ -1107,9 +1148,10 @@ def _check_arg_attributes( """Compare per-attribute consistency for one dummy argument. Compared attributes: ``intent``, ``type``, ``kind``, dimension - *rank* (number of dims). ``character`` kinds treat ``len=*`` on - either side as a wildcard against any concrete ``len=N`` / - ``len=:``. The ``optional`` attribute is checked at the call site + *rank* (number of dims). ``character`` length must be declared + CONSISTENTLY: the metadata mirrors the Fortran exactly -- ``len=*`` + matches only ``len=*`` and ``len=N`` only the identical ``len=N`` (no + wildcarding). The ``optional`` attribute is checked at the call site in :func:`_validate_scheme` because one direction emits a warning (logger-dependent) rather than an error. @@ -1153,12 +1195,17 @@ def _check_arg_attributes( meta_kind = (meta_var.kind or '').strip().lower() fort_kind = fort.kind_ if meta_type == 'character' or fort.type_ == 'character': - if meta_kind == 'len=*' or fort_kind == 'len=*': - pass # wildcard match - elif meta_kind != fort_kind: + # Character length must be CONSISTENT between metadata and Fortran: + # the metadata mirrors the declaration, it does not loosely match it. + # ``len=*`` matches only ``len=*`` (assumed length on one side vs a + # concrete length on the other is a real inconsistency), and ``len=N`` + # matches only the identical ``len=N``. + if meta_kind != fort_kind: errs.append( prefix + "character length mismatch " - "(metadata={!r}, Fortran={!r})".format(meta_kind, fort_kind) + "(metadata={!r}, Fortran={!r}); the metadata kind must mirror " + "the Fortran declaration exactly -- len=* only matches len=*, " + "len=N only matches the same len=N".format(meta_kind, fort_kind) ) else: if meta_kind != fort_kind: diff --git a/end-to-end-tests/constituents_dim/const_dim_consumer.meta b/end-to-end-tests/constituents_dim/const_dim_consumer.meta index ef2622b8..40e4a09f 100644 --- a/end-to-end-tests/constituents_dim/const_dim_consumer.meta +++ b/end-to-end-tests/constituents_dim/const_dim_consumer.meta @@ -44,7 +44,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=* intent = out [errcode] standard_name = ccpp_error_code diff --git a/end-to-end-tests/constituents_dim/const_dim_producer.meta b/end-to-end-tests/constituents_dim/const_dim_producer.meta index f56f3a1c..777644f8 100644 --- a/end-to-end-tests/constituents_dim/const_dim_producer.meta +++ b/end-to-end-tests/constituents_dim/const_dim_producer.meta @@ -62,7 +62,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=* intent = out [errcode] standard_name = ccpp_error_code diff --git a/end-to-end-tests/nested_suite/suite_lifecycle.meta b/end-to-end-tests/nested_suite/suite_lifecycle.meta index 673e348d..089ffd6e 100644 --- a/end-to-end-tests/nested_suite/suite_lifecycle.meta +++ b/end-to-end-tests/nested_suite/suite_lifecycle.meta @@ -16,7 +16,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=* intent = out [errflg] standard_name = ccpp_error_code @@ -39,7 +39,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=* intent = out [errflg] standard_name = ccpp_error_code diff --git a/unit-tests/sample_files/scheme_multipart.meta b/unit-tests/sample_files/scheme_multipart.meta index 82e04c20..fd340118 100644 --- a/unit-tests/sample_files/scheme_multipart.meta +++ b/unit-tests/sample_files/scheme_multipart.meta @@ -20,7 +20,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=* intent = out [ errflg ] standard_name = ccpp_error_code @@ -60,7 +60,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=* intent = out [ errflg ] standard_name = ccpp_error_code @@ -77,7 +77,7 @@ units = none dimensions = () type = character - kind = len=512 + kind = len=* intent = out [ errflg ] standard_name = ccpp_error_code diff --git a/unit-tests/test_validator.py b/unit-tests/test_validator.py index 99463c47..eb05b49e 100644 --- a/unit-tests/test_validator.py +++ b/unit-tests/test_validator.py @@ -850,7 +850,21 @@ def test_kind_mismatch(self): self.assertTrue(any("kind mismatch" in e and "'b'" in e for e in errs), msg=errs) - def test_character_len_star_is_wildcard(self): + def test_character_len_star_consistent_passes(self): + # len=* on BOTH sides is consistent -> no error. + errs = self._run( + self._meta(type_b='character', kind_b='len=*'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' character(len=*), intent(in) :: b\n' + )), + ) + char_errs = [e for e in errs if "'b'" in e and 'character' in e] + self.assertEqual(char_errs, [], msg=errs) + + def test_character_len_star_vs_concrete_is_mismatch(self): + # len=* must NOT wildcard against a concrete len=N: the metadata must + # mirror the Fortran exactly. errs = self._run( self._meta(type_b='character', kind_b='len=512'), self._f90('a, b', ( @@ -858,6 +872,33 @@ def test_character_len_star_is_wildcard(self): ' character(len=*), intent(in) :: b\n' )), ) + self.assertTrue( + any("character length mismatch" in e and "'b'" in e for e in errs), + msg=errs, + ) + + def test_character_concrete_len_match_passes(self): + # Identical concrete lengths agree. + errs = self._run( + self._meta(type_b='character', kind_b='len=64'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' character(len=64), intent(in) :: b\n' + )), + ) + char_errs = [e for e in errs if "'b'" in e and 'character' in e] + self.assertEqual(char_errs, [], msg=errs) + + def test_character_old_style_len_parsed_and_matched(self): + # Old-style F77 character*64 in Fortran is normalised to len=64 and + # matched against the metadata. + errs = self._run( + self._meta(type_b='character', kind_b='len=64'), + self._f90('a, b', ( + ' integer, intent(in) :: a\n' + ' character*64, intent(in) :: b\n' + )), + ) char_errs = [e for e in errs if "'b'" in e and 'character' in e] self.assertEqual(char_errs, [], msg=errs) @@ -1302,10 +1343,14 @@ class TestValidateHostCharacterAssumedLength(_HostValidationFixture): """) def test_assumed_length_error(self): + # Two complementary errors now fire: the definition-site rejection of + # len=* in host/DDT metadata, plus the exact-match inconsistency vs the + # concrete Fortran (character(len=512)). Both name scheme_name. errs = validate([], [self.f90_path], host_files=[self.meta_path]) - self.assertEqual(len(errs), 1, errs) - self.assertIn("len=*", errs[0]) - self.assertIn("scheme_name", errs[0]) + self.assertTrue( + any("concrete length" in e and "len=*" in e for e in errs), errs + ) + self.assertTrue(all("scheme_name" in e for e in errs), errs) class TestValidateHostCharacterConcreteLengthOK(_HostValidationFixture): From cda7b9a4b4846a4a161883dc1b59c9e6c960201f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 8 Jun 2026 17:09:34 -0600 Subject: [PATCH 66/74] Bug fix for double allocation of constituents --- capgen-ng/generator/suite_cap.py | 94 +++++++++++++++++-------------- unit-tests/test_suite_resolver.py | 26 ++++----- 2 files changed, 64 insertions(+), 56 deletions(-) diff --git a/capgen-ng/generator/suite_cap.py b/capgen-ng/generator/suite_cap.py index 6c40ffd2..a9b556d2 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen-ng/generator/suite_cap.py @@ -466,15 +466,16 @@ def _register_lines( if has_dyn_consts: lines.append('') if has_consts: + # Each constituent scheme's _register returns its array into this + # temp; it is appended to the buffer and reused for the next + # scheme, so _register is called EXACTLY ONCE per scheme. lines.append( '{}type({}), allocatable :: scheme_consts(:)'.format( i2, _CONST_PROP_TYPE ) ) - lines.append('{}integer :: num_consts, i'.format(i2)) - else: - # auto-clone-only path: no scheme-returned temp, no copy - # loop, so we don't need ``scheme_consts`` or ``i``. + if has_auto_cloned: + # Counter used only by the auto-clone-constituents buffer growth. lines.append('{}integer :: num_consts'.format(i2)) # Trace block: dummies referenced inside the gated write so strict @@ -522,15 +523,18 @@ def _register_lines( # ``ccpp_register_constituents`` can ``set_const_index`` on each # without conflicting with other instances. The outer wrapper # array is allocated once on first call (any instance); each - # instance then runs its own two-pass count+pack into its slot. - # The state-machine guard above this block ensures each instance - # runs the fill at most once. + # instance then appends each scheme's constituents into its slot, + # calling every scheme's ``_register`` EXACTLY ONCE (register may + # allocate persistent module state, so the earlier two-pass + # count+copy that called it twice broke non-idempotent schemes such + # as ``prescribed_aerosols_register``). The state-machine guard + # above this block ensures each instance runs the fill at most once. # # auto-clone-constituents: the legacy shim contributes one # additional ``%instantiate`` per consumer-side # ``is_constituent`` arg with no register-phase source. Those - # synthesised entries participate in the same two-pass - # count+pack against the same per-instance buffer slot. + # synthesised entries are appended to the same per-instance buffer + # slot after the scheme-registered entries. const_scheme_names = {scheme_name for scheme_name, _ in suite_res.constituent_register_calls} buf = '{}_dynamic_constituents'.format(suite_name) n_auto_clone = len(suite_res.auto_cloned_constituents) @@ -545,59 +549,63 @@ def _register_lines( lines.append('{}end if'.format(i2)) lines.append('') - # Per-instance two-pass count+pack into this instance's slot. - lines.append('{}num_consts = 0'.format(i2)) - lines.append('{}! First pass: count constituents'.format(i2)) + # Single pass: call each constituent scheme's _register EXACTLY ONCE + # and append its returned array to this instance's slot. Start from + # an empty slot and grow it; intrinsic assignment of the constituent + # array constructor deep-copies each entry (same assignment the old + # copy loop used element-wise). + lines.append( + "{}! Pack each scheme's constituents (register run once each).".format(i2) + ) + lines.append('{}allocate({}({})%items(0))'.format(i2, buf, inst_idx)) for _gname, resolved_call in _register_calls(suite_res): if resolved_call.scheme_name in const_scheme_names: _emit_register_call(resolved_call, i2, errflg_local, lines) lines.append( - '{}num_consts = num_consts + size(scheme_consts, 1)'.format( - i2, + '{0}{1}({2})%items = [{1}({2})%items, scheme_consts]'.format( + i2, buf, inst_idx, ) ) lines.append('{}deallocate(scheme_consts)'.format(i2)) - # auto-clone-constituents: synthesised entries contribute a - # static count; add a literal at the end of the count pass. + # auto-clone-constituents: reserve n_auto_clone trailing slots after + # the scheme-registered entries, then instantiate into them. if n_auto_clone > 0: + lines.append('') lines.append( - '{}! auto-clone-constituents: legacy-shim synthesised entries'.format( + '{}! auto-clone-constituents: reserve + instantiate synthesised entries'.format( i2, ) ) - lines.append('{}num_consts = num_consts + {}'.format(i2, n_auto_clone)) - lines.append('') - lines.append('{}allocate({}({})%items(num_consts))'.format( - i2, buf, inst_idx, - )) - lines.append('{}num_consts = 0'.format(i2)) - lines.append('') - lines.append('{}! Second pass: copy into per-instance buffer'.format(i2)) - for _gname, resolved_call in _register_calls(suite_res): - if resolved_call.scheme_name in const_scheme_names: - _emit_register_call(resolved_call, i2, errflg_local, lines) - lines.append('{}do i = 1, size(scheme_consts, 1)'.format(i2)) + lines.append( + '{}num_consts = size({}({})%items, 1)'.format(i2, buf, inst_idx) + ) + if has_consts: + # Grow the existing scheme-registered slot by n_auto_clone. lines.append( - '{}{}({})%items(num_consts + i) = scheme_consts(i)'.format( - i2 + _INDENT, buf, inst_idx, + '{}call move_alloc({}({})%items, scheme_consts)'.format( + i2, buf, inst_idx, ) ) - lines.append('{}end do'.format(i2)) lines.append( - '{}num_consts = num_consts + size(scheme_consts, 1)'.format( - i2, + '{}allocate({}({})%items(num_consts + {}))'.format( + i2, buf, inst_idx, n_auto_clone, ) ) - lines.append('{}deallocate(scheme_consts)'.format(i2)) - # auto-clone-constituents: emit one synthesised %instantiate - # call per entry into the per-instance buffer slot. - if n_auto_clone > 0: - lines.append('') - lines.append( - '{}! auto-clone-constituents: legacy-shim synthesised %instantiate calls'.format( - i2, + lines.append( + '{0}if (num_consts > 0) {1}({2})%items(1:num_consts) = ' + 'scheme_consts(1:num_consts)'.format(i2, buf, inst_idx) + ) + lines.append( + '{}if (allocated(scheme_consts)) deallocate(scheme_consts)'.format(i2) + ) + else: + # No scheme-registered entries: just size the slot directly. + lines.append('{}deallocate({}({})%items)'.format(i2, buf, inst_idx)) + lines.append( + '{}allocate({}({})%items({}))'.format( + i2, buf, inst_idx, n_auto_clone, + ) ) - ) for entry in suite_res.auto_cloned_constituents: _emit_auto_clone_instantiate( entry, buf, inst_idx, i2, errflg_local, errmsg_local, lines, diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 73ef4ec0..34b6a7bd 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -3544,33 +3544,33 @@ def test_no_host_object_referenced(self): self.assertNotIn('%lock_table', self.text) self.assertNotIn('%new_field', self.text) - def test_two_pass_packs_into_buffer(self): + def test_register_called_once_per_scheme(self): register_body = self.text.split('subroutine reg_consts_register')[1].split( 'end subroutine reg_consts_register' )[0] - self.assertIn('First pass: count', register_body) - self.assertIn('Second pass: copy into per-instance buffer', register_body) - # The constituent-providing scheme is called twice (one per pass). + # Single-pass append: each constituent scheme's _register is called + # EXACTLY ONCE (the old count+copy two-pass called it twice and broke + # non-idempotent schemes such as prescribed_aerosols_register). self.assertEqual( - register_body.count('call register_constituents_register'), 2, + register_body.count('call register_constituents_register'), 1, ) + self.assertNotIn('First pass', register_body) + self.assertNotIn('Second pass', register_body) def test_buffer_allocate(self): # Outer wrapper-DDT array is sized to number_of_instances on first - # call; each instance allocates its own ``%items(num_consts)`` slot. + # call; each instance starts its own slot empty (``%items(0)``) and + # appends each scheme's constituents. self.assertIn( 'allocate(reg_consts_dynamic_constituents(', self.text, ) - self.assertIn( - 'allocate(reg_consts_dynamic_constituents(', - self.text, - ) - self.assertIn('%items(num_consts))', self.text) + self.assertIn('%items(0))', self.text) - def test_buffer_populate_loop(self): + def test_buffer_append(self): + # Each scheme's returned array is appended to the per-instance slot. self.assertIn( - '%items(num_consts + i) = scheme_consts(i)', + '%items = [reg_consts_dynamic_constituents(inst_num)%items, scheme_consts]', self.text, ) From 0fb1a1aaf02a02e3fffda679b3341211aae61ed8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 9 Jun 2026 21:00:31 -0600 Subject: [PATCH 67/74] Make thread_number / number_of_threads an opt-in pair like the instances pair --- capgen-ng/ccpp_capgen_ng.py | 120 ++++++++++++------ capgen-ng/generator/group_cap.py | 4 +- capgen-ng/metadata/registered_dimensions.py | 8 +- capgen-ng/metadata/variable_resolver.py | 22 ++-- doc/briefing.md | 27 +++- doc/constituents.md | 36 +++--- doc/migration.md | 89 +++++++++---- doc/redesign_prompt.md | 24 +++- .../chunked_data/chunked_data_scheme.F90 | 12 +- .../chunked_data/chunked_data_scheme.meta | 14 -- end-to-end-tests/chunked_data/data.meta | 12 -- end-to-end-tests/chunked_data/main.F90 | 2 +- .../sample_files/control_chunked_data.meta | 6 - .../sample_files/host_chunked_data.meta | 6 +- .../sample_files/scheme_chunked_data.meta | 7 - unit-tests/test_control_validation.py | 100 ++++++++++++++- unit-tests/test_integration.py | 13 +- 17 files changed, 326 insertions(+), 176 deletions(-) diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index cc02dc3f..94f2adcc 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -689,21 +689,44 @@ def _load_metadata_files( ('group_name', 'character', 'drives per-group dispatch inside ccpp_physics_* (each suite_cap emits a select case on this name)'), ('horizontal_loop_begin', 'integer', 'lower horizontal slice bound at scheme call sites'), ('horizontal_loop_end', 'integer', 'upper horizontal slice bound at scheme call sites'), - ('thread_number', 'integer', 'current thread number (pass 1 if single-threaded)'), - ('number_of_threads', 'integer', 'total thread count (pass 1 if single-threaded)'), ('number_of_physics_threads','integer', 'physics-internal thread budget (pass 1 if unused)'), ('ccpp_error_code', 'integer', 'CCPP error flag'), ('ccpp_error_message', 'character', 'CCPP error message'), ] - -# Optional control variables that must be declared as a *pair*. Hosts that -# need a multi-instance API declare both ``instance_number`` (the index) and -# ``number_of_instances`` (the bound). Hosts that don't may omit both; the -# generator will emit a single-instance API and dimension all per-instance -# arrays to length 1. Declaring exactly one is an error. +# NOTE: the threading index/count (``thread_number`` / ``number_of_threads``) +# is NOT required — it is a paired-optional control pair, fully symmetric with +# (``instance_number`` / ``number_of_instances``); see +# ``_PAIRED_OPTIONAL_CTRL_VARS`` below. ``number_of_physics_threads`` is a +# separate, unpaired scheme-facing scalar that stays unconditionally required. + +# Paired-optional control variables. Each entry is an (index, count) pair: +# the host declares BOTH members (in ``type=control``) or NEITHER; declaring +# exactly one is a hard error. Declaring a pair opts the host into that +# multi- API — the index flows as a per-call control dummy and the count +# gives the bound. When a pair is absent the public API drops both args and +# the framework uses literal ``1`` wherever the index would appear. A host +# variable may be dimensioned by the count standard name only when its pair is +# declared (otherwise the resolver's scalar-index collapse raises — it needs +# the index variable in scope). +# +# The two pairs are fully symmetric (decision 2026-06-09): +# * (instance_number, number_of_instances) — multi-instance API. The +# framework reads ``number_of_instances`` at register/init to size its +# own per-instance state (``ccpp_suite_data(:)``, ``ccpp_group_state(:)``). +# * (thread_number, number_of_threads) — multi-threading API. +# ``thread_number`` indexes host-owned per-thread containers; +# ``number_of_threads`` is carried as a control dummy (the framework owns +# no per-thread state yet, so its value is not consumed — kept for symmetry +# with ``number_of_instances`` and future per-thread sizing). +# (A chunk/block index is intentionally NOT a control pair: capgen-ng's +# slice-based design passes the current chunk as a horizontal range via +# horizontal_loop_begin/end, so no scheme ever indexes by chunk inside a call.) +# Each entry: (index std_name, count std_name, index description, count description). _PAIRED_OPTIONAL_CTRL_VARS = [ - ('instance_number', 'integer', 'current model instance index'), - ('number_of_instances', 'integer', 'total number of model instances'), + ('instance_number', 'number_of_instances', + 'current model instance index', 'total number of model instances'), + ('thread_number', 'number_of_threads', + 'current thread index', 'total thread count'), ] @@ -775,39 +798,58 @@ def _check_control_var(std_name, expected_type, description, required: bool) -> for std_name, expected_type, description in _REQUIRED_CTRL_VARS: _check_control_var(std_name, expected_type, description, required=True) - # Paired optional: both ``instance_number`` (the per-call index) and - # ``number_of_instances`` (the bound, used at register time to size - # the per-instance state arrays) live in ``type=control``. Symmetric - # with the (thread_number, number_of_threads) pair. Either both - # declared or neither. - _check_control_var( - 'instance_number', 'integer', - 'current model instance index', required=False, - ) - _check_control_var( - 'number_of_instances', 'integer', - 'total number of model instances', required=False, - ) + # Paired-optional control pairs (see _PAIRED_OPTIONAL_CTRL_VARS): for each + # (index, count) pair the host declares both members in a type=control + # table or neither. Declaring exactly one is an error. Both pairs — + # (instance_number, number_of_instances) and (thread_number, + # number_of_threads) — are validated identically. + for idx_name, cnt_name, idx_desc, cnt_desc in _PAIRED_OPTIONAL_CTRL_VARS: + _check_control_var(idx_name, 'integer', idx_desc, required=False) + _check_control_var(cnt_name, 'integer', cnt_desc, required=False) + + idx_present = host_dict.get(idx_name) is not None + cnt_present = host_dict.get(cnt_name) is not None + if idx_present ^ cnt_present: + present, missing = ( + (idx_name, cnt_name) if idx_present else (cnt_name, idx_name) + ) + errors.append( + "Host '{}' declares '{}' in a type=control table but is " + "missing its paired variable '{}' (which must also be in a " + "type=control table).\n" + " '{}' and '{}' are a paired-optional control pair: declare " + "both members to opt into that API, or neither.".format( + host_name, present, missing, idx_name, cnt_name, + ) + ) - inst_present = host_dict.get('instance_number') is not None - ninst_present = host_dict.get('number_of_instances') is not None - if inst_present ^ ninst_present: - present, missing = ( - ('instance_number', 'number_of_instances') - if inst_present - else ('number_of_instances', 'instance_number') - ) - errors.append( - "Host '{}' declares '{}' (in a type=control table) but is " - "missing the paired variable '{}' (which must also be in a " - "type=control table).\n" - " Declare both for a multi-instance API, or neither for a " - "single-instance API.".format(host_name, present, missing) - ) + # Control-table allowlist: a type=control table may declare ONLY the + # framework's known control variables — the unconditionally required set + # plus the members of the paired-optional pairs. Anything else in a + # type=control table is a hard error. Host-specific quantities that + # schemes consume belong in a type=host table; the subcycle loop variables + # (ccpp_loop_counter / ccpp_loop_extent) are generator-owned locals the + # host never declares. + allowed_control = {name for name, _type, _desc in _REQUIRED_CTRL_VARS} + for idx_name, cnt_name, _idesc, _cdesc in _PAIRED_OPTIONAL_CTRL_VARS: + allowed_control.add(idx_name) + allowed_control.add(cnt_name) + for std_name, entry in sorted(host_dict.items()): + if entry.is_control and std_name not in allowed_control: + errors.append( + "Variable '{}' is declared in a type=control table for host " + "'{}' but is not a recognized framework control variable.\n" + " A type=control table may declare only: {}.\n" + " If '{}' is a host quantity that schemes consume, declare it " + "in a type=host table instead.".format( + std_name, host_name, ', '.join(sorted(allowed_control)), + std_name, + ) + ) if errors: raise CCPPError( - "Host '{}' is missing required control variables:\n\n{}".format( + "Host '{}' has invalid control-variable metadata:\n\n{}".format( host_name, '\n\n'.join("ERROR: " + e for e in errors), ) diff --git a/capgen-ng/generator/group_cap.py b/capgen-ng/generator/group_cap.py index f0a2f3e1..8d25f0cb 100644 --- a/capgen-ng/generator/group_cap.py +++ b/capgen-ng/generator/group_cap.py @@ -57,8 +57,8 @@ 'group_name', 'horizontal_loop_begin', 'horizontal_loop_end', - 'thread_number', - 'number_of_threads', + 'thread_number', # paired-optional (with number_of_threads); ordered here when declared + 'number_of_threads', # paired-optional; ordered here when declared 'number_of_physics_threads', 'ccpp_error_code', 'ccpp_error_message', diff --git a/capgen-ng/metadata/registered_dimensions.py b/capgen-ng/metadata/registered_dimensions.py index cba31f29..ee716d7b 100644 --- a/capgen-ng/metadata/registered_dimensions.py +++ b/capgen-ng/metadata/registered_dimensions.py @@ -140,9 +140,11 @@ 'number_of_instances': 'instance_number', # Per-thread DDT containers (e.g. ``physics%Interstitial(thread_number)``) - # — the host's openmp-thread index. ``thread_number`` is a required - # control variable (see doc/migration.md §3.1), so any host using - # this dim already has the paired index in scope. + # — the host's openmp-thread index. (thread_number, number_of_threads) is + # a paired-optional control pair (see ccpp_capgen_ng._PAIRED_OPTIONAL_CTRL_VARS + # and doc/migration.md §3.1). A host that dimensions a variable by + # ``number_of_threads`` MUST declare the pair — otherwise the collapse + # below cannot find ``thread_number`` and raises. 'number_of_threads': 'thread_number', } diff --git a/capgen-ng/metadata/variable_resolver.py b/capgen-ng/metadata/variable_resolver.py index 676fb343..d7ca1a97 100644 --- a/capgen-ng/metadata/variable_resolver.py +++ b/capgen-ng/metadata/variable_resolver.py @@ -108,13 +108,13 @@ def _split_local_name(local_name: str): """Split a local name into (base, subscript) tuple. For plain identifiers returns (local_name, ''). - For slice expressions like ``chunk_begin(ccpp_chunk_number)`` returns - (``'chunk_begin'``, ``'ccpp_chunk_number'``). + For slice expressions like ``field(idx)`` returns + (``'field'``, ``'idx'``). - >>> _split_local_name('chunk_begin') - ('chunk_begin', '') - >>> _split_local_name('chunk_begin(ccpp_chunk_number)') - ('chunk_begin', 'ccpp_chunk_number') + >>> _split_local_name('field') + ('field', '') + >>> _split_local_name('field(idx)') + ('field', 'idx') >>> _split_local_name('q(:,:,index_of_water_vapor_specific_humidity)') ('q', ':,:,index_of_water_vapor_specific_humidity') """ @@ -134,11 +134,11 @@ def _resolve_subscript(subscript: str, host_dict: Dict[str, 'HostVarEntry']) -> >>> from collections import namedtuple >>> E = namedtuple('E', ['local_name']) - >>> d = {'ccpp_chunk_number': E('inst_num')} - >>> _resolve_subscript('ccpp_chunk_number', d) - 'inst_num' - >>> _resolve_subscript(':, ccpp_chunk_number', d) - ':, inst_num' + >>> d = {'thread_number': E('thrd_no')} + >>> _resolve_subscript('thread_number', d) + 'thrd_no' + >>> _resolve_subscript(':, thread_number', d) + ':, thrd_no' >>> _resolve_subscript('1', d) '1' """ diff --git a/doc/briefing.md b/doc/briefing.md index 7de7a855..e576f54e 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -110,7 +110,9 @@ For each scheme arg: - `ccpp_validator.py` — the standalone Fortran-vs-metadata checker. The ONE place capgen-ng parses Fortran. Run by developers / CMake before generation. Checks per-arg `intent`, `type`, `kind`, - and dimension rank; `character len=*` is a wildcard; DDT and + and dimension rank; character length must match exactly + (`len=*`↔`len=*`, `len=N`↔`len=N`, no wildcard; old-style F77 + `character*N` / `character*(*)` parsed); DDT and `external::` types compare against the Fortran `type(name)` wrapper. `optional` is asymmetric: metadata `optional=True` against a Fortran-required dummy is an error; the @@ -200,11 +202,23 @@ Every host MUST declare scalar integers (and one character) with these CCPP standard names: - `suite_name`, `horizontal_loop_begin`, `horizontal_loop_end`, - `thread_number`, `number_of_threads`, `number_of_physics_threads`, - `ccpp_error_code`, `ccpp_error_message`. + `number_of_physics_threads`, `ccpp_error_code`, `ccpp_error_message`. -Optional (paired): `instance_number` (control) + -`number_of_instances` (host). +**Two paired-optional control pairs** — declare *both* members of a +pair (in `type = control`) or *neither*; declaring exactly one is a +hard error: + +- `instance_number` + `number_of_instances` → opt into the + multi-instance API. +- `thread_number` + `number_of_threads` → opt into the multi-threading + API. + +The two pairs are fully symmetric. Declaring a pair makes the index a +per-call control argument; omitting it drops both args and the +framework uses literal `1` where the index would go. A host variable +may be dimensioned by `number_of_instances` / `number_of_threads` only +when its pair is declared (otherwise the scalar-index collapse can't +find the index variable and errors). ### 6.5 DDT-instance variables with scalar-index dims @@ -387,7 +401,8 @@ don't rebuild downstream objects unless something actually moved. - **Validator** now checks per-argument `intent`, `type`, `kind`, and dimension rank in addition to the original name/count check. Asymmetric `optional` rule, DDT + `external::` - type normalisation, character `len=*` wildcard. + type normalisation, exact character-length match (no `len=*` + wildcard; old-style F77 `character*N` parsed). - **Resolver cross-metadata checks** (late 2026-05-20): host/scheme (and suite-owned-var first-writer/follow-on) consistency on type, rank, and per-position dimension entries. Default lower bound has diff --git a/doc/constituents.md b/doc/constituents.md index b888fb6e..6a8a6507 100644 --- a/doc/constituents.md +++ b/doc/constituents.md @@ -600,28 +600,28 @@ constituent array into the suite's `_dynamic_constituents` buffer (USE'd from `ccpp_host_constituents`): ```fortran +! Outer wrapper sized to number_of_instances on first call (any instance). if (.not. allocated(_dynamic_constituents)) then - ! First-instance-only two-pass count + populate. - num_consts = 0 - call _register(scheme_consts=scheme_consts, ...) - num_consts = num_consts + size(scheme_consts, 1) - deallocate(scheme_consts) - ... - allocate(_dynamic_constituents(num_consts)) - num_consts = 0 - call _register(scheme_consts=scheme_consts, ...) - do i = 1, size(scheme_consts, 1) - _dynamic_constituents(num_consts + i) = scheme_consts(i) - end do - num_consts = num_consts + size(scheme_consts, 1) - deallocate(scheme_consts) - ... + allocate(_dynamic_constituents(number_of_instances)) end if + +! Single pass: call each scheme's _register EXACTLY ONCE and append its +! returned array to THIS instance's slot. +allocate(_dynamic_constituents(inst)%items(0)) +call _register(dyn_const=scheme_consts, ...) +if (errflg /= 0) return +_dynamic_constituents(inst)%items = & + [_dynamic_constituents(inst)%items, scheme_consts] +deallocate(scheme_consts) +! ... one block like the above per constituent-registering scheme ... ``` -The buffer is **shared across instances** (registration is identical -per instance); only the first instance to call `_register` -populates it. The host-wide merge happens in +Each instance owns its own `%items` slot (the per-instance buffer, so +`ccpp_register_constituents` can `set_const_index` independently per +instance); the suite state-machine guard ensures each instance populates +it exactly once. Each scheme's `_register` is called **exactly once** — +it may safely allocate persistent module state (the earlier two-pass +count+copy called it twice). The host-wide merge happens in `ccpp_register_constituents`. ### Group-cap call sites diff --git a/doc/migration.md b/doc/migration.md index 72fe4587..084eb80d 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -206,6 +206,17 @@ consumers/writers of the same variable may use `len=*` as a wildcard. Error messages name the source as `host`, `control`, or `suite` so you know whose contract you're violating. +**Suite-owned storage is never default-initialized** — by design. +capgen-ng emits the `ccpp__data` components with no default value. +An `intent(out)` argument is the scheme's contract to define that variable +on *every* return path; the framework will not paper over an unset output +the way original capgen's zero-initialized interstitials did. A ported +scheme that returns early (e.g. a `fixed_scon` branch) without assigning +one of its `intent(out)` dummies leaves the suite-owned storage undefined, +and a later consumer reads garbage (in a debug build, often a trap value). +This is a common porting hazard original capgen used to mask — audit +early-return paths for unset `intent(out)` args. + #### 1.3.3 `allocatable` and who owns suite-data allocation A **suite-owned variable** (an interstitial: first written by a scheme @@ -285,22 +296,34 @@ matches. - **Instance-dim used without `instance_number`**: error explains the paired-opt-in requirement (see §1.7). -### 1.7 Optional `instance_number` / `number_of_instances` pair - -These two control variables are now **paired optional** and both live -in the host's `type=control` table (symmetric with the -`thread_number` / `number_of_threads` pair): - -- Declare **both** in `type=control` → multi-instance API. Both flow - as control dummies through every lifecycle and physics-phase - signature. -- Declare **neither** → single-instance API. Public entry points drop - both args; internal per-instance arrays size to length 1. -- Declare exactly one → hard error from the validator. -- Declare `number_of_instances` in `type=host` → hard error - (must be `type=control`). - -Hosts that don't need multi-instance bookkeeping can drop both declarations. +### 1.7 Paired-optional control pairs (instances and threads) + +There are two symmetric `(index, count)` control pairs: +`instance_number` / `number_of_instances` and `thread_number` / +`number_of_threads`. Both behave identically: + +- Declare **both** members in `type=control` → opt into that paired + (multi-instance / multi-threading) API. Both flow as control dummies + through every lifecycle and physics-phase signature. +- Declare **neither** → the single API. Public entry points drop both + args; where the index would appear the framework uses literal `1` + (and, for instances, internal per-instance arrays size to length 1). +- Declare exactly one of a pair → hard error from the validator. +- Declare either count in `type=host` → hard error (must be + `type=control`). +- Dimension a host variable by `number_of_instances` / + `number_of_threads` without declaring its pair → hard error (the + scalar-index collapse needs the index variable in scope; see §3.4). + +Hosts that need neither multi-instance nor multi-threading can drop +both pairs entirely. + +> Why symmetric: `instance_number` indexes framework-owned per-instance +> state, so `number_of_instances` is read at register/init to size it. +> `thread_number` indexes host-owned per-thread containers; the +> framework doesn't yet read `number_of_threads`, but it's carried as a +> control dummy so the framework can size per-thread state in future — +> exactly as it does for instances today. ### 1.8 Deprecated standard names rewritten by `--legacy-mode` @@ -556,18 +579,28 @@ Every host's `type=control` table must declare: | `suite_name` | character | Drives suite dispatch | | `horizontal_loop_begin` | integer | Lower chunk-bound | | `horizontal_loop_end` | integer | Upper chunk-bound | -| `thread_number` | integer | Current thread | -| `number_of_threads` | integer | Total threads | | `number_of_physics_threads` | integer | Physics-internal budget | | `ccpp_error_code` | integer | Error flag | | `ccpp_error_message` | character | Error message | -Optional (paired — see §1.7): +Paired-optional — two symmetric `(index, count)` pairs (see §1.7). +For each pair, declare **both** members in `type=control` or +**neither**; declaring exactly one is a hard error: | Standard name | Fortran type | Table type | Purpose | |-------------------------|--------------|------------|--------------------------------| | `instance_number` | integer | control | Current instance index | | `number_of_instances` | integer | control | Total instance count | +| `thread_number` | integer | control | Current thread / per-thread-container index | +| `number_of_threads` | integer | control | Total thread count | + +**The `type=control` table is a closed set.** It may contain *only* +the variables in the two tables above (the 7 required plus the 4 +paired-optional pair members — 11 standard names total). Any other +variable in a `type=control` table is a hard error: a host quantity +that schemes consume belongs in a `type=host` table, and the subcycle +loop variables (`ccpp_loop_counter` / `ccpp_loop_extent`) are +generator-owned locals you never declare. ### 3.2 Required entry-point call sequence @@ -590,7 +623,7 @@ The `(instance_number, number_of_instances)` pair appears in every signature only when the host declares it (§1.7). Both flow uniformly through lifecycle and physics-phase calls; the framework consumes `number_of_instances` only at register/init time but carries it -elsewhere for API symmetry with `(thread_number, number_of_threads)`. +elsewhere for API symmetry. ### 3.3 Module-name convention (host, scheme, and DDT tables) @@ -1002,6 +1035,13 @@ the buffer from creation — no ownership transfer call needed. name; a constituent-flagged `intent=out` that is not a `tendency_of_*` is a codegen error. A scheme that only READS a constituent or a `tendency_of_` need not re-flag it — see §6.5. +- **`_register` is called exactly once per scheme** (2026-06-08). + capgen-ng packs each constituent scheme's returned + `ccpp_constituent_properties_t(:)` array into the per-suite buffer in a + single append pass, so a register routine may safely allocate persistent + module state. (An earlier two-pass count+copy called register twice and + broke any non-idempotent register, e.g. `prescribed_aerosols_register` + allocating a module-level map.) ### 6.3 Host metadata wins over auto-provisioning (2026-05-12) @@ -1147,7 +1187,7 @@ For every `(scheme, phase)` declared in the supplied `.meta` files: |------------|----------| | `intent` | Strict match (`in` / `out` / `inout`). Metadata declares it but Fortran omits → error. | | `type` | Case-insensitive match. `double precision` / `doubleprecision` / `double precision` are normalized to the same form. DDT names match the Fortran `type(name)` / `class(name)` wrapper — metadata `type = ty_rad_lw` matches Fortran `type(ty_rad_lw)`. External types match by typename — metadata `type = external:mpi_f08:mpi_comm` matches Fortran `type(mpi_comm)` (the module qualifier is metadata-only). | - | `kind` | Case-insensitive match. **Character `len=*` is a wildcard** on either side — matches any concrete `len=N` or `len=:`. | + | `kind` | Case-insensitive match. **Character length must be CONSISTENT** — the metadata mirrors the Fortran exactly: `len=*` matches only `len=*`, and `len=N` only the identical `len=N` (no wildcarding; changed 2026-06-08 — the validator runs first, so a Fortran `len=*` can only pair a metadata `len=*`, and vice versa). Old-style F77 forms (`character*64`, `character*(*)`, per-entity `c*5` / `d(10)*8`) are normalized to the `len=` form before comparison. | | `rank` | Number of dimensions only. Reads both `dimension(...)` line attributes and var-attached `foo(:,:)` syntax. Per-dimension bound comparison is NOT done. | ### 7.2 Asymmetric `optional` rule @@ -1212,7 +1252,12 @@ The same per-attribute rules apply that the scheme-side check uses, which is one rule fewer than the scheme side: there is no `optional` flag on module vars / DDT components, and host metadata carries no `intent`, so the asymmetric-optional rule (§7.2) does not apply here. -Character `len=*` remains a wildcard against any concrete `len=N`. +Character length is matched exactly (§7.1). Additionally, because +host / DDT metadata *defines* its character storage (a module variable +or a derived-type component, neither of which may be assumed-length), +`len=*` is rejected there outright — host and DDT character variables +must declare a concrete `len=N`. Assumed length is valid only for +dummy arguments (scheme args and control/lifecycle variables). --- diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index 52640900..ad9ed41c 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -248,17 +248,27 @@ character arguments. | `suite_name` | `character` | Suite name for runtime dispatch | | `horizontal_loop_begin` | `integer` | Start of horizontal slice (chunk bounds for `ccpp_physics_run`; `1` for all other phases) | | `horizontal_loop_end` | `integer` | End of horizontal slice (chunk bounds for `ccpp_physics_run`; `ncols` for all other phases) | -| `thread_number` | `integer` | Current thread index (1..number_of_threads); pass `1` if single-threaded | -| `number_of_threads` | `integer` | Host blocking loop thread count; pass `1` if single-threaded | | `number_of_physics_threads` | `integer` | Thread budget for physics-internal OpenMP; pass `1` if none | | `ccpp_error_message` | `character` | Error message string | | `ccpp_error_code` | `integer` | Integer error return code | -`instance_number` is **paired-optional** with `number_of_instances` (host -table, §3.6): declare both for a multi-instance API, declare neither for a -single-instance API. Declaring exactly one is a hard error. When the pair -is absent, the static API signatures drop the `instance_number` argument -entirely and per-instance state arrays size to 1. +**Two symmetric paired-optional `(index, count)` control pairs** — +`instance_number` / `number_of_instances` and `thread_number` / +`number_of_threads`. For each pair, declare **both** members in +`type=control` for the multi-instance / multi-threading API, or +**neither** for the single API; declaring exactly one is a hard error. +When a pair is absent, the static API drops the index argument and the +framework uses literal `1` where it would appear (and, for instances, +per-instance state arrays size to 1). A host variable may be +dimensioned by a count standard name only when its pair is declared — +the `SCALAR_INDEX_DIMS` collapse substitutes the index local name +(`instance_number` / `thread_number`) and errors if it isn't in scope. + +The asymmetry between the pairs is in *who reads the count*, not in the +rules: the framework reads `number_of_instances` to size its own +per-instance state; it does not yet read `number_of_threads` +(per-thread containers are host-owned) but carries it for future +symmetry. `group_name` is **not** in the required set. It is included in the static API signature only if the host declares it in their `type=control` table. When absent: the static API diff --git a/end-to-end-tests/chunked_data/chunked_data_scheme.F90 b/end-to-end-tests/chunked_data/chunked_data_scheme.F90 index 577ee01c..e9d222ba 100644 --- a/end-to-end-tests/chunked_data/chunked_data_scheme.F90 +++ b/end-to-end-tests/chunked_data/chunked_data_scheme.F90 @@ -63,19 +63,17 @@ end subroutine chunked_data_scheme_timestep_init !! \section arg_table_chunked_data_scheme_run Argument Table !! \htmlinclude chunked_data_scheme_run.html !! - subroutine chunked_data_scheme_run(nchunk, nchunks, data_array, errmsg, errflg) + subroutine chunked_data_scheme_run(data_array, errmsg, errflg) character(len=*), intent(out) :: errmsg integer, intent(out) :: errflg - integer, intent(in) :: nchunk, nchunks integer, intent(in) :: data_array(:) ! Initialize CCPP error handling variables errmsg = '' errflg = 0 - ! Check size of data array - write(error_unit, '(2(a,i3))') 'In chunked_data_scheme_run: checking size of data array for chunk', & - nchunk, '/', nchunks, ' to be', data_array_sizes(nchunk) - if (size(data_array)/=data_array_sizes(nchunk)) then - write(errmsg, '(a,i4)') "Error in chunked_data_scheme_run, expected size(data_array)==6, got ", size(data_array) + ! Check size of data array slice + write(error_unit, '(a,i3)') 'In chunked_data_scheme_run: checking size of data array slice', size(data_array) + if (.not. any(size(data_array)==data_array_sizes)) then + write(errmsg, '(a,i4)') "Error in chunked_data_scheme_run, unexpected size(data_array)=", size(data_array) errflg = 1 return end if diff --git a/end-to-end-tests/chunked_data/chunked_data_scheme.meta b/end-to-end-tests/chunked_data/chunked_data_scheme.meta index a74bed43..5fb56556 100644 --- a/end-to-end-tests/chunked_data/chunked_data_scheme.meta +++ b/end-to-end-tests/chunked_data/chunked_data_scheme.meta @@ -76,20 +76,6 @@ dimensions = () type = integer intent = out -[nchunk] - standard_name = ccpp_chunk_number - long_name = number of chunk for chunked arrays in CCPP - units = index - dimensions = () - type = integer - intent = in -[nchunks] - standard_name = ccpp_chunk_extent - long_name = number of chunks of array data used in run phase - units = count - dimensions = () - type = integer - intent = in [data_array] standard_name = chunked_data_array long_name = chunked data array diff --git a/end-to-end-tests/chunked_data/data.meta b/end-to-end-tests/chunked_data/data.meta index 38f8bfc2..af3cb184 100644 --- a/end-to-end-tests/chunked_data/data.meta +++ b/end-to-end-tests/chunked_data/data.meta @@ -25,18 +25,6 @@ units = count dimensions = () type = integer -[nchunk] - standard_name = ccpp_chunk_number - long_name = number of current chunk - units = index - dimensions = () - type = integer -[nchunks] - standard_name = ccpp_chunk_extent - long_name = number of chunks of array data used in run phase - units = count - dimensions = () - type = integer [chunked_data_instance] standard_name = chunked_data_type_instance long_name = instance of derived data type chunked_data_type diff --git a/end-to-end-tests/chunked_data/main.F90 b/end-to-end-tests/chunked_data/main.F90 index 0845cfd2..36201afe 100644 --- a/end-to-end-tests/chunked_data/main.F90 +++ b/end-to-end-tests/chunked_data/main.F90 @@ -107,7 +107,7 @@ program test_chunked_data !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! !cdata => ccpp_data_domain - call ccpp_physics_final(lb=1, ub=ncols, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) + call ccpp_physics_final(lb=1, ub=ncols, nthreads=1, nphys_threads=1, thread_num=1, suite_name=trim(ccpp_suite), group_name='all', errmsg=errmsg, errflg=errflg) if (errflg/=0) then write(error_unit, '(a)') "An error occurred in ccpp_physics_finalize:" write(error_unit, '(a)') trim(errmsg) diff --git a/unit-tests/sample_files/control_chunked_data.meta b/unit-tests/sample_files/control_chunked_data.meta index 4873c759..e7a4c47c 100644 --- a/unit-tests/sample_files/control_chunked_data.meta +++ b/unit-tests/sample_files/control_chunked_data.meta @@ -34,12 +34,6 @@ units = index dimensions = () type = integer -[ nchunk ] - standard_name = ccpp_chunk_number - long_name = current chunk number - units = index - dimensions = () - type = integer [ thrd_no ] standard_name = thread_number long_name = current thread number diff --git a/unit-tests/sample_files/host_chunked_data.meta b/unit-tests/sample_files/host_chunked_data.meta index 278602af..df3eba11 100644 --- a/unit-tests/sample_files/host_chunked_data.meta +++ b/unit-tests/sample_files/host_chunked_data.meta @@ -1,7 +1,7 @@ # Host metadata for the chunked_data integration test. -# Tests a scheme that uses ccpp_chunk_number as a control variable and -# accesses a DDT-based data array. Chunk loop bounds are provided as -# scalars (the host manages chunking externally). +# Tests a scheme that accesses a DDT-based data array, called once per +# chunk by a host that manages chunking externally (the current chunk is +# passed as a horizontal range, not a control variable). [ccpp-table-properties] name = chunked_data_mod diff --git a/unit-tests/sample_files/scheme_chunked_data.meta b/unit-tests/sample_files/scheme_chunked_data.meta index f37e46a6..964d6bb4 100644 --- a/unit-tests/sample_files/scheme_chunked_data.meta +++ b/unit-tests/sample_files/scheme_chunked_data.meta @@ -80,13 +80,6 @@ dimensions = () type = integer intent = out -[ nchunk ] - standard_name = ccpp_chunk_number - long_name = current chunk number - units = index - dimensions = () - type = integer - intent = in [ data_array ] standard_name = chunked_data_array long_name = chunked data array diff --git a/unit-tests/test_control_validation.py b/unit-tests/test_control_validation.py index 2c4455d4..3d381089 100644 --- a/unit-tests/test_control_validation.py +++ b/unit-tests/test_control_validation.py @@ -14,6 +14,7 @@ import os import sys import tempfile +import types import unittest _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -63,11 +64,11 @@ def _build_host_dict(host_files, control_files, ddt_files=None): # --------------------------------------------------------------------------- class TestMissingControlVars(unittest.TestCase): - """All required vars missing except suite_name → 7 errors collected. + """All required vars missing except suite_name → 6 errors collected. - ``instance_number`` is *not* in the required list (it pairs with - ``number_of_instances`` as an opt-in for multi-instance hosts), so - its absence does NOT raise here. + Neither pair member is in the required list: ``instance_number`` / + ``number_of_instances`` and ``thread_number`` / ``number_of_threads`` are + both paired-optional opt-ins, so their absence does NOT raise here. """ def setUp(self): @@ -81,11 +82,16 @@ def test_raises_ccpp_error(self): _validate_required_control_vars('test_host', self._host_dict) def test_all_missing_vars_reported(self): - """All 8 missing required standard names appear in the error message.""" + """All 6 missing required standard names appear in the error message. + + ``thread_number`` and ``number_of_threads`` are intentionally absent + from this list: they form a paired-optional control pair (symmetric + with the instance pair), not required vars, so their omission is not + reported as a missing required var.""" missing = [ 'group_name', 'horizontal_loop_begin', 'horizontal_loop_end', - 'thread_number', 'number_of_threads', 'number_of_physics_threads', + 'number_of_physics_threads', 'ccpp_error_code', 'ccpp_error_message', ] try: @@ -218,6 +224,20 @@ def test_no_instance_pair_passes(self): self.assertNotIn('instance_number', host_dict) self.assertNotIn('number_of_instances', host_dict) + def test_no_thread_pair_passes(self): + """Host omitting BOTH thread_number AND number_of_threads passes — + the multi-threading API is opt-in, symmetric with the instance pair.""" + host_dict = _build_host_dict( + host_files=[_sf('host_simple.meta')], + control_files=[_sf('control_full.meta')], + ) + host_dict.pop('thread_number', None) + host_dict.pop('number_of_threads', None) + # Must not raise even though the thread pair is absent. + _validate_required_control_vars('test_host', host_dict) + self.assertNotIn('thread_number', host_dict) + self.assertNotIn('number_of_threads', host_dict) + class TestInstanceNumberPairing(unittest.TestCase): """instance_number and number_of_instances both live in type=control @@ -250,6 +270,74 @@ def test_ninstances_alone_raises(self): self.assertIn('number_of_instances', msg) self.assertIn('paired', msg.lower()) + +class TestThreadNumberPairing(unittest.TestCase): + """thread_number and number_of_threads are a paired-optional control pair, + fully symmetric with the instance pair: declaring exactly one is an error. + + These manipulate a parsed host_dict directly (rather than carrying extra + sample .meta files) since the only thing under test is the XOR check. + """ + + def _full_host_dict(self): + return _build_host_dict( + host_files=[_sf('host_simple.meta')], + control_files=[_sf('control_full.meta')], + ) + + def test_thread_number_alone_raises(self): + """control declares thread_number but not number_of_threads.""" + host_dict = self._full_host_dict() + host_dict.pop('number_of_threads', None) + with self.assertRaises(CCPPError) as ctx: + _validate_required_control_vars('test_host', host_dict) + msg = str(ctx.exception) + self.assertIn('thread_number', msg) + self.assertIn('number_of_threads', msg) + self.assertIn('paired', msg.lower()) + + def test_number_of_threads_alone_raises(self): + """control declares number_of_threads but not thread_number.""" + host_dict = self._full_host_dict() + host_dict.pop('thread_number', None) + with self.assertRaises(CCPPError) as ctx: + _validate_required_control_vars('test_host', host_dict) + msg = str(ctx.exception) + self.assertIn('thread_number', msg) + self.assertIn('number_of_threads', msg) + self.assertIn('paired', msg.lower()) + + +class TestControlAllowlist(unittest.TestCase): + """A type=control table may declare ONLY the known framework control + variables (the required set plus the paired-optional pair members). + Any other variable in a type=control table is a hard error.""" + + def _full_host_dict(self): + return _build_host_dict( + host_files=[_sf('host_simple.meta')], + control_files=[_sf('control_full.meta')], + ) + + def test_unknown_control_var_rejected(self): + host_dict = self._full_host_dict() + host_dict['some_random_host_quantity'] = types.SimpleNamespace( + is_control=True) + with self.assertRaises(CCPPError) as ctx: + _validate_required_control_vars('test_host', host_dict) + msg = str(ctx.exception) + self.assertIn('some_random_host_quantity', msg) + self.assertIn('type=host', msg) + + def test_unknown_host_table_var_not_flagged(self): + """A non-control (type=host) variable with an unknown name is fine — + the allowlist only governs type=control declarations.""" + host_dict = self._full_host_dict() + host_dict['some_random_host_quantity'] = types.SimpleNamespace( + is_control=False) + # Must not raise. + _validate_required_control_vars('test_host', host_dict) + # --------------------------------------------------------------------------- # Tests for forbidden dimension names # --------------------------------------------------------------------------- diff --git a/unit-tests/test_integration.py b/unit-tests/test_integration.py index ea3624a8..d5451739 100644 --- a/unit-tests/test_integration.py +++ b/unit-tests/test_integration.py @@ -1873,7 +1873,7 @@ def test_types_module_for_unit_conv(self): # --------------------------------------------------------------------------- class TestChunkedDataIntegration(unittest.TestCase): - """Suite with DDT-based host variable and ccpp_chunk_number control var.""" + """Suite with a DDT-based host variable accessed by a scheme.""" def setUp(self): self._tmpdir = tempfile.mkdtemp() @@ -1914,17 +1914,6 @@ def test_ddt_access_in_group_cap(self): # DDT field access should appear in the call expression. self.assertIn('chunked_data_instance%array_data', text) - def test_chunk_number_control_arg(self): - with open( - os.path.join( - self._tmpdir, - 'ccpp_chunked_data_chunked_data_group_cap.F90', - ) - ) as fh: - text = fh.read() - # ccpp_chunk_number → local name nchunk in the run subroutine. - self.assertIn('nchunk', text) - def test_no_types_module_for_chunked_data(self): self.assertFalse( os.path.isfile( From e2e25f5a5470685445ee9a8b3b910fe9f257658a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 11 Jun 2026 11:45:01 -0600 Subject: [PATCH 68/74] Add ccpp_constituent_prop_mod.F90.patch and doc/cam4_fwaut_constituent_order.md --- capgen-ng/ccpp_capgen_ng.py | 2 +- ccpp_constituent_prop_mod.F90.patch | 47 +++++++++ doc/cam4_fwaut_constituent_order.md | 151 ++++++++++++++++++++++++++++ 3 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 ccpp_constituent_prop_mod.F90.patch create mode 100644 doc/cam4_fwaut_constituent_order.md diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen-ng/ccpp_capgen_ng.py index 94f2adcc..178041e4 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen-ng/ccpp_capgen_ng.py @@ -690,7 +690,7 @@ def _load_metadata_files( ('horizontal_loop_begin', 'integer', 'lower horizontal slice bound at scheme call sites'), ('horizontal_loop_end', 'integer', 'upper horizontal slice bound at scheme call sites'), ('number_of_physics_threads','integer', 'physics-internal thread budget (pass 1 if unused)'), - ('ccpp_error_code', 'integer', 'CCPP error flag'), + ('ccpp_error_code', 'integer', 'CCPP error code'), ('ccpp_error_message', 'character', 'CCPP error message'), ] # NOTE: the threading index/count (``thread_number`` / ``number_of_threads``) diff --git a/ccpp_constituent_prop_mod.F90.patch b/ccpp_constituent_prop_mod.F90.patch new file mode 100644 index 00000000..fcea421a --- /dev/null +++ b/ccpp_constituent_prop_mod.F90.patch @@ -0,0 +1,47 @@ +--- capgen-ng/src/ccpp_constituent_prop_mod.F90 ++++ capgen-ng/src/ccpp_constituent_prop_mod.F90 +@@ -1392,6 +1392,17 @@ + type(ccpp_constituent_properties_t), pointer :: cprop + character(len=dimname_len) :: dimname + character(len=*), parameter :: subname = 'ccp_model_const_table_lock' ++ ! === ONE-OFF cam4 constituent-reorder experiment === ++ ! When .true., force the cam4 advected water species into original-capgen ++ ! order [cloud_liquid=1, cloud_ice=2, water_vapor=3] instead of hash-table ++ ! order, to prove the FWAUT b4b diff is driven purely by constituent order. ++ ! Only the 3 cam4 water-species std-names are remapped; everything else keeps ++ ! its normal hash-order index, so other suites are unaffected unless they ++ ! advect exactly these names. Flip to .false. (or delete) to restore. ++ logical, parameter :: l_const_reorder = .true. ++ integer :: const_pos ++ character(len=512) :: sname_reorder ++ ! === end experiment === + + astat = 0 + errcode_local = 0 +@@ -1460,9 +1471,24 @@ + errcode_local = errcode_local + 1 + exit + end if +- call cprop%set_const_index(index_advect, & ++ ! === ONE-OFF cam4 constituent-reorder experiment === ++ const_pos = index_advect ++ if (l_const_reorder) then ++ call cprop%standard_name(sname_reorder, & ++ errcode=errcode, errmsg=errmsg) ++ select case (trim(sname_reorder)) ++ case ('cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 1 ++ case ('cloud_ice_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 2 ++ case ('water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 3 ++ end select ++ end if ++ call cprop%set_const_index(const_pos, & + errcode=errcode, errmsg=errmsg) +- call this%const_metadata(index_advect)%set(cprop) ++ call this%const_metadata(const_pos)%set(cprop) ++ ! === end experiment === + else + index_const = index_const + 1 + if (index_const > num_vars) then diff --git a/doc/cam4_fwaut_constituent_order.md b/doc/cam4_fwaut_constituent_order.md new file mode 100644 index 00000000..c38f2c74 --- /dev/null +++ b/doc/cam4_fwaut_constituent_order.md @@ -0,0 +1,151 @@ +# cam4 (QPC4) bit-for-bit difference: root cause is constituent registration order + +**Status:** root cause found and **proven**. Decision requested from CAM-SIMA. +**Date:** 2026-06-11 **Author:** D. Heinzeller + +## Executive summary + +The CAM-SIMA test +`SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4` +(full cam4 physics on the MPAS dynamical core) fails its bit-for-bit (b4b) +comparison against the capgen baseline. The difference is **machine-epsilon +roundoff** — state and flux fields agree to 14–17 significant digits; the +comparison is loud only in RK-microphysics *ratio* diagnostics (e.g. `FWAUT`, +RMS ≈ 4.24e-2), which are ratios of two near-zero autoconversion rates and so +amplify any roundoff. The behavior is identical under GNU and Intel. + +The physics source, `suite_cam4.xml`, and `src/data/registry.xml` are +**byte-identical** between the two builds. The difference is purely in the +generated CCPP caps. We have traced it to a single cause and **proven** it: + +> **capgen-ng registers the advected constituents in a different order than the +> original capgen.** Specifically, `cloud_liquid` and `cloud_ice` +> are swapped. This changes the floating-point summation order in the energy/water +> thermodynamic diagnostics, which the energy fixer then spreads across all columns +> as a tiny, pervasive heating — the source of the b4b difference. + +A one-off patch that forces capgen-ng's advected water species into the +original-capgen order makes **QPC4 bit-for-bit identical** to the baseline. + +## The difference (runtime constituent list, `debug_output = 2`) + +| index | original capgen (baseline) | capgen-ng | +|------:|----------------------------|-----------| +| 1 | **cloud_liquid** (advected) | **cloud_ice** (advected) | +| 2 | **cloud_ice** (advected) | **cloud_liquid** (advected) | +| 3 | water_vapor (advected) | water_vapor (advected) | +| 4–10 | CFC12, O3, CH4, O2, N2O, CFC11, CO2 | CFC12, O2, CH4, CO2, O3, N2O, CFC11 | + +Indices 1–3 are the advected water species; 4–10 are non-advected trace gases. +The advected block is what matters (see mechanism). `water_vapor` is index 3 in +both — the only advected difference is the **cloud_liquid ↔ cloud_ice swap**. + +## Mechanism + +1. `air_composition` builds `thermodynamic_active_species_idx` by walking the + advected constituents in **constituent-index order**. +2. `get_hydrostatic_energy` (`cam_thermo`) sums the water species in that order. + Baseline sums `cloud_liquid + cloud_ice + water_vapor`; capgen-ng sums + `cloud_ice + cloud_liquid + water_vapor`. Same values, **different FP order**. +3. The resulting machine-eps difference in total energy/water is picked up by the + global energy fixer (`check_energy_fix`), which redistributes it as a uniform + heating across all columns. From that point the two runs differ at roundoff + level everywhere, surfacing loudly only in ratio diagnostics like `FWAUT`. + +`air_composition.F90` and `cam_constituents.F90` are byte-identical between the +two builds, so the entire difference originates in the registration order the +generated cap produces. In the CCPP framework, registration order is the +hash-table iteration order in `ccpp_model_constituents_t%lock_table` (advected +packed first) — i.e. an arbitrary, generator-dependent order, not a deliberate +physical ordering. + +## Proof + +Forcing capgen-ng's advected water species into the baseline order +`[cloud_liquid = 1, cloud_ice = 2, water_vapor = 3]` (a flag-guarded one-off +patch in the framework's `ccp_model_const_table_lock`) makes QPC4 reproduce the +ccpp-prebuild baseline **bit-for-bit** (cprnc: all fields identical). This +isolates constituent ordering as the *sole* cause. Patch (file `ccpp_constituent_prop_mod.F90.patch` in the top-level directory of the `feature/capgen-ng` ccpp-framework branch): + +``` +--- capgen-ng/src/ccpp_constituent_prop_mod.F90 ++++ capgen-ng/src/ccpp_constituent_prop_mod.F90 +@@ -1392,6 +1392,17 @@ + type(ccpp_constituent_properties_t), pointer :: cprop + character(len=dimname_len) :: dimname + character(len=*), parameter :: subname = 'ccp_model_const_table_lock' ++ ! === ONE-OFF cam4 constituent-reorder experiment === ++ ! When .true., force the cam4 advected water species into original-capgen ++ ! order [cloud_liquid=1, cloud_ice=2, water_vapor=3] instead of hash-table ++ ! order, to prove the FWAUT b4b diff is driven purely by constituent order. ++ ! Only the 3 cam4 water-species std-names are remapped; everything else keeps ++ ! its normal hash-order index, so other suites are unaffected unless they ++ ! advect exactly these names. Flip to .false. (or delete) to restore. ++ logical, parameter :: l_const_reorder = .true. ++ integer :: const_pos ++ character(len=512) :: sname_reorder ++ ! === end experiment === + + astat = 0 + errcode_local = 0 +@@ -1460,9 +1471,24 @@ + errcode_local = errcode_local + 1 + exit + end if +- call cprop%set_const_index(index_advect, & ++ ! === ONE-OFF cam4 constituent-reorder experiment === ++ const_pos = index_advect ++ if (l_const_reorder) then ++ call cprop%standard_name(sname_reorder, & ++ errcode=errcode, errmsg=errmsg) ++ select case (trim(sname_reorder)) ++ case ('cloud_liquid_water_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 1 ++ case ('cloud_ice_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 2 ++ case ('water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water') ++ const_pos = 3 ++ end select ++ end if ++ call cprop%set_const_index(const_pos, & + errcode=errcode, errmsg=errmsg) +- call this%const_metadata(index_advect)%set(cprop) ++ call this%const_metadata(const_pos)%set(cprop) ++ ! === end experiment === + else + index_const = index_const + 1 + if (index_const > num_vars) then +``` + +## Assessment — neither order is "wrong" + +Both builds register the same constituents with identical properties; the +ordering is not physically meaningful, and the resulting solutions are +roundoff-equivalent and both physically correct. The b4b failure reflects only +that capgen-ng's (arbitrary) order differs from the (equally arbitrary) order +the capgen baseline happened to produce. + +## Decision requested + +To resolve QPC4 (and any other case sensitive to constituent order), we propose: + +1. Give capgen-ng a **deterministic, documented** constituent-registration order + (e.g. water vapor first, with a clear rule for how constituents land in the + array) — replacing today's hash-bucket order. +2. Adopt the new documented order and **re-baseline** the affected CAM-SIMA cases once. + +The temporary proof patch will be removed once the path is agreed. + +## Artifacts + +- **Patch (git diff):** `` — + reproduce with + `git -C EXT/cam-sima-ng/ccpp_framework/capgen-ng diff src/ccpp_constituent_prop_mod.F90`. +- **Run directories (Derecho):** + - Baseline (original capgen): `` + - capgen-ng, unpatched (shows the FWAUT diff): `` + - capgen-ng + reorder patch (**b4b**): `` +- **cprnc summaries:** + - unpatched vs baseline: `` + - patched vs baseline: `` +- **Constituent lists (`debug_output = 2`, `atm.log`):** as tabulated above. From 5a0356e91d71642c7e17dde70794f662f8c65094 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 11 Jun 2026 13:38:33 -0600 Subject: [PATCH 69/74] Update doc/cam4_fwaut_constituent_order.md --- doc/cam4_fwaut_constituent_order.md | 55 +++++++++++++---------------- 1 file changed, 25 insertions(+), 30 deletions(-) diff --git a/doc/cam4_fwaut_constituent_order.md b/doc/cam4_fwaut_constituent_order.md index c38f2c74..4d207cff 100644 --- a/doc/cam4_fwaut_constituent_order.md +++ b/doc/cam4_fwaut_constituent_order.md @@ -65,8 +65,32 @@ Forcing capgen-ng's advected water species into the baseline order `[cloud_liquid = 1, cloud_ice = 2, water_vapor = 3]` (a flag-guarded one-off patch in the framework's `ccp_model_const_table_lock`) makes QPC4 reproduce the ccpp-prebuild baseline **bit-for-bit** (cprnc: all fields identical). This -isolates constituent ordering as the *sole* cause. Patch (file `ccpp_constituent_prop_mod.F90.patch` in the top-level directory of the `feature/capgen-ng` ccpp-framework branch): +isolates constituent ordering as the *sole* cause. See section "Artifacts" +below for the full patch. +## Assessment — neither order is "wrong" + +Both builds register the same constituents with identical properties; the +ordering is not physically meaningful, and the resulting solutions are +roundoff-equivalent and both physically correct. The b4b failure reflects only +that capgen-ng's (arbitrary) order differs from the (equally arbitrary) order +the capgen baseline happened to produce. + +## Decision requested + +To resolve QPC4 (and any other case sensitive to constituent order), we propose: + +1. Give capgen-ng a **deterministic, documented** constituent-registration order + (e.g. water vapor first, with a clear rule for how constituents land in the + array) — replacing today's hash-bucket order. +2. Adopt the new documented order and **re-baseline** the affected CAM-SIMA cases once. + +The temporary proof patch will be removed once the path is agreed. + +## Artifacts + +- **Patch:** Stored as `ccpp_constituent_prop_mod.F90.patch` in the top-level +directory of the `feature/capgen-ng` ccpp-framework branch): ``` --- capgen-ng/src/ccpp_constituent_prop_mod.F90 +++ capgen-ng/src/ccpp_constituent_prop_mod.F90 @@ -116,36 +140,7 @@ isolates constituent ordering as the *sole* cause. Patch (file `ccpp_constituent index_const = index_const + 1 if (index_const > num_vars) then ``` - -## Assessment — neither order is "wrong" - -Both builds register the same constituents with identical properties; the -ordering is not physically meaningful, and the resulting solutions are -roundoff-equivalent and both physically correct. The b4b failure reflects only -that capgen-ng's (arbitrary) order differs from the (equally arbitrary) order -the capgen baseline happened to produce. - -## Decision requested - -To resolve QPC4 (and any other case sensitive to constituent order), we propose: - -1. Give capgen-ng a **deterministic, documented** constituent-registration order - (e.g. water vapor first, with a clear rule for how constituents land in the - array) — replacing today's hash-bucket order. -2. Adopt the new documented order and **re-baseline** the affected CAM-SIMA cases once. - -The temporary proof patch will be removed once the path is agreed. - -## Artifacts - -- **Patch (git diff):** `` — - reproduce with - `git -C EXT/cam-sima-ng/ccpp_framework/capgen-ng diff src/ccpp_constituent_prop_mod.F90`. - **Run directories (Derecho):** - Baseline (original capgen): `` - capgen-ng, unpatched (shows the FWAUT diff): `` - capgen-ng + reorder patch (**b4b**): `` -- **cprnc summaries:** - - unpatched vs baseline: `` - - patched vs baseline: `` -- **Constituent lists (`debug_output = 2`, `atm.log`):** as tabulated above. From befff742ca5883da4eb5367fdac5fd1c5d1445ea Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 11 Jun 2026 13:42:58 -0600 Subject: [PATCH 70/74] Update doc/cam4_fwaut_constituent_order.md --- doc/cam4_fwaut_constituent_order.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/cam4_fwaut_constituent_order.md b/doc/cam4_fwaut_constituent_order.md index 4d207cff..d4112872 100644 --- a/doc/cam4_fwaut_constituent_order.md +++ b/doc/cam4_fwaut_constituent_order.md @@ -140,7 +140,8 @@ directory of the `feature/capgen-ng` ccpp-framework branch): index_const = index_const + 1 if (index_const > num_vars) then ``` -- **Run directories (Derecho):** - - Baseline (original capgen): `` - - capgen-ng, unpatched (shows the FWAUT diff): `` - - capgen-ng + reorder patch (**b4b**): `` +- **Run directories (Derecho):** Because the SIMA baselines change continuously, + - Baseline (original capgen, https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng-reference): `` + - capgen-ng differences to be evaluated against this baseline, because the official baseline changes frequently + - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng), unpatched (shows the FWAUT diff): `` + - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng) + reorder patch (**b4b**): `` From d568a4433662603fe2b3884d6e3ec4aa11703dbe Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 15 Jun 2026 17:13:30 -0700 Subject: [PATCH 71/74] Update doc/cam4_fwaut_constituent_order.md with code and run directories on Derecho --- doc/cam4_fwaut_constituent_order.md | 41 ++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/doc/cam4_fwaut_constituent_order.md b/doc/cam4_fwaut_constituent_order.md index d4112872..2cd72f1b 100644 --- a/doc/cam4_fwaut_constituent_order.md +++ b/doc/cam4_fwaut_constituent_order.md @@ -140,8 +140,41 @@ directory of the `feature/capgen-ng` ccpp-framework branch): index_const = index_const + 1 if (index_const > num_vars) then ``` -- **Run directories (Derecho):** Because the SIMA baselines change continuously, - - Baseline (original capgen, https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng-reference): `` + +- **Run directories (Derecho) Intel:** Because the SIMA baselines change continuously, + - Baseline (original capgen, https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng-reference): + - `/glade/derecho/scratch/heinzell/aux_sima_intel_20260614203021/` + - capgen-ng differences to be evaluated against this baseline, because the official baseline changes frequently + - Both the capgen baseline and the capgen-ng test fail for this test: +``` + SMS_Ln9.ne3pg3_ne3pg3_mg37.FKESSLER.derecho_intel.cam-outfrq_se_cslam_multitape (Overall: NLFAIL) details: + FAIL SMS_Ln9.ne3pg3_ne3pg3_mg37.FKESSLER.derecho_intel.cam-outfrq_se_cslam_multitape NLCOMP +``` + - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng), unpatched (shows the FWAUT diff): + - `/glade/derecho/scratch/heinzell/aux_sima_intel_20260614202951/` with the following `mpasa120_mpasa120.QPC4` test dirs: + - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951.ORIGINAL_NO_PATCH` + - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951.ORIGINAL_NO_PATCH` + - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng) + reorder patch (**b4b**): + - `/glade/derecho/scratch/heinzell/aux_sima_intel_20260614202951/` with the following `mpasa120_mpasa120.QPC4` test dirs: + - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951` + - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951` + +- **Run directories (Derecho) GNU:** Because the SIMA baselines change continuously, + - Baseline (original capgen, https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng-reference): + - `/glade/derecho/scratch/heinzell/aux_sima_gnu_20260611123848/` - capgen-ng differences to be evaluated against this baseline, because the official baseline changes frequently - - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng), unpatched (shows the FWAUT diff): `` - - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng) + reorder patch (**b4b**): `` + - Both the capgen baseline and the capgen-ng test fail for this test: +``` + SMS_Ln2.ne3pg3_ne3pg3_mg37.FPHYStest.derecho_gnu.cam-outfrq_hb_vdiff_derecho (Overall: FAIL) details: + FAIL SMS_Ln2.ne3pg3_ne3pg3_mg37.FPHYStest.derecho_gnu.cam-outfrq_hb_vdiff_derecho RUN time=13 + SMS_Ln9.ne3pg3_ne3pg3_mg37.FADIAB.derecho_gnu.cam-outfrq_se_cslam (Overall: FAIL) details: + FAIL SMS_Ln9.ne3pg3_ne3pg3_mg37.FADIAB.derecho_gnu.cam-outfrq_se_cslam RUN time=13 +``` + - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng), unpatched (shows the FWAUT diff): + - `/glade/derecho/scratch/heinzell/aux_sima_gnu_20260611123837/` with the following `mpasa120_mpasa120.QPC4` test dirs: + - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837.ORIGINAL_NO_PATCH` + - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837.ORIGINAL_NO_PATCH` + - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng) + reorder patch (**b4b**): + - `/glade/derecho/scratch/heinzell/aux_sima_gnu_20260611123837/` with the following `mpasa120_mpasa120.QPC4` test dirs: + - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837` + - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837` From 2d604c98bb0df9ddfc7fee1c852acb94edf488da Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 24 Jun 2026 10:49:46 -0600 Subject: [PATCH 72/74] Add doc/file_catalogue_DRAFT.md (will be moved into developer documentation later) --- doc/file_catalogue_DRAFT.md | 149 ++++++++++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 doc/file_catalogue_DRAFT.md diff --git a/doc/file_catalogue_DRAFT.md b/doc/file_catalogue_DRAFT.md new file mode 100644 index 00000000..aba0b3e0 --- /dev/null +++ b/doc/file_catalogue_DRAFT.md @@ -0,0 +1,149 @@ +# capgen-ng repository — file catalogue (DRAFT) + +> **Status: temporary draft for the code-walkthrough prep.** One row per file, except +> the many test/example *input* fixtures, which are collapsed. Once reviewed, the +> relevant sections will be folded into `README.md` / `doc/DevelopersGuide/`. +> External checkouts under `EXT/` (UFS reference + capgen-ng integration trees) are +> intentionally excluded — they are not part of this repository. + +## Top level + +| File | Description | +|------|-------------| +| `README.md` | Repository overview and entry point. | +| `LICENSE` | License. | +| `end-to-end-tests.sh` | Driver script that builds and runs all end-to-end tests. | +| `ccpp_constituent_prop_mod.F90.patch` | Patch applied to the runtime constituent-properties module for host integrations. | +| `CODEOWNERS`, `.codecov.yml`, `.codee-format`, `.gitignore` | Repo/CI configuration (code owners, coverage, formatter, ignore rules). | +| `.github/` | GitHub Actions CI workflows (unit tests, end-to-end tests, doxygen). | + +## `capgen-ng/` — command-line entry points + +| File | Description | +|------|-------------| +| `__init__.py` | Package marker (“next-generation CCPP code generator”). | +| `ccpp_capgen_ng.py` | **Main generator CLI.** Parses metadata + the SDF, resolves variables, and writes the caps, `ccpp_kinds.F90`, and `datatable.xml`. Hosts flags like `--kind-type`, `--trace`, `--no-host-introspection`, and the compat shims. | +| `ccpp_datafile.py` | CLI to query the generated `datatable.xml` (generated files, scheme files, dependencies) for build systems / CMake. | +| `ccpp_validator.py` | **Standalone validator** — checks scheme Fortran source against its `.meta` (intent/type/kind/rank/dimensions). Separate tool from the generator; owns the one Fortran parser. | + +## `capgen-ng/generator/` — cap code generation + +| File | Description | +|------|-------------| +| `__init__.py` | Package marker. | +| `datatable.py` | Writes/reads `datatable.xml` mapping suites → generated files, scheme modules, and dependencies (the build-system interface). | +| `suite_xml.py` | Parses the Suite Definition File (SDF) XML into the suite object model (groups, subcycles, subcolumns). | +| `suite_types.py` | Object model for suites/groups/schemes, incl. intrinsic-vs-external scheme classification. | +| `suite_resolver.py` | Resolves a suite end-to-end: matches variables across schemes + host, constituents, index symbols, unit normalization. | +| `suite_cap.py` | Emits the **suite-level cap** (`ccpp_physics_run`/`_init`/… dispatching to groups; register-before-init contract). | +| `group_cap.py` | Emits the **per-group caps** that call the schemes, with argument marshalling and inline transforms. | +| `host_cap.py` | Emits the **host cap** (registration + runtime introspection API; introspection routines stubbed under `--no-host-introspection`). | +| `host_constituents.py` | Host-side constituent handling (`type=host` constituent tables). | +| `suite_data.py` | Emits the generated suite **data module** — pointer-wrapper DDTs plus transform local temporaries. | +| `kinds_writer.py` | Writes `ccpp_kinds.F90` (kind definitions the caps `use`). | +| `trace.py` | Shared helpers emitting the gated `if (trace) write(...) 'CCPP TRACE …'` lines in every cap (toggled by `--trace`). | + +## `capgen-ng/metadata/` — metadata parsing & variable resolution + +| File | Description | +|------|-------------| +| `__init__.py` | Package marker. | +| `metadata_table.py` | Parser for `.meta` metadata-table files (`[ccpp-table-properties]` / `[ccpp-arg-table]`). | +| `variable_resolver.py` | Core variable matching/transform engine — unit + kind conversions, vertical flip, DDT typing. | +| `unit_conversion.py` | Unit-conversion formula table (`{var}` substitution) feeding the auto-inserted unit transforms. | +| `registered_dimensions.py` | Registry of count-dim ↔ index-var pairings (`SCALAR_INDEX_DIMS`) and framework count dimensions. | +| `legacy_compat.py` | **Transient shim** — rewrites legacy CCPP standard names (e.g. `horizontal_loop_extent`) at parse time. | +| `dim_aliases.py` | **Transient shim** — collapses equivalent GFS-physics dimension names. | +| `auto_clone_constituents.py` | **Transient shim** — reinstates original-capgen auto-cloning of static constituents. | + +## `capgen-ng/metadata/parse_tools/` — shared parse utilities + +| File | Description | +|------|-------------| +| `__init__.py` | Package marker. | +| `parse_source.py` | Parsing primitives: parse context + exception types. | +| `parse_checkers.py` | Metadata field validators (`check_units`, `check_dimensions`, `check_cf_standard_name`, …). | +| `parse_log.py` | Shared logging utilities for parse processes. | +| `io_helpers.py` | File-write helpers with write-if-changed (no-op-if-unchanged) semantics. | +| `fortran_conditional.py` | Builds Fortran conditional expressions (in local names) for active/optional-argument handling. | +| `xml_tools.py` | XML helpers — entity expansion and pretty-printed writing (SDF / datatable). | + +## `capgen-ng/schema/` & `capgen-ng/src/` — schema + shipped runtime Fortran + +| File | Description | +|------|-------------| +| `schema/suite_v1_0.xsd` | XML schema for SDF v1.0. | +| `schema/suite_v2_0.xsd` | XML schema for SDF v2.0 (adds suite-level ``/``). | +| `src/ccpp_constituent_prop_mod.F90` (+ `.meta`) | Runtime constituent-properties DDT module shipped with the framework. | +| `src/ccpp_hash_table.F90` | Runtime hash-table support. | +| `src/ccpp_hashable.F90` | Hashable base type used by the hash table. | +| `src/ccpp_scheme_utils.F90` | Runtime scheme utility routines. | + +## `unit-tests/` — pytest suite (one row per driver; fixtures collapsed) + +| File | Description | +|------|-------------| +| `run_tests.py`, `conftest.py`, `__init__.py` | Test runner, pytest fixtures, package marker. | +| `test_metadata_table.py` | Tests for `metadata/metadata_table.py`. | +| `test_variable_resolver.py` | Tests for `metadata/variable_resolver.py`. | +| `test_registered_dimensions.py` | Tests for `metadata/registered_dimensions.py`. | +| `test_dim_aliases.py` | Tests for `metadata/dim_aliases.py`. | +| `test_legacy_compat.py` | Tests for `metadata/legacy_compat.py`. | +| `test_auto_clone_constituents.py` | Tests for `metadata/auto_clone_constituents.py`. | +| `test_io_helpers.py` | Tests for `parse_tools/io_helpers.py`. | +| `test_suite_xml.py` | Tests for `generator/suite_xml.py`. | +| `test_suite_types.py` | Tests for `generator/suite_types.py`. | +| `test_suite_resolver.py` | Tests for `generator/suite_resolver.py`. | +| `test_suite_cap.py` | Tests for `generator/suite_cap.py`. | +| `test_suite_data.py` | Tests for `generator/suite_data.py`. | +| `test_host_cap.py` | Tests for `generator/host_cap.py`. | +| `test_host_constituents.py` | Tests for `generator/host_constituents.py`. | +| `test_kinds_writer.py` | Tests for `generator/kinds_writer.py`. | +| `test_datatable.py` | Tests for `generator/datatable.py`. | +| `test_trace.py` | Tests for `generator/trace.py`. | +| `test_ccpp_datafile.py` | Tests for `ccpp_datafile.py`. | +| `test_validator.py` | Tests for `ccpp_validator.py` (incl. the Fortran parser). | +| `test_control_validation.py` | Tests for control-variable validation rules. | +| `test_integration.py` | End-to-end generator integration tests (full parse → resolve → emit). | +| `sample_files/`, `sample_suite_files/` | **~100 metadata / SDF / Fortran fixtures** consumed by the tests above — not catalogued individually. | + +## `end-to-end-tests/` — full build-and-run cases (one row per case; fixtures collapsed) + +Each case directory bundles host + scheme Fortran, `.meta`, an SDF, a `*_test_reports.py` +comparison driver, and CMake glue. The fixtures are collapsed; the row describes what the case exercises. + +| Case | What it exercises | +|------|-------------------| +| `capgen_ng/` | **Overall generator capabilities** — multiple suites & groups, DDT usage (incl. an undocumented DDT member), `ccpp_constant_one:N` and bare-`N` dimensions, non-standard/integer dimensions, variables promoted to suite level, dimensions set in the register phase and used to allocate module-level interstitials, and threading. | +| `advection/` | Constituent advection — cloud liquid/ice constituents with tendency application (`apply_constituent_tendencies` invoked twice); includes a deliberate error suite (`cld_suite_error.xml`) to exercise diagnostics. | +| `advection_auto_clone/` | Same fixtures as `advection/`, run through the `--legacy-auto-clone-constituents` shim path. | +| `ddthost/` | A host whose CCPP data is carried in a derived type (`host_ccpp_ddt`); runs the temp + DDT suites against it. | +| `var_compat/` | The variable-compatibility object (`VarCompatObj`): unit conversions (forward & reverse), vertical flip (`top_at_one`), kind conversions, and combinations — plus subcycles (nested, dynamic vs fixed iteration length, shared length-defining standard names). | +| `nested_suite/` | Nested suites (a suite that includes a sub-suite), expanded at and inside groups; SDF schema **2.0**; suite-level single ``/`` schemes. Inherited from `var_compat`. | +| `constituents_dim/` | Variables dimensioned by the framework constituent count `number_of_ccpp_constituents` (host never declares it); covers host-owned, framework-allocated, and scheme-allocated count-dim cases, plus consuming constituents without re-flagging (rule b). | +| `suite_allocate/` | A suite-owned, **scheme-allocated** (`allocatable`) variable promoted to `ccpp__data`; its dimension is also suite-owned and set in the `timestep_init` phase (so it can't be allocated at init). | +| `instances/` | **Multiple model instances** — `instance`/`number_of_instances` paired control; the host loops `ccpp_physics_run` over instances with per-instance data (unit-conversion schemes are just the vehicle). | +| `instances_advection/` | Multiple instances combined with constituent advection — scheme-registered constituents with a per-instance buffer (`ninstances` × cloud-liquid + tendency application). | +| `opt_arg/` | Optional-argument handling — present/absent dummy arguments (pointer association vs runtime guard). | +| `chunked_data/` | Chunked/blocked host data — `chunk_begin`/`chunk_end` bounds over `nchunks` chunks. | +| `*_test_reports.py` (where present) | Per-case driver that builds, runs, and diffs expected vs actual output (older cases; newer cases run via `ctest`/CMake). | +| `CMakeLists.txt`, `cmake/`, `utils/` | Shared CMake configuration and helpers for the e2e harness. | + +## `doc/` — documentation + +| File | Description | +|------|-------------| +| `README.md` | Documentation index. | +| `redesign_prompt.md`, `redesign_analysis.md`, `redesign_analysis_original_*.md` | Original redesign brief and analysis that motivated capgen-ng. | +| `briefing.md`, `briefing_pm.md` | Design briefings. | +| `migration.md` | Guide for migrating a host from ccpp-prebuild/original-capgen to capgen-ng. | +| `capgen_compat_layer.md` | Documents the transient compatibility shims (legacy names, dim aliases, auto-clone). | +| `constituents.md` | Constituent-handling design. | +| `constituents_overhaul.md` | Proposed constituent-model overhaul (proposals A/B/C). | +| `auto_clone_constituents.md` | Design notes for the auto-clone-constituents shim. | +| `cam4_fwaut_constituent_order.md` | Case study: CAM4 FWAUT constituent-ordering b4b investigation. | +| `Doxyfile.in` | Doxygen configuration. | +| `CMakeLists.txt` | Build glue for the docs. | +| `DevelopersGuide/` | Developers Guide (`README.md`, generated PDFs, LaTeX style) — bundle, not catalogued per file. | +| `HelloWorld/` | Worked “hello world” host + scheme + suite + build example — bundle, not catalogued per file. | +| `img/` | Documentation images. | From 19b01c383c65daf4c43222b7ae802fc949cceb67 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 25 Jun 2026 14:27:23 -0600 Subject: [PATCH 73/74] Add doc/code_walkthrough_DRAFT.md, update doc/constituents_overhaul.md --- doc/code_walkthrough_DRAFT.md | 532 ++++++++++++++++++++++++++++++++++ doc/constituents_overhaul.md | 50 ++++ 2 files changed, 582 insertions(+) create mode 100644 doc/code_walkthrough_DRAFT.md diff --git a/doc/code_walkthrough_DRAFT.md b/doc/code_walkthrough_DRAFT.md new file mode 100644 index 00000000..c8ee2fb8 --- /dev/null +++ b/doc/code_walkthrough_DRAFT.md @@ -0,0 +1,532 @@ +# capgen-ng — code walkthrough for prebuild/capgen developers (DRAFT) + +> **Status: temporary draft for the developer walkthrough.** All `file → routine → line` +> anchors were verified against the current tree; line numbers drift, so treat them as +> “go here,” not gospel. Three running examples: a **simple** one +> (`end-to-end-tests/instances/`) used to teach the whole pipeline, an **advanced** +> one (`end-to-end-tests/capgen_ng/`) for the resolver’s harder features, and a +> **constituents** one (`end-to-end-tests/advection/`) for the constituent subsystem. +> Once reviewed, this folds into `doc/DevelopersGuide/`. + +--- + +## 0. Orientation for prebuild/capgen developers + +If you come from **ccpp-prebuild**: there is no Python-templated giant cap and no +`ccpp_prebuild_config.py`. capgen-ng parses metadata and the SDF, **resolves every scheme +argument into an explicit Python object** that records *exactly* where the host data lives +and what (if any) unit/kind/flip transform it needs, then emits Fortran from those objects. + +If you come from **original capgen**: the shape is familiar (metadata → host dict → suite +resolution → caps), but the data model is flatter and the resolution result is a plain +dataclass tree (`SuiteResolution → ResolvedGroup → ResolvedCall → ResolvedArg`) you can +print and inspect. + +The single sentence to keep in mind: + +> **A `ResolvedArg` is the unit of truth.** It stores the host-side access expression +> (`call_expr`) *and* the transform plan (`transform_case` + the forward/backward +> expressions). The emitter does almost no thinking — it just renders `ResolvedArg`s. + +--- + +## 1. The pipeline at a glance + +Everything is orchestrated by `capgen()` in **`ccpp_capgen_ng.py:863`**. + +```mermaid +flowchart TD + A["parse .meta files
parse_metadata_file()
metadata_table.py:1166"] --> B["build flat host dict
build_flat_host_dict()
variable_resolver.py:614"] + A --> C["build scheme store
SchemeStore.build_from()
variable_resolver.py:809"] + D["parse SDF XML
parse_suite_xml_files()
suite_xml.py"] --> E + B --> E["resolve_suite()
suite_resolver.py:2313"] + C --> E + E --> F["SuiteResolution
(groups → calls → args)"] + F --> G["write_group_cap()
group_cap.py:1272
(emit the call string)"] + F --> H["write_suite_data / _types / _cap
write_host_cap, write_datatable"] +``` + +| # | Stage | Routine (file:line) | Produces | +|---|-------|---------------------|----------| +| 1 | Parse `.meta` | `parse_metadata_file` (`metadata_table.py:1166`) → `MetadataTable` (`:940`) / `MetaVar` (`:414`) | per-file tables | +| 2 | Flat host dict | `build_flat_host_dict` (`variable_resolver.py:614`) | `{std_name: HostVarEntry}` | +| 3 | Scheme store | `SchemeStore.build_from` (`variable_resolver.py:809`) | per-scheme ordered arg lists | +| 4 | Parse SDF | `parse_suite_xml_files` (`suite_xml.py`) | suite/group/subcycle/scheme objects | +| 5 | **Resolve** | `resolve_suite` (`suite_resolver.py:2313`) | `SuiteResolution` | +| 6 | **Emit calls** | `write_group_cap` (`group_cap.py:1272`) | `ccpp___cap.F90` | +| 7 | Emit rest | `write_suite_data/_types/_cap`, `write_host_cap`, `write_datatable` | suite data module, host cap, datatable | + +--- + +## 2. The dictionaries — what exists *before* matching + +### 2a. Per-file tables (`MetadataTable` / `MetaVar`) + +`parse_metadata_file` returns one `MetadataTable` per `[ccpp-table-properties]` block; each +holds the `[ccpp-arg-table]` variables as `MetaVar`s (standard_name, local_name, type, kind, +units, dimensions, intent, optional, active, …). This is a faithful in-memory copy of the +`.meta` text — no matching yet. + +### 2b. The flat host dictionary — `host_dict` + +Built once by `build_flat_host_dict` (`variable_resolver.py:614`). It is a flat map keyed by +**standard name**; each value is a `HostVarEntry` (`variable_resolver.py:244`): + +``` +host_dict : { standard_name -> HostVarEntry } + +HostVarEntry +├─ standard_name "data_array2" +├─ local_name "data_array2" +├─ access_path "instance_data(instance_number)%data_array2" ← fully-qualified! +├─ module_name "data" (None for control vars) +├─ type / kind "real" / "kind_phys" +├─ units "m2 s-2" +├─ dimensions ["horizontal_dimension"] +├─ protected / optional / allocatable / active +└─ top_at_one (vertical orientation, for flip detection) +``` + +Key point for prebuild devs: **DDT flattening happens here, at dict-build time.** A host +DDT instance (e.g. `instance_data(number_of_instances)` of type `instance_type`) is walked +recursively (`build_ddt`, `variable_resolver.py:~454`) so that each leaf becomes its own +`HostVarEntry` whose `access_path` already contains the component path and the instance +subscript — e.g. `instance_data(instance_number)%data_array2`. By the time resolution runs, +there are no DDTs left to chase; just standard-name → access-path. + +### 2c. The scheme store + +`SchemeStore.build_from` (`variable_resolver.py:809`) holds, per scheme + phase, the ordered +list of dummy arguments (each a `MetaVar`). This is the *demand* side: “scheme X, run phase, +wants these standard names with these intents/units/dims.” + +### 2d. Snapshot — `instances/` before resolution + +Host side (from `instances/data.meta`), after DDT flattening: + +``` +host_dict = { + "horizontal_dimension" -> ncols (control/host scalar, int) + "number_of_species" -> nspecies + "data_array_all_species" -> instance_data(instance_number)%data_array dims (horiz, species) + "data_array" -> instance_data(instance_number)%data_array(:,2) ← scalar-index sub-var + "data_array2" -> instance_data(instance_number)%data_array2 units m2 s-2 + "data_array_opt" -> instance_data(instance_number)%data_array(:,1) optional, active=(flag_for_opt_array) + "flag_for_opt_array" -> instance_data(instance_number)%opt_array_flag + "instance_number" -> (control var) + "number_of_instances" -> (control var, the instance count) +} +``` + +Demand side (`unit_conv_scheme_1.meta`, run phase): `ccpp_error_message`, +`ccpp_error_code`, `instance_number`, `data_array` (inout, **m**), `data_array2` +(inout, **J kg-1**), `data_array_opt` (inout, **m**, optional). + +Notice the two mismatches the resolver must handle: `data_array2` is **`m2 s-2`** on the host +but **`J kg-1`** in the scheme (a unit transform), and `data_array_opt` is **optional**. + +--- + +## 3. Resolution — matching schemes against the dictionaries + +`resolve_suite` (`suite_resolver.py:2313`) walks the suite in execution order +(groups → subcycles → schemes → phases). For each scheme argument it looks the standard +name up and lands in one of three cases: + +```mermaid +flowchart TD + S["scheme arg: standard_name, intent"] --> Q{"in host_dict?"} + Q -- yes --> H["bind to host/control
source='host'/'control'"] + Q -- no --> Q2{"already a suite_var?"} + Q2 -- yes --> SU["bind to suite-owned var
source='suite'"] + Q2 -- no --> Q3{"intent == out?"} + Q3 -- yes --> P["PROMOTE: create SuiteVar,
add to suite_vars dict
(interstitial)"] + Q3 -- no --> ERR["ERROR: in/inout var
nobody produces
(suite_resolver.py:~1493)"] +``` + +- **Found in host** → `host_dict.get(std)` (`_resolve_single_bound`, `suite_resolver.py:429`). +- **Not found, first use is `intent(out)`** → it’s an interstitial; **promote** it to a + suite-owned variable: a `SuiteVar` (`:964`) is created and added to the running `suite_vars` + dict, so later schemes that read it bind via `source='suite'`. This is capgen-ng’s answer + to prebuild’s “where do interstitials live” — they’re emitted into `ccpp__data.F90`. +- **Not found, first use is `in`/`inout`** → hard error (nobody ever writes it). See the + “undefined intent(out)” discipline — capgen-ng refuses to silently read an unproduced var. + +The two dictionaries in play during resolution: + +| Dict | Lifetime | Keyed by | Value | +|------|----------|----------|-------| +| `host_dict` | built once, read-only | standard_name | `HostVarEntry` | +| `suite_vars` | **grows during resolution** | standard_name | `SuiteVar` | + +So “how are variables resolved between schemes?” → scheme A’s `intent(out)` arg that isn’t a +host variable creates a `SuiteVar`; scheme B later in the same suite, reading the same +standard name, matches that `SuiteVar`. The connection is by **standard name**, and the +storage is `ccpp_suite_data%` (`SuiteVar.access_path`). + +--- + +## 4. Where the resolution is stored — `ResolvedArg` + +This is the crux of your question. Each matched argument becomes a **`ResolvedArg`** +(`suite_resolver.py:1010`). It carries both halves: *where the host data is* and *how to +shuttle it into/out of the scheme*. + +``` +ResolvedArg +├─ scheme_local_name "data_array2" ← keyword in the Fortran call +├─ intent / is_optional / active(_local) +├─ source "host" | "control" | "suite" | "constituent" +├─ host_entry -> HostVarEntry (None if suite-owned) +├─ suite_var -> SuiteVar (None if host) +│ +│ ── WHERE THE DATA IS ─────────────────────────────────────── +├─ base_expr "instance_data(instance_number)%data_array2" +├─ subscript "(:)" or "(lb:ub, 1:nlev)" … +├─ call_expr base_expr + subscript ← the access string +├─ used_dim_std_names {standard names used in the subscript} → drives USE + dummy args +│ +│ ── HOW TO SHUTTLE IT ────────────────────────────────────── +├─ transform_case 1=direct · 2=pointer · 3=transform · 4=pointer+transform +├─ needs_unit/kind/vert flags +├─ unit_forward host→scheme expr (pre-call, intent in/inout) +├─ unit_backward scheme→host expr (post-call, intent out/inout) +├─ temp_name "_l" transform temporary +└─ ptr_name "_p" optional pointer wrapper +``` + +The objects nest exactly like the suite: + +```mermaid +flowchart TD + SR["SuiteResolution (:1333)
suite_vars, init/final calls"] --> RG["ResolvedGroup (:1301)
one per group, phases"] + RG --> RC["ResolvedCall (:1153)
one per scheme invocation"] + RC --> RA["ResolvedArg (:1010)
one per argument"] + RG -.-> RSub["ResolvedSubcycle (:1240)
wraps calls in a do-loop"] +``` + +`ResolvedCall.used_modules` (`:1168`) aggregates `{module: {symbols}}` across its args so the +emitter can write the `use … only:` lines. `write_suite_meta` (`suite_data.py:481`) dumps this +whole tree to a `.meta` for inspection — **the fastest way to see a resolution is to read that +file after a run.** + +### `instances/` resolution snapshot + +| scheme arg | std name | source | `call_expr` | transform_case | +|---|---|---|---|---| +| `instance` | `instance_number` | control | `instance_number` (dummy arg) | 1 (direct) | +| `data_array` | `data_array` | host | `instance_data(instance_number)%data_array(:,2)` | 1 (direct; m=m) | +| `data_array2` | `data_array2` | host | `instance_data(instance_number)%data_array2` | **3** (m2 s-2 ↔ J kg-1) | +| `data_array_opt` | `data_array_opt` | host | `instance_data(instance_number)%data_array(:,1)` | **2** (optional) | + +(`instance_number` is a control var → `module_name = None` → passed as a dummy argument +threaded down from `ccpp_physics_run`, not `use`d.) + +--- + +## 5. Emission — `ResolvedArg` → the call string + +`write_group_cap` (`group_cap.py:1272`) renders each `ResolvedCall`: + +1. **USE statements** from `used_modules` (host/suite modules + symbols). +2. **Pre-call lines** — `_pre_call_lines` (`group_cap.py:575`). +3. **The call** — `=` for every arg. +4. **Post-call lines** — `_post_call_lines` (`group_cap.py:613`). + +The `transform_case` decides everything (the “actual arg” passed is in column 3): + +| case | meaning | pre-call | actual arg | post-call | +|------|---------|----------|------------|-----------| +| 1 | direct | — | `call_expr` | — | +| 2 | optional ptr | `ptr%ptr => call_expr` (or `nullify` if inactive) | `ptr%ptr` | `nullify(ptr%ptr)` | +| 3 | transform | `temp = unit_forward` (in/inout) | `temp` | `call_expr = unit_backward` (out/inout) | +| 4 | ptr + transform | `temp = unit_forward; ptr%ptr => temp` | `ptr%ptr` | `nullify; call_expr = unit_backward` | + +For `instances/`, the emitted group cap (schematically) is: + +```fortran +use data, only: instance_data +... +! data_array2 (case 3): host m2 s-2 -> scheme J kg-1 +data_array2_l = +! data_array_opt (case 2): optional pointer +if (flag_for_opt_array) then + data_array_opt_p%ptr => instance_data(instance_number)%data_array(:,1) +else + nullify(data_array_opt_p%ptr) +end if + +call unit_conv_scheme_1_run( & + instance = instance_number, & ! case 1 + data_array = instance_data(instance_number)%data_array(:,2),& ! case 1 + data_array2 = data_array2_l, & ! case 3 (temp) + data_array_opt = data_array_opt_p%ptr, & ! case 2 (ptr) + errmsg=errmsg, errflg=errflg) + +! data_array2 post: copy back scheme -> host +instance_data(instance_number)%data_array2 = +nullify(data_array_opt_p%ptr) +``` + +That is the whole chain: **`.meta` → `host_dict`/scheme args → `ResolvedArg.call_expr` + +`transform_case` → these emitted lines.** + +> Note the **scheme appears twice** in `instances/` (`unit_conv_scheme_1`, `_2`, `_1`). +> Each appearance is its own `ResolvedCall`; capgen-ng dedups *init/finalize* phases by +> scheme name within a group, but **run** phases emit every appearance. + +--- + +## 6. Running example 1 (simple) — `instances/` end-to-end + +Walk it through the 7 stages: + +1. **Parse** `data.meta` (host + `instance_type` DDT) and the two scheme `.meta`s. +2. **Host dict** — DDT flattened; note `data_array`/`data_array_opt` are scalar-index + sub-views (`data_array(:,2)`, `data_array(:,1)`) of one stored array, and the instance + subscript `(instance_number)` is baked into every `access_path` (§2d). +3. **Scheme store** — `unit_conv_scheme_1/2` run-phase arg lists. +4. **SDF** — one group, `scheme_1`, `scheme_2`, `scheme_1`. +5. **Resolve** — all args hit the host (no promotion here); one unit transform, one optional + (§4 table). +6. **Emit** — §5 listing; plus the host driver loops `do ins = 1, ninstances` passing + `instance = ins`, so the same cap re-runs per instance with the `(instance_number)` + subscript selecting that instance’s slice. +7. **Rest** — `ccpp_instances_data.F90` is essentially empty (nothing promoted); the suite/ + host caps wire `instance_number`/`number_of_instances` as the paired instance control. + +**What this example teaches:** the full parse→dict→resolve→emit spine, DDT flattening, +scalar-index dimensions, control vars, a unit transform (case 3), and an optional arg +(case 2) — with *zero* promotion, so the host↔scheme matching is unobscured. + +--- + +## 7. Running example 2 (advanced) — `capgen_ng/` : what the resolver adds + +Same pipeline; this case exercises the features `instances/` doesn’t. Read it for: + +- **Suite-level promotion (interstitials).** A scheme’s `intent(out)` var that no host table + declares becomes a `SuiteVar` (§3) and is emitted into `ccpp_temp_suite_data.F90`. Trace a + variable that is *not* in `test_host.meta` but is produced by one scheme and consumed by a + later one — that is the `suite_vars` path, and the “variables that should be promoted to + suite level” bullet in the case’s `README.md`. +- **Deeper DDT usage**, including an *undocumented* DDT member — exercises `build_ddt` + recursion and the “don’t require every component to be documented” rule. +- **Non-standard / integer dimensions** and `ccpp_constant_one:N` vs bare `N` — exercises the + subscript builder (`_build_call_subscript`, `suite_resolver.py:479`) and dimension + canonicalisation (`_canonical_dim`, `:761`). +- **Register-phase dimensions** set by a scheme and then used to size module-level + interstitials — the resolver must order phases so the dimension is known before allocation + (`validate_init_dimensions`, called from `capgen()`). +- **Multiple suites & groups + threading** — one `ResolvedGroup` per group, dispatched by the + suite cap’s `select case` on `group_name`. + +Suggested walkthrough move: open `write_suite_meta`’s output for `temp_suite`, find a +promoted variable, and show its `SuiteVar` (no `host_entry`, `access_path = +ccpp_suite_data(1)%…`) next to the two `ResolvedArg`s that produce and consume it. + +--- + +## 8. Running example 3 (constituents) — `advection/` + +Constituents are the one major subsystem the first two examples don’t touch — and the entire +constituent half of `ResolvedArg` (`source='constituent'`, `is_constituent_arg`, +`constituent_module_name`, `constituent_extra_symbols`, `constituent_index_std_names`, +`used_const_dim_std_names`) only comes alive here. `advection/` (cloud liquid/ice tracers) +exercises **three distinct constituent paths**. Suite: `const_indices`, `cld_liq`, +`apply_constituent_tendencies`, `cld_ice`, `apply_constituent_tendencies`. + +### The model (for prebuild developers) + +A **constituent** is a tracer the *host’s dynamical core* owns — water vapor, cloud liquid, +ozone, a chemistry species — that physics reads and updates, together with an optional +**tendency** (the rate of change physics writes back for the dycore to advect it forward). +The crucial difference from an ordinary host variable: the host does **not** hand you a Fortran +array per constituent. The framework owns **one `ccpp_model_constituents_t` object per model +instance**, holding all constituent values (`%vars_layer`), tendencies (`%vars_layer_tend`), +metadata (`%const_metadata`), and the count (`%num_layer_vars`). The resolver translates +standard-name references into subscripts into that object. *(Authoritative deep-dive: +`doc/constituents.md` — “the four rules.”)* + +Three questions prebuild developers always ask: + +**1. What makes a variable a constituent?** Two independent triggers, plus host override: +- a scheme arg carries a **hint attribute** — any of `advected = true`, `constituent = true`, + or a non-default `molar_mass` (`MetaVar.is_constituent`, `metadata_table.py:724`); **or** +- the arg’s **type** is `ccpp_constituent_properties_t` — the register-phase descriptor array + (a separate flag, `is_constituent_arg`); **and** +- constituent-ness is ultimately the **host’s** decision. A scheme that only *reads* a name + need not re-flag it — capgen-ng infers it from the set of names *some* scheme flags (“rule + b”). If the host declares the name as an ordinary variable, that wins + (`design_constituent_host_wins`). + +**2. Where/how are constituents registered?** Exactly one way to declare a *new* one (Rule 1): +a **register-phase** scheme returns an `intent=out, allocatable` array of +`ccpp_constituent_properties_t`, populating each entry via +`%instantiate(std_name=…, units=…, vertical_dim=…, advected=…, …)`. The framework collects +every register scheme’s array and merges them into each instance’s constituent object at +`ccpp_register_constituents`. You **cannot** create a base constituent in a physics phase — +that’s a hard error (Rule 4). + +**3. Are they always tracer + tendency + index triples?** **No** — the pieces are independent: +- the **base constituent** (the tracer): registered once, stored in `%vars_layer`, read via + `index_of_`; +- an **optional tendency**: a *separate* arg whose standard name starts with `tendency_of_`, + stored in `%vars_layer_tend`, implicitly tied to the base of the same name (Rule 3) — a + scheme emits one only if it has a tendency to give; +- the **index** `index_of_`: not something you declare — the framework derives it and fills + it at init via `%const_index()`; the value is identical for every instance. + +So a constituent is *“one registered base + zero-or-more tendency references + a framework +index,”* not a fixed triple. + +**How standard names map to storage** (this is what the resolver emits): + +| Scheme arg references… | Resolves to | +|---|---| +| a base constituent (by name, via its index) | `ccpp_model_constituents_obj(inst)%vars_layer(, index_of_)` | +| `tendency_of_` | `…%vars_layer_tend(, index_of_)` | +| `ccpp_constituents` | `…%vars_layer(:,:,:)` (whole array) | +| `ccpp_constituent_tendencies` | `…%vars_layer_tend(:,:,:)` | +| `number_of_ccpp_constituents` | `…%num_layer_vars` (scalar count) | +| `index_of_` | module-level `integer :: index_of_` | + +Now, how each of these shows up in `advection/`: + +```mermaid +flowchart LR + R["register phase
cld_liq_register"] -->|"dyn_const :
ccpp_constituent_properties_t
(intent out, allocatable)"| FW["framework builds the
per-instance constituent object"] + FW --> IDX["index_of_<X> symbols
+ number_of_ccpp_constituents"] + IDX --> RUN["run phase
base(:,:,index_of_…) · ccpp_constituents(:,:,:)"] +``` + +### 8.1 Registration — the `ccpp_constituent_properties_t` argument + +`cld_liq_register` (a **register**-phase entry) declares: + +``` +[ dyn_const ] + standard_name = dynamic_constituents_for_cld_liq + type = ccpp_constituent_properties_t + dimensions = (:) + intent = out + allocatable = true +``` + +The resolver flags this `is_constituent_arg = True` and — unlike every other not-in-host +`intent(out)` variable — **does not promote it to a `SuiteVar`**. It is a local temporary the +scheme allocates and fills with constituent descriptors (`suite_resolver.py:~1570`). The +framework gathers each register scheme’s `dyn_const` array to build the model’s constituent +set and the per-instance constituent object. + +> Contrast with §3: a normal not-in-host `intent(out)` → `SuiteVar` (suite-owned data). A +> `ccpp_constituent_properties_t` `intent(out)` → *local temp, collected by the framework*. +> This special-case is the one exception to the promotion rule. + +### 8.2 Flagging a produced constituent + +In `cld_liq_run`: + +``` +[ cld_liq_array ] standard_name = cloud_liquid_dry_mixing_ratio advected = .true. intent=inout +[ cld_liq_tend ] standard_name = tendency_of_cloud_liquid_dry_mixing_ratio constituent = True intent=out +``` + +`advected` / `constituent` (non-default) set `ResolvedArg.is_constituent = True`. The backing +store is the framework constituent array, reached by index (next). + +### 8.3 Indexing one constituent — `index_of_` (`source='constituent'`) + +Schemes that touch a single constituent slice declare an access like: + +``` +[ q(:,:,index_of_water_vapor_specific_humidity) ] + standard_name = water_vapor_specific_humidity +``` + +`q` is the constituent backing array; `index_of_water_vapor_specific_humidity` is a +per-constituent index symbol. This is the `source='constituent'` path, and the `ResolvedArg` +records: + +| field | value | purpose | +|---|---|---| +| `constituent_module_name` | suite cap module | module to `use` | +| `constituent_extra_symbols` | `{index_of_water_vapor_specific_humidity}` | symbols to `use` (the index integers) | +| `constituent_index_std_names` | `{water_vapor_specific_humidity}` | the **real** standard name, kept verbatim | + +Why keep the real name separately? The Fortran `index_of_*` symbol is **mangled** to fit the +63-char identifier limit (`_index_symbol_name`, `suite_resolver.py:148`), so its suffix is not +a reliable source of the standard name. At init, the framework fills each index via +`%const_index()`. + +> **Host-wins:** if the host itself declares the `index_of_*` / framework names, the resolver +> short-circuits to ordinary host-arg resolution (the constituent path is skipped). That’s the +> `design_constituent_host_wins` rule. + +### 8.4 The whole constituent axis — `number_of_ccpp_constituents` + +`apply_constituent_tendencies_run` takes the full arrays: + +``` +[ const ] standard_name = ccpp_constituents dims (horizontal, vertical, number_of_ccpp_constituents) +[ const_tend ] standard_name = ccpp_constituent_tendencies dims (horizontal, vertical, number_of_ccpp_constituents) +``` + +`number_of_ccpp_constituents` is a **framework count dimension** — the host never declares it +as a scalar; its value comes from the per-instance constituent object at runtime. The resolver +files it under `ResolvedArg.used_const_dim_std_names` (note: *not* `used_dim_std_names`), so it +produces **no** `use`/dummy-arg, but it *is* surfaced by the host introspection routines. The +emitted call passes the whole axis (`:`) on that dimension. + +### 8.5 The error suite + +`cld_suite_error.xml` swaps in `dlc_liq` — a scheme whose `ccpp_constituent_properties_t` +setup is wrong — so the resolver’s constituent error reporting can be demonstrated live (a bad +constituent declaration is rejected, not silently mis-wired). + +### Callout — combining with instances (`instances_advection/`) + +`instances_advection/` is `advection/` **plus** multiple instances. The *only* delta: the +constituent buffer is dimensioned `(number_of_instances)` — a **wrapper-DDT array, one +constituent object per instance** — so per-instance `set_const_index` calls don’t collide +(the per-instance dynamic-constituent buffer). The constituent resolution itself is identical +to §8.1–8.4. Use it only when the audience needs the multi-instance constituent story. + +--- + +## 9. How to follow along live + +- **Run it:** point `capgen()` / `ccpp_capgen_ng.py` at the example’s `.meta` + SDF and inspect + the generated `ccpp_*_cap.F90`, `ccpp_*_data.F90`, and `datatable.xml`. +- **Read the resolution:** `write_suite_meta` (`suite_data.py:481`) emits the resolved suite as + a `.meta` — the cleanest dump of `SuiteResolution`. +- **`--trace`:** regenerate with `--trace` (or flip `logical, parameter :: trace = .true.` in + one cap) to get `CCPP TRACE ` lines at runtime — useful for seeing the *call order* the + resolution produced. +- **Unit tests as specs:** `unit-tests/test_suite_resolver.py`, `test_variable_resolver.py`, + `test_group_cap.py` are small, readable assertions about exactly these structures. + +--- + +## Appendix — `file → routine → line` quick reference + +| Concept | Routine | File:line | +|---|---|---| +| Orchestrator | `capgen` | `ccpp_capgen_ng.py:863` | +| Load metadata | `_load_metadata_files` | `ccpp_capgen_ng.py:637` | +| Parse `.meta` | `parse_metadata_file` | `metadata/metadata_table.py:1166` | +| Parsed table / var | `MetadataTable` / `MetaVar` | `metadata/metadata_table.py:940` / `:414` | +| Host dict entry | `HostVarEntry` | `metadata/variable_resolver.py:244` | +| Build host dict | `build_flat_host_dict` | `metadata/variable_resolver.py:614` | +| DDT flatten | `build_ddt` | `metadata/variable_resolver.py:~454` | +| Scheme store | `SchemeStore.build_from` | `metadata/variable_resolver.py:809` | +| Parse SDF | `parse_suite_xml_files` | `generator/suite_xml.py` | +| **Resolve suite** | `resolve_suite` | `generator/suite_resolver.py:2313` | +| Resolve one bound | `_resolve_single_bound` | `generator/suite_resolver.py:429` | +| Build subscript | `_build_call_subscript` | `generator/suite_resolver.py:479` | +| Suite-owned var | `SuiteVar` | `generator/suite_resolver.py:964` | +| **Resolved arg** | `ResolvedArg` | `generator/suite_resolver.py:1010` | +| Resolved call/group/suite | `ResolvedCall` / `ResolvedGroup` / `SuiteResolution` | `:1153` / `:1301` / `:1333` | +| **Emit group cap** | `write_group_cap` | `generator/group_cap.py:1272` | +| Pre/post transform | `_pre_call_lines` / `_post_call_lines` | `generator/group_cap.py:575` / `:613` | +| Dump resolution | `write_suite_meta` | `generator/suite_data.py:481` | diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index 90731663..a599b89e 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -930,6 +930,56 @@ These are the calls we need to make in the meeting. - **Recommendation**: (b). It's a one-line doc note and zero code change. +### Q7. The `_layer` suffix — was a parallel `_interfaces` storage ever intended? (raised 2026-06-25, walkthrough prep) + +- **Observation.** The per-instance constituent object stores values and + tendencies as `%vars_layer(:,:,:)` and `%vars_layer_tend(:,:,:)` + (`src/ccpp_constituent_prop_mod.F90:167-168`), both allocated over the full + constituent axis (`:1558`, `num_values()` = every registered constituent). + The `_layer` qualifier in the names implies an anticipated **parallel + interface storage** (`vars_interface` / `vars_interface_tend`) that was + never added. +- **Evidence it was anticipated, not accidental.** The type carries + `is_layer_var` (`:637`, tests `vertical_layer_dimension`) **and** + `is_interface_var` (`:652`, tests `vertical_interface_dimension`) + predicates — so the design already distinguishes layer- vs + interface-located constituents, but only layer storage exists. +- **Latent gap.** A constituent declared on `vertical_interface_dimension` + has no storage slot today; `is_interface_var` would return true but there + is nowhere to put it. Whether any host/scheme actually needs interface + constituents is unknown (CAM-SIMA audit in §3 did not surface one). +- **Questions for discussion.** (a) Was `_interfaces` intended and dropped, or + is `_layer` just a (now-misleading) name? (b) Does any consumer need + interface-level constituents? (c) If **no** → drop the `_layer` suffix to + simplify; if **yes** → add the parallel `vars_interface` / `_tend` arrays and + route `is_interface_var` constituents to them. + +### Q8. Should a constituent always be a triplet — base + tendency + index? (raised 2026-06-25, walkthrough prep) + +- **Today (per `constituents.md` four rules):** a constituent is **not** a + forced triplet. It is *one registered base* (Rule 1, the only declaration + path), *zero-or-more optional* `tendency_of_` references (Rule 3 — a + scheme emits one only if it has a tendency), and a *framework-derived* + `index_of_` (never user-declared; filled at init via `%const_index`). +- **But storage already half-implies the triplet.** `%vars_layer_tend` is + allocated over the **whole** constituent axis (`:1558`), so every + constituent has a tendency *column* whether or not any scheme writes it. + So the "triplet" is already true at the **storage** level, but optional at + the **metadata/registration** level. +- **Question for discussion.** Should registration *force* the triplet + (declare base + tendency + index together, uniformly)? + - *For:* uniform mental model; matches the storage; removes the "did anyone + register a tendency?" ambiguity; could let the resolver validate + tendency producers against registered bases. + - *Against:* many constituents have no physics tendency (the column is + already there regardless, so forcing a declaration buys little); the + index is implicit by design and exposing it as a required member + re-introduces the index bookkeeping capgen-ng deliberately hid; the + base is the only thing that *must* be registered. + - *Open sub-question:* if not forced, should the resolver at least **warn** + when a `tendency_of_` is produced for an `` that no register scheme + declared? (relates to §4.9 — no codegen-time cross-check of registration.) + --- ## 8. Three proposals — minimal / clean / deep From c4b7d9aed935e8985309e2d974ed43cb46324e50 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 29 Jun 2026 15:37:41 -0600 Subject: [PATCH 74/74] Rename capgen-ng to capgen --- .github/workflows/end-to-end-tests.yaml | 2 +- .github/workflows/unit-tests.yaml | 10 +- capgen-ng/__init__.py | 1 - capgen-ng/generator/__init__.py | 1 - capgen-ng/metadata/__init__.py | 1 - .../ccpp_capgen.py | 28 ++--- {capgen-ng => capgen}/ccpp_datafile.py | 14 +-- {capgen-ng => capgen}/ccpp_validator.py | 8 +- capgen/generator/__init__.py | 1 + {capgen-ng => capgen}/generator/datatable.py | 12 +-- {capgen-ng => capgen}/generator/group_cap.py | 2 +- {capgen-ng => capgen}/generator/host_cap.py | 2 +- .../generator/host_constituents.py | 4 +- .../generator/kinds_writer.py | 8 +- {capgen-ng => capgen}/generator/suite_cap.py | 6 +- {capgen-ng => capgen}/generator/suite_data.py | 4 +- .../generator/suite_resolver.py | 12 +-- .../generator/suite_types.py | 4 +- {capgen-ng => capgen}/generator/suite_xml.py | 12 +-- {capgen-ng => capgen}/generator/trace.py | 0 capgen/metadata/__init__.py | 1 + .../metadata/auto_clone_constituents.py | 16 +-- {capgen-ng => capgen}/metadata/dim_aliases.py | 6 +- .../metadata/legacy_compat.py | 12 +-- .../metadata/metadata_table.py | 4 +- .../metadata/parse_tools/__init__.py | 0 .../parse_tools/fortran_conditional.py | 0 .../metadata/parse_tools/io_helpers.py | 4 +- .../metadata/parse_tools/parse_checkers.py | 0 .../metadata/parse_tools/parse_log.py | 0 .../metadata/parse_tools/parse_source.py | 0 .../metadata/parse_tools/xml_tools.py | 0 .../metadata/registered_dimensions.py | 20 ++-- .../metadata/unit_conversion.py | 0 .../metadata/variable_resolver.py | 14 +-- {capgen-ng => capgen}/schema/suite_v1_0.xsd | 0 {capgen-ng => capgen}/schema/suite_v2_0.xsd | 0 .../src/ccpp_constituent_prop_mod.F90 | 0 .../src/ccpp_constituent_prop_mod.meta | 0 {capgen-ng => capgen}/src/ccpp_hash_table.F90 | 0 {capgen-ng => capgen}/src/ccpp_hashable.F90 | 0 .../src/ccpp_scheme_utils.F90 | 0 ccpp_constituent_prop_mod.F90.patch | 4 +- doc/auto_clone_constituents.md | 26 ++--- doc/briefing.md | 50 ++++----- doc/briefing_pm.md | 78 +++++++------- doc/cam4_fwaut_constituent_order.md | 40 +++---- doc/capgen-ng_review_plan.xlsx | Bin 0 -> 11801 bytes doc/capgen_compat_layer.md | 26 ++--- doc/code_walkthrough_DRAFT.md | 24 ++--- doc/constituents.md | 24 ++--- doc/constituents_overhaul.md | 102 +++++++++--------- doc/file_catalogue_DRAFT.md | 22 ++-- doc/migration.md | 88 +++++++-------- doc/redesign_analysis.md | 2 +- doc/redesign_prompt.md | 30 +++--- end-to-end-tests/CMakeLists.txt | 2 +- .../advection_auto_clone/cld_liq.meta | 2 +- .../{capgen_ng => capgen}/CMakeLists.txt | 22 ++-- .../{capgen_ng => capgen}/README.md | 0 .../adjust/temp_kinds.F90 | 0 .../capgen_test_reports.py | 0 .../{capgen_ng => capgen}/ddt2.F90 | 0 .../{capgen_ng => capgen}/ddt_suite.xml | 0 .../environ_conditions.meta | 0 .../{capgen_ng => capgen}/make_ddt.F90 | 0 .../{capgen_ng => capgen}/make_ddt.meta | 0 .../{capgen_ng => capgen}/setup_coeffs.F90 | 0 .../{capgen_ng => capgen}/setup_coeffs.meta | 0 .../source_dir1/environ_conditions.F90 | 0 .../source_dir2/temp_set.F90 | 0 .../{capgen_ng => capgen}/temp_adjust.F90 | 0 .../{capgen_ng => capgen}/temp_adjust.meta | 0 .../temp_calc_adjust.F90 | 0 .../temp_calc_adjust.meta | 0 .../{capgen_ng => capgen}/temp_set.meta | 0 .../{capgen_ng => capgen}/temp_suite.xml | 0 .../test_capgen_host_integration.F90 | 0 .../{capgen_ng => capgen}/test_host.F90 | 0 .../{capgen_ng => capgen}/test_host.meta | 0 .../{capgen_ng => capgen}/test_host_data.F90 | 0 .../{capgen_ng => capgen}/test_host_data.meta | 0 .../{capgen_ng => capgen}/test_host_mod.F90 | 0 .../{capgen_ng => capgen}/test_host_mod.meta | 0 end-to-end-tests/chunked_data/CMakeLists.txt | 2 +- end-to-end-tests/cmake/ccpp_capgen.cmake | 16 +-- .../constituents_dim/CMakeLists.txt | 2 +- .../constituents_dim/register_consts.F90 | 2 +- end-to-end-tests/ddthost/CMakeLists.txt | 2 +- end-to-end-tests/instances/CMakeLists.txt | 2 +- end-to-end-tests/nested_suite/CMakeLists.txt | 2 +- end-to-end-tests/opt_arg/CMakeLists.txt | 2 +- end-to-end-tests/opt_arg/main.F90 | 6 +- .../suite_allocate/CMakeLists.txt | 2 +- end-to-end-tests/suite_allocate/README.md | 4 +- .../suite_allocate/make_workspace.F90 | 2 +- .../suite_allocate/use_workspace.F90 | 2 +- end-to-end-tests/var_compat/CMakeLists.txt | 2 +- unit-tests/__init__.py | 2 +- unit-tests/conftest.py | 10 +- unit-tests/run_tests.py | 4 +- .../scheme_consume_constituent.meta | 2 +- .../scheme_module_name_override.meta | 2 +- unit-tests/test_auto_clone_constituents.py | 4 +- unit-tests/test_ccpp_datafile.py | 4 +- unit-tests/test_control_validation.py | 6 +- unit-tests/test_dim_aliases.py | 2 +- unit-tests/test_host_cap.py | 4 +- unit-tests/test_host_constituents.py | 2 +- unit-tests/test_integration.py | 38 +++---- unit-tests/test_legacy_compat.py | 4 +- unit-tests/test_metadata_table.py | 16 +-- unit-tests/test_registered_dimensions.py | 4 +- unit-tests/test_suite_resolver.py | 12 +-- unit-tests/test_suite_types.py | 2 +- unit-tests/test_suite_xml.py | 6 +- unit-tests/test_variable_resolver.py | 6 +- 117 files changed, 466 insertions(+), 463 deletions(-) delete mode 100644 capgen-ng/__init__.py delete mode 100644 capgen-ng/generator/__init__.py delete mode 100644 capgen-ng/metadata/__init__.py rename capgen-ng/ccpp_capgen_ng.py => capgen/ccpp_capgen.py (98%) rename {capgen-ng => capgen}/ccpp_datafile.py (99%) rename {capgen-ng => capgen}/ccpp_validator.py (99%) create mode 100644 capgen/generator/__init__.py rename {capgen-ng => capgen}/generator/datatable.py (98%) rename {capgen-ng => capgen}/generator/group_cap.py (99%) rename {capgen-ng => capgen}/generator/host_cap.py (99%) rename {capgen-ng => capgen}/generator/host_constituents.py (99%) rename {capgen-ng => capgen}/generator/kinds_writer.py (96%) rename {capgen-ng => capgen}/generator/suite_cap.py (99%) rename {capgen-ng => capgen}/generator/suite_data.py (99%) rename {capgen-ng => capgen}/generator/suite_resolver.py (99%) rename {capgen-ng => capgen}/generator/suite_types.py (99%) rename {capgen-ng => capgen}/generator/suite_xml.py (98%) rename {capgen-ng => capgen}/generator/trace.py (100%) create mode 100644 capgen/metadata/__init__.py rename {capgen-ng => capgen}/metadata/auto_clone_constituents.py (94%) rename {capgen-ng => capgen}/metadata/dim_aliases.py (97%) rename {capgen-ng => capgen}/metadata/legacy_compat.py (94%) rename {capgen-ng => capgen}/metadata/metadata_table.py (99%) rename {capgen-ng => capgen}/metadata/parse_tools/__init__.py (100%) rename {capgen-ng => capgen}/metadata/parse_tools/fortran_conditional.py (100%) rename {capgen-ng => capgen}/metadata/parse_tools/io_helpers.py (98%) rename {capgen-ng => capgen}/metadata/parse_tools/parse_checkers.py (100%) rename {capgen-ng => capgen}/metadata/parse_tools/parse_log.py (100%) rename {capgen-ng => capgen}/metadata/parse_tools/parse_source.py (100%) rename {capgen-ng => capgen}/metadata/parse_tools/xml_tools.py (100%) rename {capgen-ng => capgen}/metadata/registered_dimensions.py (91%) rename {capgen-ng => capgen}/metadata/unit_conversion.py (100%) rename {capgen-ng => capgen}/metadata/variable_resolver.py (98%) rename {capgen-ng => capgen}/schema/suite_v1_0.xsd (100%) rename {capgen-ng => capgen}/schema/suite_v2_0.xsd (100%) rename {capgen-ng => capgen}/src/ccpp_constituent_prop_mod.F90 (100%) rename {capgen-ng => capgen}/src/ccpp_constituent_prop_mod.meta (100%) rename {capgen-ng => capgen}/src/ccpp_hash_table.F90 (100%) rename {capgen-ng => capgen}/src/ccpp_hashable.F90 (100%) rename {capgen-ng => capgen}/src/ccpp_scheme_utils.F90 (100%) create mode 100644 doc/capgen-ng_review_plan.xlsx rename end-to-end-tests/{capgen_ng => capgen}/CMakeLists.txt (86%) rename end-to-end-tests/{capgen_ng => capgen}/README.md (100%) rename end-to-end-tests/{capgen_ng => capgen}/adjust/temp_kinds.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/capgen_test_reports.py (100%) rename end-to-end-tests/{capgen_ng => capgen}/ddt2.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/ddt_suite.xml (100%) rename end-to-end-tests/{capgen_ng => capgen}/environ_conditions.meta (100%) rename end-to-end-tests/{capgen_ng => capgen}/make_ddt.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/make_ddt.meta (100%) rename end-to-end-tests/{capgen_ng => capgen}/setup_coeffs.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/setup_coeffs.meta (100%) rename end-to-end-tests/{capgen_ng => capgen}/source_dir1/environ_conditions.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/source_dir2/temp_set.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/temp_adjust.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/temp_adjust.meta (100%) rename end-to-end-tests/{capgen_ng => capgen}/temp_calc_adjust.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/temp_calc_adjust.meta (100%) rename end-to-end-tests/{capgen_ng => capgen}/temp_set.meta (100%) rename end-to-end-tests/{capgen_ng => capgen}/temp_suite.xml (100%) rename end-to-end-tests/{capgen_ng => capgen}/test_capgen_host_integration.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/test_host.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/test_host.meta (100%) rename end-to-end-tests/{capgen_ng => capgen}/test_host_data.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/test_host_data.meta (100%) rename end-to-end-tests/{capgen_ng => capgen}/test_host_mod.F90 (100%) rename end-to-end-tests/{capgen_ng => capgen}/test_host_mod.meta (100%) diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 0bdf2924..3d64ef76 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -1,4 +1,4 @@ -name: capgen-ng end-to-end tests +name: capgen end-to-end tests on: workflow_dispatch: diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 2b99bc13..7a87f6d7 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -1,4 +1,4 @@ -name: capgen-ng unit tests +name: capgen unit tests on: workflow_dispatch: @@ -12,7 +12,7 @@ concurrency: jobs: unit-tests: - name: capgen-ng unit tests + name: capgen unit tests runs-on: ubuntu-latest strategy: matrix: @@ -40,9 +40,9 @@ jobs: which xmllint xmllint --version which pytest - - name: Run capgen-ng unit tests + - name: Run capgen unit tests run: | pytest -v unit-tests/ - - name: Run capgen-ng module doctests + - name: Run capgen module doctests run: | - PYTHONPATH=capgen-ng pytest --doctest-modules capgen-ng + PYTHONPATH=capgen pytest --doctest-modules capgen diff --git a/capgen-ng/__init__.py b/capgen-ng/__init__.py deleted file mode 100644 index 59cfaf70..00000000 --- a/capgen-ng/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CCPP capgen-ng: next-generation CCPP code generator.""" diff --git a/capgen-ng/generator/__init__.py b/capgen-ng/generator/__init__.py deleted file mode 100644 index bb54d8e4..00000000 --- a/capgen-ng/generator/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Cap code generation for ccpp-capgen-ng.""" diff --git a/capgen-ng/metadata/__init__.py b/capgen-ng/metadata/__init__.py deleted file mode 100644 index 5daa68f9..00000000 --- a/capgen-ng/metadata/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Metadata parsing and variable resolution for ccpp-capgen-ng.""" diff --git a/capgen-ng/ccpp_capgen_ng.py b/capgen/ccpp_capgen.py similarity index 98% rename from capgen-ng/ccpp_capgen_ng.py rename to capgen/ccpp_capgen.py index 178041e4..3f01918b 100755 --- a/capgen-ng/ccpp_capgen_ng.py +++ b/capgen/ccpp_capgen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""ccpp_capgen_ng — next-generation CCPP cap code generator. +"""ccpp_capgen — next-generation CCPP cap code generator. This script replaces both ``ccpp_prebuild.py`` and ``ccpp_capgen.py`` from the legacy toolchain. It reads host-model metadata files, scheme metadata files, @@ -20,7 +20,7 @@ ----- :: - ccpp_capgen_ng.py \\ + ccpp_capgen.py \\ --host-name \\ --host-files \\ --scheme-files \\ @@ -75,7 +75,7 @@ import sys from typing import Dict, List, Optional, Tuple -# Ensure the capgen-ng package is importable when invoked directly. +# Ensure the capgen package is importable when invoked directly. _SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) _PACKAGE_DIR = os.path.dirname(_SCRIPT_DIR) if _PACKAGE_DIR not in sys.path: @@ -106,7 +106,7 @@ # Logging ######################################################################## -_LOGGER = init_log('ccpp_capgen_ng') +_LOGGER = init_log('ccpp_capgen') ######################################################################## @@ -125,8 +125,8 @@ # generated cap modules whenever any suite touches constituent state. # Listed in datatable.xml's so host CMake projects pick them # up via ccpp_datafile.py --utility-files / --ccpp-files queries. All -# of these live in :data:`_FRAMEWORK_SRC_DIR` (capgen-ng's own ``src/``); -# capgen-ng ships self-contained — no external src/ companion needed. +# of these live in :data:`_FRAMEWORK_SRC_DIR` (capgen's own ``src/``); +# capgen ships self-contained — no external src/ companion needed. _FRAMEWORK_F90_FILES = [ 'ccpp_constituent_prop_mod.F90', 'ccpp_hashable.F90', @@ -139,8 +139,8 @@ def _resolve_framework_f90_files() -> List[str]: """Return absolute paths for the framework F90 files. Each name in :data:`_FRAMEWORK_F90_FILES` is looked up under - :data:`_FRAMEWORK_SRC_DIR` (``capgen-ng/src/``). A missing file is - a hard error: capgen-ng/src/ is the canonical (and only) location; + :data:`_FRAMEWORK_SRC_DIR` (``capgen/src/``). A missing file is + a hard error: capgen/src/ is the canonical (and only) location; a missing file means the deployment is incomplete and the host build would fail later with an opaque "Cannot open module file" error. Surface it now with a precise message instead. @@ -155,10 +155,10 @@ def _resolve_framework_f90_files() -> List[str]: missing.append(p) if missing: raise CCPPError( - "capgen-ng deployment is incomplete: required framework " + "capgen deployment is incomplete: required framework " "Fortran source file(s) not found under {!r}:\n {}\n" - "Vendor the missing file(s) into capgen-ng/src/ (the " - "canonical location for files capgen-ng emits a USE for).".format( + "Vendor the missing file(s) into capgen/src/ (the " + "canonical location for files capgen emits a USE for).".format( _FRAMEWORK_SRC_DIR, '\n '.join(missing), ) ) @@ -177,7 +177,7 @@ def _build_arg_parser() -> argparse.ArgumentParser: argparse.ArgumentParser """ parser = argparse.ArgumentParser( - prog='ccpp_capgen_ng.py', + prog='ccpp_capgen.py', description='CCPP next-generation cap code generator', formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__, @@ -253,7 +253,7 @@ def _build_arg_parser() -> argparse.ArgumentParser: "TRANSIENT MIGRATION SHIM. Accept legacy CCPP standard " "names (currently 'horizontal_loop_extent') in scheme " "metadata and silently rewrite them to their canonical " - "capgen-ng equivalents ('horizontal_dimension'). Emits a " + "capgen equivalents ('horizontal_dimension'). Emits a " "loud warning at startup. Will be removed." ), ) @@ -718,7 +718,7 @@ def _load_metadata_files( # ``number_of_threads`` is carried as a control dummy (the framework owns # no per-thread state yet, so its value is not consumed — kept for symmetry # with ``number_of_instances`` and future per-thread sizing). -# (A chunk/block index is intentionally NOT a control pair: capgen-ng's +# (A chunk/block index is intentionally NOT a control pair: capgen's # slice-based design passes the current chunk as a horizontal range via # horizontal_loop_begin/end, so no scheme ever indexes by chunk inside a call.) # Each entry: (index std_name, count std_name, index description, count description). diff --git a/capgen-ng/ccpp_datafile.py b/capgen/ccpp_datafile.py similarity index 99% rename from capgen-ng/ccpp_datafile.py rename to capgen/ccpp_datafile.py index 8976fedf..4ebfcd62 100755 --- a/capgen-ng/ccpp_datafile.py +++ b/capgen/ccpp_datafile.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""Query CLI for the ``datatable.xml`` produced by ``ccpp_capgen_ng.py``. +"""Query CLI for the ``datatable.xml`` produced by ``ccpp_capgen.py``. This is the read-side companion to :mod:`generator.datatable`. The writer half lives in the generator package; this module is a pure-Python, @@ -24,7 +24,7 @@ Exactly one report action is required per invocation. -Notes specific to capgen-ng +Notes specific to capgen --------------------------- * ``--host-files`` returns ``_ccpp_cap.F90`` (the per-host static API; filename and module name derived from ``--host-name`` at generation time). @@ -32,7 +32,7 @@ artifacts (``ccpp_.meta``, ``ccpp__expanded.xml``) are reported via ``--inspection-files``. * ``--process-list`` is supported syntactically but will return an empty - string: capgen-ng does not currently record a ``process`` attribute on + string: capgen does not currently record a ``process`` attribute on scheme entries. """ @@ -167,7 +167,7 @@ def valid_actions(cls): def _command_line_parser(): """Create and return an ArgumentParser for parsing the command line.""" description = """ - Retrieve information about a ccpp_capgen_ng run. + Retrieve information about a ccpp_capgen run. The returned information is controlled by selecting an action from the list of optional arguments below. Note that exactly one action is required. @@ -297,7 +297,7 @@ def _retrieve_scheme_files(table): (or ``.F`` / ``.f90`` / ``.f``) sources for schemes that the loaded suites actually reference. Build systems use this to compile exactly the scheme set the suites consume; unreferenced scheme metadata - files passed on the capgen-ng CLI for convenience are filtered out. + files passed on the capgen CLI for convenience are filtered out. # Test valid scheme files >>> table = ET.fromstring(""\ @@ -345,7 +345,7 @@ def _retrieve_scheme_files(table): def _retrieve_inspection_files(table, file_type=None): """Find and retrieve a list of inspection filenames from
. - Inspection files are non-Fortran artifacts emitted by capgen-ng for + Inspection files are non-Fortran artifacts emitted by capgen for debugging and downstream tooling: suite ``.meta`` files and expanded suite-definition XML. Each kind lives in its own subsection of ````. @@ -390,7 +390,7 @@ def _retrieve_inspection_files(table, file_type=None): def _retrieve_process_list(table): """Find and return a list of all physics scheme processes in
. - capgen-ng does not currently record a ``process`` attribute on + capgen does not currently record a ``process`` attribute on scheme entries, so this returns an empty list when no scheme carries one. The flag is kept for CLI compatibility. diff --git a/capgen-ng/ccpp_validator.py b/capgen/ccpp_validator.py similarity index 99% rename from capgen-ng/ccpp_validator.py rename to capgen/ccpp_validator.py index 7b42f71e..1247e250 100755 --- a/capgen-ng/ccpp_validator.py +++ b/capgen/ccpp_validator.py @@ -59,7 +59,7 @@ import sys from typing import Dict, List, NamedTuple, Optional, Set, Tuple -# Ensure the capgen-ng package is importable when invoked directly. +# Ensure the capgen package is importable when invoked directly. _SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) _PACKAGE_DIR = os.path.dirname(_SCRIPT_DIR) if _PACKAGE_DIR not in sys.path: @@ -1750,7 +1750,7 @@ def _build_parser() -> argparse.ArgumentParser: "TRANSIENT MIGRATION SHIM. Accept legacy CCPP standard " "names (currently 'horizontal_loop_extent') in scheme " "metadata and silently rewrite them to their canonical " - "capgen-ng equivalents ('horizontal_dimension'). Emits a " + "capgen equivalents ('horizontal_dimension'). Emits a " "loud warning at startup. Will be removed." ), ) @@ -1758,7 +1758,7 @@ def _build_parser() -> argparse.ArgumentParser: # effect inside generator.suite_resolver._canonical_dim, which the # validator never invokes (the validator compares metadata against # Fortran source, not host metadata against scheme metadata), so - # the flag would be a no-op. See capgen-ng/ccpp_capgen_ng.py. + # the flag would be a no-op. See capgen/ccpp_capgen.py. # auto-clone-constituents: transient legacy shim. This one DOES # belong on the validator because the shim extends the parser's # ``_KNOWN_ATTRS`` set — without the flag the validator rejects @@ -1774,7 +1774,7 @@ def _build_parser() -> argparse.ArgumentParser: "TRANSIENT LEGACY SHIM. Accept four legacy constituent " "attributes (default_value, min_value, water_species, " "mixing_ratio_type) on scheme args. Mirrors the same " - "flag on ccpp_capgen_ng so legacy scheme metadata that " + "flag on ccpp_capgen so legacy scheme metadata that " "needs auto-clone-static-constituent codegen can be " "validated against its Fortran source. Emits a loud " "warning at startup. Will be removed." diff --git a/capgen/generator/__init__.py b/capgen/generator/__init__.py new file mode 100644 index 00000000..9eabbc64 --- /dev/null +++ b/capgen/generator/__init__.py @@ -0,0 +1 @@ +"""Cap code generation for ccpp-capgen.""" diff --git a/capgen-ng/generator/datatable.py b/capgen/generator/datatable.py similarity index 98% rename from capgen-ng/generator/datatable.py rename to capgen/generator/datatable.py index 9053b52c..97235ed9 100644 --- a/capgen-ng/generator/datatable.py +++ b/capgen/generator/datatable.py @@ -6,7 +6,7 @@ XML layout ---------- -Two top-level file sections partition capgen-ng's outputs by language: +Two top-level file sections partition capgen's outputs by language: * ```` lists *Fortran sources only* — utilities, the host-facing API, and per-suite cap modules. All paths here end in ``.F90`` and are @@ -129,13 +129,13 @@ def _build_capgen_files( ) -> None: """Append ```` to *root*. - ```` lists Fortran sources generated by capgen-ng (utilities, + ```` lists Fortran sources generated by capgen (utilities, the host-facing API, and per-suite caps). Non-Fortran inspection artifacts (e.g. ``.meta`` files, expanded SDFs) live in ```` and are emitted by :func:`_build_inspection_files`. ``host_file_paths`` lists files generated for the host-facing API. In - capgen-ng this is the static API (``_ccpp_cap.F90``); the section is + capgen this is the static API (``_ccpp_cap.F90``); the section is emitted unconditionally (possibly empty) to keep the schema stable. """ capgen_files = ET.SubElement(root, 'capgen_files') @@ -164,7 +164,7 @@ def _build_scheme_files( Lists the Fortran source files of schemes actually referenced by some loaded suite (group phase calls + suite-level ```` / ````). - These files are *not* generated by capgen-ng — they are the + These files are *not* generated by capgen — they are the user-supplied scheme implementations the build system must compile. The section is always written (possibly empty) so the schema stays stable for ``ccpp_datafile.py`` consumers. @@ -182,7 +182,7 @@ def _build_inspection_files( ) -> None: """Append ```` to *root*. - Inspection artifacts are non-Fortran outputs that capgen-ng emits for + Inspection artifacts are non-Fortran outputs that capgen emits for debugging and downstream tooling. They are *not* compiled. Each kind of artifact lives in its own subsection so consumers can pick a specific file type or take them all together via ``--inspection-files``. @@ -400,7 +400,7 @@ def write_datatable( output_root : str Output directory. host_file_paths : list of str, optional - Absolute paths to host-facing API files (capgen-ng emits + Absolute paths to host-facing API files (capgen emits ``_ccpp_cap.F90`` here). The ```` section is always written (possibly empty). scheme_file_paths : list of str, optional diff --git a/capgen-ng/generator/group_cap.py b/capgen/generator/group_cap.py similarity index 99% rename from capgen-ng/generator/group_cap.py rename to capgen/generator/group_cap.py index 8d25f0cb..53927a7c 100644 --- a/capgen-ng/generator/group_cap.py +++ b/capgen/generator/group_cap.py @@ -1157,7 +1157,7 @@ def _generate_group_cap( # ---- module header -------------------------------------------------- lines.append( - '! ccpp_{}_{}_cap.F90 -- generated by ccpp_capgen_ng, do not edit'.format( + '! ccpp_{}_{}_cap.F90 -- generated by ccpp_capgen, do not edit'.format( suite_name, group_name ) ) diff --git a/capgen-ng/generator/host_cap.py b/capgen/generator/host_cap.py similarity index 99% rename from capgen-ng/generator/host_cap.py rename to capgen/generator/host_cap.py index 1c4dabd5..f790b318 100644 --- a/capgen-ng/generator/host_cap.py +++ b/capgen/generator/host_cap.py @@ -1013,7 +1013,7 @@ def _generate_host_cap( lines: List[str] = [] lines.append( - '! {}.F90 -- generated by ccpp_capgen_ng, do not edit'.format(mod_name) + '! {}.F90 -- generated by ccpp_capgen, do not edit'.format(mod_name) ) lines.append('module {}'.format(mod_name)) lines.append('') diff --git a/capgen-ng/generator/host_constituents.py b/capgen/generator/host_constituents.py similarity index 99% rename from capgen-ng/generator/host_constituents.py rename to capgen/generator/host_constituents.py index e280a8e2..6109bb10 100644 --- a/capgen-ng/generator/host_constituents.py +++ b/capgen/generator/host_constituents.py @@ -2,7 +2,7 @@ """Generate ``ccpp_host_constituents.F90`` — the host-wide constituent module. -In capgen-ng's per-instance design, the constituent state is sized to +In capgen's per-instance design, the constituent state is sized to ``number_of_instances`` (declared by the host's ``type=control`` table, paired with ``instance_number``). This module owns: @@ -575,7 +575,7 @@ def _generate_host_constituents( lines: List[str] = [] lines.append( - '! ccpp_host_constituents.F90 -- generated by ccpp_capgen_ng, do not edit' + '! ccpp_host_constituents.F90 -- generated by ccpp_capgen, do not edit' ) lines.append('module {}'.format(_HOST_CONST_MOD)) lines.append('') diff --git a/capgen-ng/generator/kinds_writer.py b/capgen/generator/kinds_writer.py similarity index 96% rename from capgen-ng/generator/kinds_writer.py rename to capgen/generator/kinds_writer.py index 9597c9ca..c6cbf4dd 100644 --- a/capgen-ng/generator/kinds_writer.py +++ b/capgen/generator/kinds_writer.py @@ -14,7 +14,7 @@ Example output for ``--kind-type kind_phys=REAL64`` (default module):: - ! ccpp_kinds.F90 -- generated by ccpp_capgen_ng, do not edit + ! ccpp_kinds.F90 -- generated by ccpp_capgen, do not edit module ccpp_kinds use iso_fortran_env, only: REAL64 @@ -27,7 +27,7 @@ Example output for ``--kind-type kind_phys=my_host_kinds:kind_r8`` (host module):: - ! ccpp_kinds.F90 -- generated by ccpp_capgen_ng, do not edit + ! ccpp_kinds.F90 -- generated by ccpp_capgen, do not edit module ccpp_kinds use my_host_kinds, only: kind_r8 @@ -126,7 +126,7 @@ def _generate_ccpp_kinds(kind_types: KindMap) -> List[str]: -------- >>> lines = _generate_ccpp_kinds({'kind_phys': ('iso_fortran_env', 'REAL64')}) >>> lines[0] - '! ccpp_kinds.F90 -- generated by ccpp_capgen_ng, do not edit' + '! ccpp_kinds.F90 -- generated by ccpp_capgen, do not edit' >>> 'module ccpp_kinds' in lines True >>> any('kind_phys' in l and 'REAL64' in l for l in lines) @@ -190,7 +190,7 @@ def _generate_ccpp_kinds(kind_types: KindMap) -> List[str]: by_module[mod].sort() lines: List[str] = [ - '! ccpp_kinds.F90 -- generated by ccpp_capgen_ng, do not edit', + '! ccpp_kinds.F90 -- generated by ccpp_capgen, do not edit', 'module {}'.format(_KINDS_MODULE), ] diff --git a/capgen-ng/generator/suite_cap.py b/capgen/generator/suite_cap.py similarity index 99% rename from capgen-ng/generator/suite_cap.py rename to capgen/generator/suite_cap.py index a9b556d2..af3a8ddd 100644 --- a/capgen-ng/generator/suite_cap.py +++ b/capgen/generator/suite_cap.py @@ -413,7 +413,7 @@ def _register_lines( inst_idx = _instance_idx(host_dict) # ``number_of_instances`` is now a paired control variable (see - # ccpp_capgen_ng._PAIRED_OPTIONAL_CTRL_VARS). When present it enters + # ccpp_capgen._PAIRED_OPTIONAL_CTRL_VARS). When present it enters # the suite-cap signature as a dummy alongside ``instance_number``; # the framework consumes it at register-time to size the per-instance # state arrays. When absent we fall back to the literal ``1`` @@ -457,7 +457,7 @@ def _register_lines( # Constituent merge: declare a per-scheme array temporary and a counter. has_consts = bool(suite_res.constituent_register_calls) # auto-clone-constituents: the legacy shim contributes additional - # constituent registrations synthesised in capgen-ng from + # constituent registrations synthesised in capgen from # is_constituent consumer metadata; ``has_dyn_consts`` covers # both sources so the buffer-allocation + counter machinery is # set up whenever any synthesised constituent will be emitted. @@ -1237,7 +1237,7 @@ def _generate_suite_cap( # Module header. lines.append( - '! ccpp_{}_cap.F90 -- generated by ccpp_capgen_ng, do not edit'.format( + '! ccpp_{}_cap.F90 -- generated by ccpp_capgen, do not edit'.format( suite_name ) ) diff --git a/capgen-ng/generator/suite_data.py b/capgen/generator/suite_data.py similarity index 99% rename from capgen-ng/generator/suite_data.py rename to capgen/generator/suite_data.py index bc1d9134..1864c37e 100644 --- a/capgen-ng/generator/suite_data.py +++ b/capgen/generator/suite_data.py @@ -209,7 +209,7 @@ def _generate_suite_data( lines: List[str] = [] lines.append( - '! ccpp_{}_data.F90 -- generated by ccpp_capgen_ng, do not edit'.format( + '! ccpp_{}_data.F90 -- generated by ccpp_capgen, do not edit'.format( suite_name ) ) @@ -455,7 +455,7 @@ def _generate_suite_meta( i1 = _INDENT lines: List[str] = [] lines.append( - '! ccpp_{}_data.meta -- generated by ccpp_capgen_ng, do not edit'.format(suite_name) + '! ccpp_{}_data.meta -- generated by ccpp_capgen, do not edit'.format(suite_name) ) lines.append('[ccpp-table-properties]') lines.append('{}name = {}'.format(i1, mod_name)) diff --git a/capgen-ng/generator/suite_resolver.py b/capgen/generator/suite_resolver.py similarity index 99% rename from capgen-ng/generator/suite_resolver.py rename to capgen/generator/suite_resolver.py index 6113ad5b..006dc7d2 100644 --- a/capgen-ng/generator/suite_resolver.py +++ b/capgen/generator/suite_resolver.py @@ -80,7 +80,7 @@ # Dimension standard names that map to horizontal loop bounds. The # legacy spelling ``horizontal_loop_extent`` is rejected at parse time -# (see ``_FORBIDDEN_DIMENSION_NAMES`` in ccpp_capgen_ng.py) or rewritten +# (see ``_FORBIDDEN_DIMENSION_NAMES`` in ccpp_capgen.py) or rewritten # by the ``--legacy-mode`` shim, so it can never appear here. _HORIZ_LOOP_DIMS: frozenset = frozenset({ 'horizontal_dimension', @@ -159,7 +159,7 @@ def _index_symbol_name(base_std_name: str) -> str: emit/reference sites (host_constituents.py public/declaration/ reset/const_index/init-guard, suite_resolver.py auto-provisioned subscript, Path 1a call_expr) MUST route through this helper to - keep the symbol consistent within a single capgen-ng run. The + keep the symbol consistent within a single capgen run. The underlying std_name is still passed to ``const_index`` as a string literal, so the framework lookup keys remain unchanged -- only the Fortran-side mapping symbol is mangled. @@ -224,7 +224,7 @@ def _index_symbol_name(base_std_name: str) -> str: def _constituent_module_name(suite_name: str) -> str: """Return the module name that owns the host-wide constituent state. - Constant across suites: in capgen-ng (option A, matching original + Constant across suites: in capgen (option A, matching original capgen) the constituent object is host-wide, not suite-local. """ return _HOST_CONST_MOD @@ -588,7 +588,7 @@ def _one_dim_part( # Registered scalar-index dimension: collapse to the paired index # variable's local Fortran name regardless of lower bound. See - # capgen-ng/metadata/registered_dimensions.py for the contract. + # capgen/metadata/registered_dimensions.py for the contract. idx_std = scalar_index_for(upper_str) if idx_std is not None: idx_entry = host_dict.get(idx_std) @@ -600,7 +600,7 @@ def _one_dim_part( "or type=host table. Either declare '{idx}' as a scalar " "integer in the host control/host metadata, or remove " "the '{dim}' dimension from the affected metadata. See " - "capgen-ng/metadata/registered_dimensions.py for the full " + "capgen/metadata/registered_dimensions.py for the full " "table of registered scalar-index pairings.".format( dim=upper_str, idx=idx_std, ) @@ -2548,7 +2548,7 @@ def resolve_suite( def validate_init_dimensions(suite_res: SuiteResolution) -> None: """Reject suite-owned vars whose suite-time allocation can't be sized. - capgen-ng allocates every non-allocatable, dimensioned suite-owned + capgen allocates every non-allocatable, dimensioned suite-owned variable once in ``suite_data_init_fields``, which runs at the very start of ``_init`` -- before any ``init`` / ``timestep_init`` / ``run`` scheme code. Only the ``register`` phase completes earlier, so diff --git a/capgen-ng/generator/suite_types.py b/capgen/generator/suite_types.py similarity index 99% rename from capgen-ng/generator/suite_types.py rename to capgen/generator/suite_types.py index 20a46fa8..73c5f6fd 100644 --- a/capgen-ng/generator/suite_types.py +++ b/capgen/generator/suite_types.py @@ -165,7 +165,7 @@ def _sanitize_len_suffix(len_spec: str, context: str = '') -> str: if spec == '*': raise CCPPError( "{}character(len=*) cannot appear as a DDT component, so " - "capgen-ng cannot generate a pointer-wrapper type for it. " + "capgen cannot generate a pointer-wrapper type for it. " "Use a concrete length, a parameter constant, or 'len=:' " "(deferred length, paired with allocatable / pointer) " "in the metadata instead.".format(prefix) @@ -422,7 +422,7 @@ def _generate_suite_types( lines: List[str] = [] lines.append( - '! {}.F90 -- generated by ccpp_capgen_ng, do not edit'.format(mod_name) + '! {}.F90 -- generated by ccpp_capgen, do not edit'.format(mod_name) ) lines.append('module {}'.format(mod_name)) lines.append('') diff --git a/capgen-ng/generator/suite_xml.py b/capgen/generator/suite_xml.py similarity index 98% rename from capgen-ng/generator/suite_xml.py rename to capgen/generator/suite_xml.py index d04cbad8..fb8f50e7 100644 --- a/capgen-ng/generator/suite_xml.py +++ b/capgen/generator/suite_xml.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""Suite Definition File (SDF) parser for ccpp-capgen-ng. +"""Suite Definition File (SDF) parser for ccpp-capgen. A Suite Definition File is an XML document that describes which physics schemes to run, in which order, and how to group them. This module: @@ -58,7 +58,7 @@ Schema location --------------- -The XSD files live in ``capgen-ng/schema/``. Their path is resolved relative +The XSD files live in ``capgen/schema/``. Their path is resolved relative to this source file at runtime, so no extra configuration is needed. """ @@ -87,17 +87,17 @@ ) #: Accepted ```` element name. The old ccpp-prebuild schema also -#: tolerated ```` (typo) and ````; capgen-ng +#: tolerated ```` (typo) and ````; capgen #: rejects both with a hard error so SDFs migrate to the canonical #: short name. _INIT_TAG = 'init' #: Accepted ```` element name. The old ccpp-prebuild schema -#: also tolerated ````; capgen-ng rejects it. +#: also tolerated ````; capgen rejects it. _FINAL_TAG = 'final' #: Element names that were valid in the old schema but are rejected in -#: capgen-ng's v2.0 SDF format. Map to the canonical short form so the +#: capgen's v2.0 SDF format. Map to the canonical short form so the #: error message can point at the right replacement. _REJECTED_INIT_TAGS = frozenset({'initalize', 'initialize'}) _REJECTED_FINAL_TAGS = frozenset({'finalize'}) @@ -540,7 +540,7 @@ def parse_suite_xml( Logger. A module-level logger is used if ``None``. schema_path : str, optional Directory containing XSD files. Defaults to the bundled - ``capgen-ng/schema/`` directory. + ``capgen/schema/`` directory. skip_validation : bool If ``True``, skip XML schema validation (useful in test environments where ``xmllint`` is not available). diff --git a/capgen-ng/generator/trace.py b/capgen/generator/trace.py similarity index 100% rename from capgen-ng/generator/trace.py rename to capgen/generator/trace.py diff --git a/capgen/metadata/__init__.py b/capgen/metadata/__init__.py new file mode 100644 index 00000000..f84ce9a7 --- /dev/null +++ b/capgen/metadata/__init__.py @@ -0,0 +1 @@ +"""Metadata parsing and variable resolution for ccpp-capgen.""" diff --git a/capgen-ng/metadata/auto_clone_constituents.py b/capgen/metadata/auto_clone_constituents.py similarity index 94% rename from capgen-ng/metadata/auto_clone_constituents.py rename to capgen/metadata/auto_clone_constituents.py index 919b71ad..1e0a1233 100644 --- a/capgen-ng/metadata/auto_clone_constituents.py +++ b/capgen/metadata/auto_clone_constituents.py @@ -4,12 +4,12 @@ every ``is_constituent`` scheme arg by lifting its metadata properties (``long_name``, ``diagnostic_name``, ``units``, ``default_value``, …) into a synthetic ``%instantiate(...)`` call emitted into the generated -host cap. Capgen-ng deliberately dropped that path in favour of +host cap. Capgen deliberately dropped that path in favour of explicit registration (``host_constituents`` host arg + register-phase ``ccpp_constituent_properties_t(:)`` scheme args). This module re-enables the legacy auto-clone path as an opt-in shim -(``--legacy-auto-clone-constituents`` on the capgen-ng CLI). It exists +(``--legacy-auto-clone-constituents`` on the capgen CLI). It exists for legacy host models — notably CAM-SIMA — that drive original capgen heavily today and have not yet migrated to explicit registration. @@ -31,10 +31,10 @@ The legacy code paths these models came from never supported multiple in-memory host instances. The shim follows the same restriction: when enabled, the host metadata MUST NOT declare both ``instance_number`` -and ``number_of_instances`` (the capgen-ng multi-instance opt-in pair +and ``number_of_instances`` (the capgen multi-instance opt-in pair — see :data:`metadata.registered_dimensions.SCALAR_INDEX_DIMS`). The gate is enforced in :func:`require_single_instance_host`, called from -:mod:`ccpp_capgen_ng` after host metadata parse. Code paths under the +:mod:`ccpp_capgen` after host metadata parse. Code paths under the shim assume ``instance_number`` is the literal ``1``. ## Self-contained for clean removal @@ -51,7 +51,7 @@ marked with a ``# auto-clone-constituents:`` comment). Every hook is a no-op when the mode is not enabled — the shim has zero -impact on default capgen-ng workflows. +impact on default capgen workflows. Examples -------- @@ -94,7 +94,7 @@ # # The remaining %instantiate kwargs (std_name, long_name, diag_name, # units, vertical_dim, advected, molar_mass) are already accepted by -# the strict-mode parser, just under canonical capgen-ng names. +# the strict-mode parser, just under canonical capgen names. # ---------------------------------------------------------------------- _EXTRA_KNOWN_ATTRS: FrozenSet[str] = frozenset({ 'default_value', @@ -157,7 +157,7 @@ def _pad(s: str) -> str: _pad(''), _pad('This is a TRANSIENT shim for legacy hosts that have'), _pad('not migrated to explicit registration. It WILL BE'), - _pad('REMOVED in a future capgen-ng release.'), + _pad('REMOVED in a future capgen release.'), border, '', ] @@ -214,7 +214,7 @@ def require_single_instance_host(host_dict) -> None: for every per-instance subscript. Multi-instance support is not in scope for this transient shim — legacy hosts that need this path were always single-instance. Called from - :mod:`ccpp_capgen_ng` after the host metadata has been flattened. + :mod:`ccpp_capgen` after the host metadata has been flattened. No-op when the shim is disabled. Accepts the resolved ``host_dict`` flat mapping (or any container with ``__contains__``) diff --git a/capgen-ng/metadata/dim_aliases.py b/capgen/metadata/dim_aliases.py similarity index 97% rename from capgen-ng/metadata/dim_aliases.py rename to capgen/metadata/dim_aliases.py index fe9f8d1a..faa22187 100644 --- a/capgen-ng/metadata/dim_aliases.py +++ b/capgen/metadata/dim_aliases.py @@ -17,7 +17,7 @@ side) are compared for identity*. This module provides an opt-in shim (``--gfs-dim-aliases`` on the -capgen-ng / ccpp_validator CLI) that collapses each member of an alias +capgen / ccpp_validator CLI) that collapses each member of an alias group to a single canonical representative when ``_canonical_dim`` prepares a dimension entry for the strict identity comparison in :func:`generator.suite_resolver._check_compat`. Every other consumer @@ -141,7 +141,7 @@ def _pad(s: str) -> str: _pad(''), _pad('Variables keep their original names everywhere'), _pad('else. This is a TRANSIENT GFS-physics shim and'), - _pad('WILL BE REMOVED in a future capgen-ng release.'), + _pad('WILL BE REMOVED in a future capgen release.'), border, '', ] @@ -183,7 +183,7 @@ def canonical(name: str) -> str: When the shim is **disabled** this is a strict identity — the aliased names compare distinct, just as they would in a default - capgen-ng workflow. When the shim is **enabled** every entry in + capgen workflow. When the shim is **enabled** every entry in :data:`_DIM_ALIAS_MAP` collapses to its representative; everything else passes through unchanged. diff --git a/capgen-ng/metadata/legacy_compat.py b/capgen/metadata/legacy_compat.py similarity index 94% rename from capgen-ng/metadata/legacy_compat.py rename to capgen/metadata/legacy_compat.py index cb380914..059a81f4 100644 --- a/capgen-ng/metadata/legacy_compat.py +++ b/capgen/metadata/legacy_compat.py @@ -1,9 +1,9 @@ """TRANSIENT compatibility shim for legacy CCPP standard names. The original ccpp-prebuild + ccpp-capgen toolchain used the standard -name ``horizontal_loop_extent`` where capgen-ng uses +name ``horizontal_loop_extent`` where capgen uses ``horizontal_dimension``. This module provides an opt-in shim -(``--legacy-mode`` on the capgen-ng / ccpp_validator CLI) that +(``--legacy-mode`` on the capgen / ccpp_validator CLI) that silently rewrites legacy names to their canonical equivalents at metadata parse time so the rest of the toolchain only ever sees the canonical names. @@ -60,12 +60,12 @@ # ---------------------------------------------------------------------- _LEGACY_NAME_MAP: Dict[str, str] = { # ccpp-prebuild / original ccpp-capgen used ``horizontal_loop_extent`` - # in scheme metadata where capgen-ng uses ``horizontal_dimension``. + # in scheme metadata where capgen uses ``horizontal_dimension``. 'horizontal_loop_extent': 'horizontal_dimension', # Legacy CCPP-physics hosts (and SCM 17p8 in particular) sized # per-thread DDT containers by ``number_of_openmp_threads``; the - # capgen-ng convention is ``number_of_threads`` (matching the + # capgen convention is ``number_of_threads`` (matching the # ``thread_number`` control variable name). Aliasing here lets the # host metadata flow through unchanged; once hosts have migrated, # drop this entry. @@ -137,7 +137,7 @@ def _pad(s: str) -> str: _pad(''), _pad('This is a TRANSIENT migration shim. Update your'), _pad('metadata to use the canonical names; legacy mode'), - _pad('WILL BE REMOVED in a future capgen-ng release.'), + _pad('WILL BE REMOVED in a future capgen release.'), border, '', ] @@ -184,7 +184,7 @@ def translate(name: str) -> str: The function is tolerant of any input that lookup-by-string is valid for (``str``). Callers may pre-lowercase the input — the - map keys are already lowercase to match capgen-ng's + map keys are already lowercase to match capgen's case-insensitive standard-name convention. """ if not _ENABLED: diff --git a/capgen-ng/metadata/metadata_table.py b/capgen/metadata/metadata_table.py similarity index 99% rename from capgen-ng/metadata/metadata_table.py rename to capgen/metadata/metadata_table.py index 568c73e3..995ca7c6 100644 --- a/capgen-ng/metadata/metadata_table.py +++ b/capgen/metadata/metadata_table.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""Metadata table parser for ccpp-capgen-ng. +"""Metadata table parser for ccpp-capgen. Each ``.meta`` file contains one or more CCPP metadata tables. Every table begins with a ``[ccpp-table-properties]`` header, followed by one or more @@ -990,7 +990,7 @@ def __init__(self, table_name: str, table_type: str, # lines targeting the actual module rather than the table name. self.module_name: str = '' # Each entry is ``(kind_name, module, spec)``; aggregated by - # ccpp_capgen_ng into the kind map for ccpp_kinds.F90. + # ccpp_capgen into the kind map for ccpp_kinds.F90. self.kind_specs: List[Tuple[str, str, str]] = [] def apply_table_props(self, props: dict) -> None: diff --git a/capgen-ng/metadata/parse_tools/__init__.py b/capgen/metadata/parse_tools/__init__.py similarity index 100% rename from capgen-ng/metadata/parse_tools/__init__.py rename to capgen/metadata/parse_tools/__init__.py diff --git a/capgen-ng/metadata/parse_tools/fortran_conditional.py b/capgen/metadata/parse_tools/fortran_conditional.py similarity index 100% rename from capgen-ng/metadata/parse_tools/fortran_conditional.py rename to capgen/metadata/parse_tools/fortran_conditional.py diff --git a/capgen-ng/metadata/parse_tools/io_helpers.py b/capgen/metadata/parse_tools/io_helpers.py similarity index 98% rename from capgen-ng/metadata/parse_tools/io_helpers.py rename to capgen/metadata/parse_tools/io_helpers.py index be74d464..44108fee 100644 --- a/capgen-ng/metadata/parse_tools/io_helpers.py +++ b/capgen/metadata/parse_tools/io_helpers.py @@ -4,7 +4,7 @@ generated cap files when their content was unchanged — preserving each file's mtime so downstream build systems (CMake, Make, Ninja) don't trigger unnecessary recompilation cascades. This module reproduces that -behaviour for ``capgen-ng``. +behaviour for ``capgen``. Staging strategy ---------------- @@ -51,7 +51,7 @@ def write_if_changed( encoding : str Encoding passed to :func:`open` for both the read-back comparison and the staged write. Defaults to ``'utf-8'`` to match every - capgen-ng writer. + capgen writer. logger : logging.Logger, optional When supplied, the helper logs an ``info``-level message after each call: ``"Wrote "`` if the file was newly written or diff --git a/capgen-ng/metadata/parse_tools/parse_checkers.py b/capgen/metadata/parse_tools/parse_checkers.py similarity index 100% rename from capgen-ng/metadata/parse_tools/parse_checkers.py rename to capgen/metadata/parse_tools/parse_checkers.py diff --git a/capgen-ng/metadata/parse_tools/parse_log.py b/capgen/metadata/parse_tools/parse_log.py similarity index 100% rename from capgen-ng/metadata/parse_tools/parse_log.py rename to capgen/metadata/parse_tools/parse_log.py diff --git a/capgen-ng/metadata/parse_tools/parse_source.py b/capgen/metadata/parse_tools/parse_source.py similarity index 100% rename from capgen-ng/metadata/parse_tools/parse_source.py rename to capgen/metadata/parse_tools/parse_source.py diff --git a/capgen-ng/metadata/parse_tools/xml_tools.py b/capgen/metadata/parse_tools/xml_tools.py similarity index 100% rename from capgen-ng/metadata/parse_tools/xml_tools.py rename to capgen/metadata/parse_tools/xml_tools.py diff --git a/capgen-ng/metadata/registered_dimensions.py b/capgen/metadata/registered_dimensions.py similarity index 91% rename from capgen-ng/metadata/registered_dimensions.py rename to capgen/metadata/registered_dimensions.py index ee716d7b..eae837a1 100644 --- a/capgen-ng/metadata/registered_dimensions.py +++ b/capgen/metadata/registered_dimensions.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 -"""Registered scalar-index dimensions for capgen-ng. +"""Registered scalar-index dimensions for capgen. This module is the **single source of truth** for the small set of CCPP -standard-name dimensions that capgen-ng treats specially: each one is a +standard-name dimensions that capgen treats specially: each one is a *count* (e.g. ``number_of_instances``, ``number_of_threads``) whose access pattern collapses to a paired scalar *index* variable (e.g. ``instance_number``, ``thread_number``). @@ -19,14 +19,14 @@ type = GFS_interstitial_type dimensions = (number_of_threads) # <-- registered scalar dim -When a scheme reaches into a field of that container, capgen-ng emits the +When a scheme reaches into a field of that container, capgen emits the scalar index automatically:: physics%Interstitial(thread_number)%alpha(lb:ub, 1:nlev) ^^^^^^^^^^^^^^^ substituted from the registered scalar-index pair -The same machinery applies anywhere capgen-ng needs to subscript a +The same machinery applies anywhere capgen needs to subscript a container by an index variable that the host carries as a separate control/host variable. @@ -35,7 +35,7 @@ **Rule 1 (generalized, NOT enforced as a hard gate)**: A container DDT-instance variable may carry registered scalar-index -dimensions in its ``dimensions`` clause. When it does, capgen-ng emits +dimensions in its ``dimensions`` clause. When it does, capgen emits the paired index variable's local Fortran name at every call site that reaches into the container. Anything *not* in :data:`SCALAR_INDEX_DIMS` flows through the normal slice/bounds machinery @@ -53,7 +53,7 @@ Why both rules matter --------------------- -* Rule 1 generalization lets capgen-ng support multi-instance +* Rule 1 generalization lets capgen support multi-instance (``number_of_instances``) **and** per-thread (``number_of_threads``) container DDTs from one mechanism — no per-dimension code path. * Rule 2 keeps the substitution mechanism contained: the rule "leaves @@ -88,7 +88,7 @@ Variable '' (standard_name='') declares dimension '' on a leaf-data variable, but '' is a registered scalar-index dimension reserved for DDT-instance containers (see - capgen-ng/metadata/registered_dimensions.py). + capgen/metadata/registered_dimensions.py). [...] it means you wrote something like:: @@ -129,19 +129,19 @@ #: otherwise). #: #: Every entry here is treated as a hard convention across the entire -#: CCPP ecosystem. Adding an entry binds capgen-ng to a specific +#: CCPP ecosystem. Adding an entry binds capgen to a specific #: standard-name pairing; once an entry lands and hosts adopt it, #: removing or renaming it is a breaking change. SCALAR_INDEX_DIMS: Dict[str, str] = { # Multi-instance API: the framework's instance_number paired opt-in. # Hosts that declare instance_number + number_of_instances opt into - # the multi-instance API; capgen-ng auto-substitutes (instance_number) + # the multi-instance API; capgen auto-substitutes (instance_number) # wherever a container DDT carries this dimension. 'number_of_instances': 'instance_number', # Per-thread DDT containers (e.g. ``physics%Interstitial(thread_number)``) # — the host's openmp-thread index. (thread_number, number_of_threads) is - # a paired-optional control pair (see ccpp_capgen_ng._PAIRED_OPTIONAL_CTRL_VARS + # a paired-optional control pair (see ccpp_capgen._PAIRED_OPTIONAL_CTRL_VARS # and doc/migration.md §3.1). A host that dimensions a variable by # ``number_of_threads`` MUST declare the pair — otherwise the collapse # below cannot find ``thread_number`` and raises. diff --git a/capgen-ng/metadata/unit_conversion.py b/capgen/metadata/unit_conversion.py similarity index 100% rename from capgen-ng/metadata/unit_conversion.py rename to capgen/metadata/unit_conversion.py diff --git a/capgen-ng/metadata/variable_resolver.py b/capgen/metadata/variable_resolver.py similarity index 98% rename from capgen-ng/metadata/variable_resolver.py rename to capgen/metadata/variable_resolver.py index d7ca1a97..82cd7236 100644 --- a/capgen-ng/metadata/variable_resolver.py +++ b/capgen/metadata/variable_resolver.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -"""Variable resolution and access-path construction for ccpp-capgen-ng. +"""Variable resolution and access-path construction for ccpp-capgen. This module flattens the host/control/DDT metadata hierarchy into a single keyed dictionary and provides the ``SchemeStore`` lookup table that the code @@ -60,7 +60,7 @@ # ``number_of_instances`` → ``instance_number``, # ``number_of_threads`` → ``thread_number``) lives in a single # documented module so the contract is easy for users and developers to -# find and extend. See ``capgen-ng/metadata/registered_dimensions.py`` +# find and extend. See ``capgen/metadata/registered_dimensions.py`` # for the full table and the two rules that govern it. from .registered_dimensions import ( SCALAR_INDEX_DIMS, @@ -194,7 +194,7 @@ def _validate_leaf_dims(var: 'MetaVar', source_label: str) -> None: "will emit '({idx})%{name}(...)' at every scheme " "call site automatically.\n" "\n" - "See capgen-ng/metadata/registered_dimensions.py for the full " + "See capgen/metadata/registered_dimensions.py for the full " "table of registered scalar-index pairings and how to extend " "it.".format( name=var.local_name, @@ -382,7 +382,7 @@ def build_ddt_module_map( 2. **Co-located table's resolved module.** Failing the DDT's own override, inherit from a co-located ``host``, ``control``, or ``scheme`` table in the same ``.meta`` file. Its module is - resolved by the same rule used elsewhere in capgen-ng + resolved by the same rule used elsewhere in capgen (:func:`_resolve_module_name`): the co-located table's own ``module_name = …`` if declared, else its table name. @@ -508,7 +508,7 @@ def _flatten_ddt_instance( ddt_table = ddt_index[ddt_name] subscript = _instance_subscript(var) # If the DDT instance has dimensions but NONE of them are a - # registered scalar-index dim, capgen-ng can't bake a meaningful + # registered scalar-index dim, capgen can't bake a meaningful # scalar subscript into field access paths. Two outcomes are both # legitimate, depending on how schemes use this DDT: # @@ -519,7 +519,7 @@ def _flatten_ddt_instance( # Fortran the compiler rejects. # (b) Schemes request individual inner fields by standard name, # which would require ``parent%var()%field(…)`` access - # with a meaningful ```` capgen-ng can't synthesize. + # with a meaningful ```` capgen can't synthesize. # # Skip the recursion either way: the DDT-instance's own entry is # still recorded (case (a) just works), and case (b) trips the @@ -848,7 +848,7 @@ def build_from(cls, scheme_tables: List[MetadataTable]) -> 'SchemeStore': if first_path == dup_path: hint = (' (both paths are identical — likely a ' 'duplicate entry in the --scheme-files ' - 'list passed to capgen-ng)') + 'list passed to capgen)') else: hint = '' raise CCPPError( diff --git a/capgen-ng/schema/suite_v1_0.xsd b/capgen/schema/suite_v1_0.xsd similarity index 100% rename from capgen-ng/schema/suite_v1_0.xsd rename to capgen/schema/suite_v1_0.xsd diff --git a/capgen-ng/schema/suite_v2_0.xsd b/capgen/schema/suite_v2_0.xsd similarity index 100% rename from capgen-ng/schema/suite_v2_0.xsd rename to capgen/schema/suite_v2_0.xsd diff --git a/capgen-ng/src/ccpp_constituent_prop_mod.F90 b/capgen/src/ccpp_constituent_prop_mod.F90 similarity index 100% rename from capgen-ng/src/ccpp_constituent_prop_mod.F90 rename to capgen/src/ccpp_constituent_prop_mod.F90 diff --git a/capgen-ng/src/ccpp_constituent_prop_mod.meta b/capgen/src/ccpp_constituent_prop_mod.meta similarity index 100% rename from capgen-ng/src/ccpp_constituent_prop_mod.meta rename to capgen/src/ccpp_constituent_prop_mod.meta diff --git a/capgen-ng/src/ccpp_hash_table.F90 b/capgen/src/ccpp_hash_table.F90 similarity index 100% rename from capgen-ng/src/ccpp_hash_table.F90 rename to capgen/src/ccpp_hash_table.F90 diff --git a/capgen-ng/src/ccpp_hashable.F90 b/capgen/src/ccpp_hashable.F90 similarity index 100% rename from capgen-ng/src/ccpp_hashable.F90 rename to capgen/src/ccpp_hashable.F90 diff --git a/capgen-ng/src/ccpp_scheme_utils.F90 b/capgen/src/ccpp_scheme_utils.F90 similarity index 100% rename from capgen-ng/src/ccpp_scheme_utils.F90 rename to capgen/src/ccpp_scheme_utils.F90 diff --git a/ccpp_constituent_prop_mod.F90.patch b/ccpp_constituent_prop_mod.F90.patch index fcea421a..b1889cac 100644 --- a/ccpp_constituent_prop_mod.F90.patch +++ b/ccpp_constituent_prop_mod.F90.patch @@ -1,5 +1,5 @@ ---- capgen-ng/src/ccpp_constituent_prop_mod.F90 -+++ capgen-ng/src/ccpp_constituent_prop_mod.F90 +--- capgen/src/ccpp_constituent_prop_mod.F90 ++++ capgen/src/ccpp_constituent_prop_mod.F90 @@ -1392,6 +1392,17 @@ type(ccpp_constituent_properties_t), pointer :: cprop character(len=dimname_len) :: dimname diff --git a/doc/auto_clone_constituents.md b/doc/auto_clone_constituents.md index 0f731e24..97db84c9 100644 --- a/doc/auto_clone_constituents.md +++ b/doc/auto_clone_constituents.md @@ -1,6 +1,6 @@ # `--legacy-auto-clone-constituents` — transient shim -A capgen-ng CLI flag that re-enables the **auto-clone-static-constituent** +A capgen CLI flag that re-enables the **auto-clone-static-constituent** registration path the original ccpp-capgen toolchain provided to CAM-SIMA. Off by default; turned on with a single flag and a loud startup banner. Intended as a migration aid — every line of code it @@ -9,17 +9,17 @@ legacy hosts have moved on. ## How to enable it -Pass the flag to both `capgen-ng` and `ccpp_validator` (the +Pass the flag to both `capgen` and `ccpp_validator` (the ccpp-physics build system already does this for `end-to-end-tests/advection_auto_clone/`): ``` -ccpp_capgen_ng.py --legacy-auto-clone-constituents ... +ccpp_capgen.py --legacy-auto-clone-constituents ... ccpp_validator.py --legacy-auto-clone-constituents ... ``` It is **single-instance only**. If the host metadata declares the -`instance_number` + `number_of_instances` pair (capgen-ng's +`instance_number` + `number_of_instances` pair (capgen's multi-instance opt-in), the run aborts with a clear error before parsing any suite. Legacy hosts predate multi-instance support, so this restriction matches the use case. @@ -29,7 +29,7 @@ this restriction matches the use case. Same shape as original capgen's auto-clone: For every scheme argument flagged `advected = True`, `constituent = True`, -or `molar_mass = `, capgen-ng synthesises a `%instantiate(...)` +or `molar_mass = `, capgen synthesises a `%instantiate(...)` call into the generated host code, lifting field values straight from the scheme metadata. The constituent ends up registered in the per-suite dynamic-constituents buffer alongside any constituents the @@ -52,13 +52,13 @@ legacy metadata writes the values in source form. The other `%instantiate` kwargs (`std_name`, `long_name`, `diag_name`, `units`, `vertical_dim`, `advected`, `molar_mass`) -already had accepted spellings in capgen-ng; the shim just wires them +already had accepted spellings in capgen; the shim just wires them into the synthesised call. ## Defaults that match original capgen - **`long_name` is synthesised when missing.** If the scheme metadata - has no `long_name` on a constituent arg, capgen-ng builds one from + has no `long_name` on a constituent arg, capgen builds one from the standard name by replacing underscores with spaces and capitalising the first character. Example: `cloud_liquid_dry_mixing_ratio` → @@ -72,13 +72,13 @@ into the synthesised call. ## What's stricter than original capgen -Capgen-ng's general rules apply even with the flag on. Two of them +Capgen's general rules apply even with the flag on. Two of them trip up legacy fixtures: 1. **Metadata args must match the Fortran subroutine signature.** Original capgen tolerated a metadata arg-table that listed a constituent in `_init` even when the Fortran `_init` - didn't accept it as a dummy. Capgen-ng passes the metadata args + didn't accept it as a dummy. Capgen passes the metadata args at the call site as Fortran keyword arguments, and the validator catches divergence. Either include the constituent as a Fortran dummy, or remove it from the init's arg-table. @@ -104,7 +104,7 @@ register constituents explicitly (`rrtmgp_constituents`, `state_converters`, `geopotential_temp`, `cloud_particle_sedimentation`, …) declare `advected = True intent = inout` on their `_run` arguments and let the framework register the constituent. Without -the flag, capgen-ng's runtime check fires for every consumer +the flag, capgen's runtime check fires for every consumer because no source actually registered the species. The flag closes that gap by re-creating the auto-clone behaviour from the metadata. @@ -120,7 +120,7 @@ The fixture is a port of CAM-SIMA's `ccpp_framework/test/advection_test`. It exercises the full legacy attr surface (`default_value`, `diagnostic_name`, `advected`) and the unusual init-phase `intent = out`-on-base-constituent pattern. Three small edits were -needed to make the port build under capgen-ng: +needed to make the port build under capgen: 1. **`cld_liq.meta`** — in `cld_liq_init`'s `[ cld_liq_array ]` block, change `intent = out` to `intent = inout`. @@ -133,7 +133,7 @@ needed to make the port build under capgen-ng: `(tfreeze, errmsg, errflg)`; the run phase already triggers auto-clone registration of `cloud_ice_dry_mixing_ratio`. -No `long_name` additions to the metadata were necessary — capgen-ng +No `long_name` additions to the metadata were necessary — capgen synthesises the long_name from the standard name automatically (see the defaults section above). @@ -141,7 +141,7 @@ The CTest target `test_advection_auto_clone` passes after these edits. ## When to retire the flag -When all consumers have been moved to capgen-ng's explicit +When all consumers have been moved to capgen's explicit registration model — either by declaring constituents in the host's `host_constituents(:)` array, or by writing a register-phase scheme with a `ccpp_constituent_properties_t(:), intent=out` argument that diff --git a/doc/briefing.md b/doc/briefing.md index e576f54e..34c47057 100644 --- a/doc/briefing.md +++ b/doc/briefing.md @@ -1,4 +1,4 @@ -# capgen-ng — Briefing for CCPP Framework Developers & Power Users +# capgen — Briefing for CCPP Framework Developers & Power Users *Prepared for the 2026-05-14 walk-through; last revised 2026-06-05. Companion document to `doc/migration.md` (the detailed migration @@ -22,7 +22,7 @@ The CCPP Framework runs two code generators today: and even when it does compile it produces unmaintainably large source files; nobody on the team fully understands it. -**`capgen-ng`** starts fresh, drawing lessons from both. Guiding +**`capgen`** starts fresh, drawing lessons from both. Guiding principle: **simplicity of prebuild, feature set of capgen**. What we wanted to fix: @@ -37,9 +37,9 @@ What we wanted to fix: --- -## 2. What capgen-ng is (in one paragraph) +## 2. What capgen is (in one paragraph) -capgen-ng reads metadata for the **host model**, the **physics +capgen reads metadata for the **host model**, the **physics schemes**, and the **suite definition files** (SDFs), produces a small set of Fortran cap modules that bridge them, and writes a `datatable.xml` describing the result for CMake / Make to consume. @@ -105,10 +105,10 @@ For each scheme arg: ### 3.6 Two tools, one parser -- `ccpp_capgen_ng.py` — the code generator. Trusts metadata; no +- `ccpp_capgen.py` — the code generator. Trusts metadata; no Fortran parsing. - `ccpp_validator.py` — the standalone Fortran-vs-metadata checker. - The ONE place capgen-ng parses Fortran. Run by developers / + The ONE place capgen parses Fortran. Run by developers / CMake before generation. Checks per-arg `intent`, `type`, `kind`, and dimension rank; character length must match exactly (`len=*`↔`len=*`, `len=N`↔`len=N`, no wildcard; old-style F77 @@ -123,9 +123,9 @@ Both share the same metadata-parsing library (`metadata/`). --- -## 4. How capgen-ng differs from `ccpp-prebuild` +## 4. How capgen differs from `ccpp-prebuild` -| Topic | prebuild | capgen-ng | +| Topic | prebuild | capgen | |-----------------------------|-----------------------------------|---------------------------------------------------| | Host metadata mechanism | Hard-coded Python dict (`TYPEDEFS_NEW_METADATA`) | Regular `type = ddt` + `type = host` tables | | Framework-owned variables | Not supported | First-class (suite-owned interstitial via Case 2) | @@ -137,9 +137,9 @@ Both share the same metadata-parsing library (`metadata/`). --- -## 5. How capgen-ng differs from `ccpp-capgen` +## 5. How capgen differs from `ccpp-capgen` -| Topic | capgen | capgen-ng | +| Topic | capgen | capgen | |-----------------------------|-----------------------------------|---------------------------------------------------| | Group-cap arguments | Flat fields (1200+ at UFS scale) | DDT arguments (as in prebuild) | | Variable matching algorithm | Five-layer scope-chain promotion | Flat host+control dict + suite-owned discovery (inherited from prebuild — primary reason runtime is comparable to prebuild) | @@ -187,9 +187,9 @@ every touchpoint is grep-tagged for clean removal: | Flag | What it does | Removal grep | |---------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------| -| `--legacy-mode` | Parse-time substitution of two deprecated CCPP standard names (see §6.3 above). Active on both `ccpp_capgen_ng.py` and `ccpp_validator.py`. | `legacy-compat` | -| `--gfs-dim-aliases` | Treats GFS-physics names `adjusted_vertical_layer_dimension_for_radiation` and `vertical_composition_dimension` as equivalent to `vertical_layer_dimension` **inside the upper-bound dim identity check only** (the variables themselves stay distinct). Resolver-only, so `ccpp_capgen_ng.py` carries the flag; `ccpp_validator.py` does not (the validator never reaches the dim canonicaliser). | `dim-aliases` | -| `--legacy-auto-clone-constituents` | Reinstates original ccpp-capgen's auto-clone-static-constituent registration path: every `is_constituent` consumer (`advected = True` / `constituent = True` / `molar_mass = …`) with no register-phase source is auto-registered into the per-suite dynamic-constituents buffer using values lifted straight from the scheme metadata. Adds four legacy `%instantiate` kwargs to the parser (`default_value`, `min_value`, `water_species`, `mixing_ratio_type`). **Single-instance only** — declaring the `instance_number` + `number_of_instances` pair while the flag is on is a hard error. Available on both `ccpp_capgen_ng.py` and `ccpp_validator.py`. | `auto-clone-constituents` | +| `--legacy-mode` | Parse-time substitution of two deprecated CCPP standard names (see §6.3 above). Active on both `ccpp_capgen.py` and `ccpp_validator.py`. | `legacy-compat` | +| `--gfs-dim-aliases` | Treats GFS-physics names `adjusted_vertical_layer_dimension_for_radiation` and `vertical_composition_dimension` as equivalent to `vertical_layer_dimension` **inside the upper-bound dim identity check only** (the variables themselves stay distinct). Resolver-only, so `ccpp_capgen.py` carries the flag; `ccpp_validator.py` does not (the validator never reaches the dim canonicaliser). | `dim-aliases` | +| `--legacy-auto-clone-constituents` | Reinstates original ccpp-capgen's auto-clone-static-constituent registration path: every `is_constituent` consumer (`advected = True` / `constituent = True` / `molar_mass = …`) with no register-phase source is auto-registered into the per-suite dynamic-constituents buffer using values lifted straight from the scheme metadata. Adds four legacy `%instantiate` kwargs to the parser (`default_value`, `min_value`, `water_species`, `mixing_ratio_type`). **Single-instance only** — declaring the `instance_number` + `number_of_instances` pair while the flag is on is a hard error. Available on both `ccpp_capgen.py` and `ccpp_validator.py`. | `auto-clone-constituents` | All three flags are listed as runways, not destinations: drop the underlying legacy spelling from host/scheme metadata and the flag can @@ -225,7 +225,7 @@ find the index variable and errors). Container DDT-instance variables (`physics%Interstitial`, `physics%Coupling`, ...) dimensioned by a count standard name (`number_of_threads`, `number_of_instances`) get their scalar index -inserted **automatically** by capgen-ng. The host metadata declares +inserted **automatically** by capgen. The host metadata declares the dim; the generator emits `physics%Interstitial(thread_number)%alpha(...)` at every call site. @@ -249,7 +249,7 @@ control-variable arguments to the public entry points. --- -## 7. What capgen-ng does NOT support (yet) +## 7. What capgen does NOT support (yet) ### 7.1 Deferred — to be resolved in upcoming work @@ -278,7 +278,7 @@ control-variable arguments to the public entry points. constituents (file is correct-but-empty under host-wins; should not be emitted at all). - **Python linter / formatter pass.** Pick `ruff`, apply across - `capgen-ng/`. + `capgen/`. ### 7.2 Intentionally NOT supported @@ -300,7 +300,7 @@ control-variable arguments to the public entry points. ## 8. Validation and error reporting -A deliberate design choice across capgen-ng: **errors are loud, +A deliberate design choice across capgen: **errors are loud, specific, and actionable**. Examples surfaced during the SCM shake-down: @@ -314,7 +314,7 @@ shake-down: `--scheme-files`. Replaces silent empty-cap emission. - DDT-instance variable with a non-registered scalar-index dim AND flattenable fields → error shows the broken access pattern - capgen-ng WOULD have emitted and quotes the Fortran compiler + capgen WOULD have emitted and quotes the Fortran compiler error verbatim ("Component to the right of a part reference with nonzero rank must not have the POINTER attribute"). - Generated `case default` on `select case(suite_name)` / @@ -370,17 +370,17 @@ don't rebuild downstream objects unless something actually moved. ## 10. Where things stand right now -- **Unit tests**: 1516 passing on `feature/capgen-ng` (as of +- **Unit tests**: 1516 passing on `feature/capgen` (as of 2026-06-05). - **End-to-end tests passing** (12): `advection`, - `advection_auto_clone`, `capgen_ng`, `chunked_data`, + `advection_auto_clone`, `capgen`, `chunked_data`, `constituents_dim`, `ddthost`, `instances`, `instances_advection`, `nested_suite`, `opt_arg`, `suite_allocate`, `var_compat`. The two newest — `constituents_dim` (a variable dimensioned by `number_of_ccpp_constituents`) and `suite_allocate` (suite-owned allocatable interstitials sized by a scheme-written dimension) — were added while hardening the CAM-SIMA HPC build. -- **Code size**: ~17.8k LOC of Python under `capgen-ng/` (includes +- **Code size**: ~17.8k LOC of Python under `capgen/` (includes docstrings, inline comments, and the three transient shim modules) + ~18k LOC of unit/doctest under `unit-tests/`. Still procedural, still flat data classes. @@ -391,7 +391,7 @@ don't rebuild downstream objects unless something actually moved. cleanup pass once the underlying legacy spelling is gone from host/scheme metadata. - **CCPP-SCM**: actively driving development — every build / runtime - failure surfaced this month landed as a fix in capgen-ng (rather + failure surfaced this month landed as a fix in capgen (rather than being patched around in the host). Most of the `phys_ps` group now builds end-to-end via `--legacy-mode` + `--gfs-dim-aliases`. On 2026-05-20 the per-arg-attribute validator caught **67 real @@ -429,12 +429,12 @@ don't rebuild downstream objects unless something actually moved. - **UFS Weather Model**: not yet attempted; SCM is the proving ground first. An anticipated complication is the "fast physics" called directly from the FV3 dynamical core as a separate group. -- **CAM-SIMA**: **reconnected (2026-06-03 → 06-05).** capgen-ng now +- **CAM-SIMA**: **reconnected (2026-06-03 → 06-05).** capgen now drives the real CAM-SIMA build on Derecho via a thin compatibility layer (`cime_config/capgen_compat/`, in the CAM-SIMA tree) that re-implements original ccpp-capgen's Python API surface (`cap_database`, `host_model_dict`, `call_list`, the per-variable - `Var` accessors) on top of capgen-ng's `datatable.xml` + + `Var` accessors) on top of capgen's `datatable.xml` + `ResolvedArg` / `HostVarEntry`. CAM-SIMA's `cam_autogen.py`, `generate_registry_data.py`, and `write_init_files.py` are unchanged. Three cases build **and run to completion** on Derecho under **both @@ -455,7 +455,7 @@ don't rebuild downstream objects unless something actually moved. ## 11. Walk-through outline (suggested order for the meeting) -1. Live `ccpp_capgen_ng.py --help` (CLI shape). +1. Live `ccpp_capgen.py --help` (CLI shape). 2. Show one scheme's `.meta` + its generated group-cap fragment. 3. Run the generator twice — note the `Unchanged: …` messages on the second pass (write-if-changed in action). diff --git a/doc/briefing_pm.md b/doc/briefing_pm.md index 2c2ce066..8a25cfd9 100644 --- a/doc/briefing_pm.md +++ b/doc/briefing_pm.md @@ -1,9 +1,9 @@ -# capgen-ng — Briefing for Project Management +# capgen — Briefing for Project Management *Companion to `doc/briefing.md` (the developer walk-through) and `doc/redesign_analysis.md` (the deep-dive technical comparison of prebuild and capgen). This document targets project leadership and -program managers; it summarises the case for `capgen-ng` in terms of +program managers; it summarises the case for `capgen` in terms of product risk, schedule, and cross-organization impact rather than implementation detail.* @@ -25,7 +25,7 @@ same problem differently: **do not support multi-instance hosts** at all. Neither generator can be the basis for a single shared toolchain. -**`capgen-ng`** is a third generator, started in early May 2026, +**`capgen`** is a third generator, started in early May 2026, designed to do everything both other generators do, in code small enough for a few people to own, with the architectural choices that make it work at UFS/NEPTUNE scale and beyond. The redesign is @@ -60,7 +60,7 @@ limits that make it impractical for UFS, NEPTUNE, or multi-instance hosts, and the implementation is concentrated enough that few people can extend it safely (if at all - primary developer gone). -**`capgen-ng`** (new, 2026-05). Procedural Python (~17.8k lines +**`capgen`** (new, 2026-05). Procedural Python (~17.8k lines including inline comments and the three transient shim modules; flat data classes); reads the same metadata format; passes arguments like prebuild; supports the features capgen pioneered @@ -123,7 +123,7 @@ when it does compile produces unmaintainably large source files. **This is one technical reason capgen cannot drive UFS today**, independent of any other concern. -`capgen-ng` reverts to prebuild's DDT-argument convention. Host +`capgen` reverts to prebuild's DDT-argument convention. Host authors pass their physics DDTs by reference (one or a few arguments per scheme call); component access happens **at the scheme call level**. This works at every scale we've measured. @@ -132,13 +132,13 @@ This works at every scale we've measured. CAM-SIMA runs one host per executable, so capgen generates a single module-level `ccpp_model_constituents_obj`. The constituent -mechanism — the central feature capgen-ng inherited from capgen — +mechanism — the central feature capgen inherited from capgen — references that global directly. Re-targeting capgen to multi-instance is not a configuration toggle; it requires re-emitting the constituent module per-instance throughout the generator, plus refactoring the framework setters. -`capgen-ng` was multi-instance **from day one**: every constituent +`capgen` was multi-instance **from day one**: every constituent entry point takes an `instance_number` argument; the property storage, the state machine, the dynamic-constituent buffers are all per-instance. As of 2026-05-18, the per-suite dynamic-constituents @@ -160,7 +160,7 @@ to know the generator semantics. This makes scheme code harder to read, harder to port between hosts, and harder to debug when registrations collide. -`capgen-ng` keeps only the first two (explicit) paths. Auto-clone +`capgen` keeps only the first two (explicit) paths. Auto-clone is deliberately gone from the default behaviour — see `doc/constituents_overhaul.md` §2.3. For legacy hosts that already ship metadata in the original-capgen shape (production CAM-SIMA's @@ -181,7 +181,7 @@ host-specific strings into their own metadata. Porting a scheme between hosts requires either editing the scheme or maintaining a fork. -`capgen-ng` is moving `diagnostic_name` (and a handful of other +`capgen` is moving `diagnostic_name` (and a handful of other host-configuration properties) to a host-side override mechanism; schemes carry physics-portable defaults only. The reform is documented in `doc/constituents_overhaul.md`; the decision is on the @@ -197,11 +197,11 @@ resolver to handle multi-instance dimensions, scalar-index substitution, or constituent host-wins semantics required undoing parts of the synthetic scope. -`capgen-ng`'s resolver is flat: each scheme arg is classified into +`capgen`'s resolver is flat: each scheme arg is classified into exactly one source (control / host / suite / constituent), recorded on a small data class (`ResolvedArg`), and used directly by the emitter. No synthetic dictionary. **This design inherits from -`prebuild` and is the primary reason `capgen-ng` is comparable +`prebuild` and is the primary reason `capgen` is comparable in performance to `prebuild`. ### 3.6 Code volume and team coverage @@ -210,11 +210,11 @@ capgen is roughly an order of magnitude larger than prebuild, with a deeply layered class hierarchy. This is not a moral failing — it reflects the feature set — but the practical consequence is that the maintenance burden falls on a small subset of the framework -team. capgen-ng is comparable to prebuild in *shape* (procedural +team. capgen is comparable to prebuild in *shape* (procedural Python with small data classes — no deep class hierarchy), and the generator itself sits at ~17.8k lines. The "who can fix this" pool is closer to "anyone with -framework context". capgen-ng comes with ~1.4k docstring + unit +framework context". capgen comes with ~1.4k docstring + unit tests (~18k lines of test code), plus an end-to-end test suite of 12 fixtures that covers all of prebuild's and capgen's existing end-to-end tests and adds new ones for multi-instance + constituents @@ -222,19 +222,19 @@ end-to-end tests and adds new ones for multi-instance + constituents (`advection_auto_clone`), constituent-count dimensions (`constituents_dim`), and suite-owned allocatable interstitials (`suite_allocate`). Including these tests and the rich inline -comments puts capgen-ng's full tree on the same order of magnitude as +comments puts capgen's full tree on the same order of magnitude as capgen — about half of which is test coverage and human-readable prose, not load-bearing logic. --- -## 4. What `capgen-ng` does better than capgen — at any scale +## 4. What `capgen` does better than capgen — at any scale For audiences who already accept the multi-instance and UFS-scale arguments, the day-to-day quality-of-life improvements that apply even to CAM-SIMA-shape problems: -| Topic | capgen | capgen-ng | +| Topic | capgen | capgen | |---|---|---| | Scheme call argument shape | Flat fields | DDT references | | Variable resolution | Scope-chain promotion via synthetic dict | Flat 4-source classification on `ResolvedArg` | @@ -248,18 +248,18 @@ even to CAM-SIMA-shape problems: --- -## 5. Additional features of `capgen-ng` compared to `capgen` +## 5. Additional features of `capgen` compared to `capgen` -Features that exist only in capgen-ng (some exist in prebuild): +Features that exist only in capgen (some exist in prebuild): | Capability | Why it matters | |---|---| | **Multi-instance host support** (per-instance state machine, per-instance constituent objects, per-instance dynamic-constituents buffers as of 2026-05-18) | Required by NEPTUNE (prebuild has basic solution) | -| **Registered scalar-index dimensions** | When metadata says a variable is dimensioned by `number_of_threads` or `number_of_instances`, capgen-ng injects the right per-call subscript automatically; the host's OpenMP-thread-private DDT layout works unchanged | +| **Registered scalar-index dimensions** | When metadata says a variable is dimensioned by `number_of_threads` or `number_of_instances`, capgen injects the right per-call subscript automatically; the host's OpenMP-thread-private DDT layout works unchanged | | **Subcycle loop-counter automation** | Schemes inside a `` element can access `ccpp_loop_counter` / `ccpp_loop_extent` directly; the generator emits the Fortran `do` loop and binds the locals | | **`--legacy-mode` migration shim** | One CLI flag enables silent rewrite of two known-good deprecated standard names (`horizontal_loop_extent` → `horizontal_dimension`, `number_of_openmp_threads` → `number_of_threads`) with a loud warning — buys time for host metadata to migrate | -| **`--gfs-dim-aliases` migration shim** (2026-05-21) | One CLI flag treats GFS-physics names (`adjusted_vertical_layer_dimension_for_radiation`, `vertical_composition_dimension`) as equivalent to `vertical_layer_dimension` in the dim-identity check only — variables remain distinct everywhere else. Resolver-only; clean grep-revert. Required for CCPP-SCM v17p8 to build under capgen-ng. | -| **`--legacy-auto-clone-constituents` migration shim** (2026-05-21) | One CLI flag reinstates original ccpp-capgen's auto-clone-static-constituent registration path for the ~16 production-CAM-SIMA schemes that depend on it. Single-instance only (predates multi-instance); fails fast if a multi-instance host is supplied. This is the no-decision-needed bridge that lets capgen-ng accept CAM-SIMA's atmospheric_physics metadata before any constituent-overhaul work lands. | +| **`--gfs-dim-aliases` migration shim** (2026-05-21) | One CLI flag treats GFS-physics names (`adjusted_vertical_layer_dimension_for_radiation`, `vertical_composition_dimension`) as equivalent to `vertical_layer_dimension` in the dim-identity check only — variables remain distinct everywhere else. Resolver-only; clean grep-revert. Required for CCPP-SCM v17p8 to build under capgen. | +| **`--legacy-auto-clone-constituents` migration shim** (2026-05-21) | One CLI flag reinstates original ccpp-capgen's auto-clone-static-constituent registration path for the ~16 production-CAM-SIMA schemes that depend on it. Single-instance only (predates multi-instance); fails fast if a multi-instance host is supplied. This is the no-decision-needed bridge that lets capgen accept CAM-SIMA's atmospheric_physics metadata before any constituent-overhaul work lands. | | **`--no-host-introspection` flag** | The five runtime introspection routines (`ccpp_physics_suite_list`, etc.) emit large `select case` blocks at SCM scale; this flag stubs the bodies, dropping the generated static API from ~33,000 lines to ~800 for the SCM build (the introspection routines were making `-O1` compilation effectively hang) | | **Consistent handling of external types** (MPI f08 communicator, ESMF clock) | Tabled in capgen because of the complexity of the solution | @@ -270,17 +270,17 @@ Features that exist only in capgen-ng (some exist in prebuild): - **Unit tests**: 1516 passing. No known failures. - **End-to-end tests**: 12 passing — `advection`, `advection_auto_clone` (CAM-SIMA advection_test port exercising the - auto-clone shim), `capgen_ng`, `chunked_data`, `constituents_dim`, + auto-clone shim), `capgen`, `chunked_data`, `constituents_dim`, `ddthost`, `instances`, `instances_advection` (multi-instance + constituents), `nested_suite`, `opt_arg`, `suite_allocate`, `var_compat`. The two newest (`constituents_dim`, `suite_allocate`) were added while hardening the CAM-SIMA HPC build. -- **Code size**: ~17.8k lines of Python under `capgen-ng/` including +- **Code size**: ~17.8k lines of Python under `capgen/` including inline comments and the three transient shim modules; ~18k lines of unit/doctest under `unit-tests/`. Still procedural; still flat data classes; still well below capgen. - **CCPP-SCM**: actively driving development. Each build / runtime - issue surfaced this month landed as a fix in capgen-ng rather than + issue surfaced this month landed as a fix in capgen rather than a host-side workaround. All available suites in CCPP-SCM now build and run end-to-end via `--legacy-mode` + `--gfs-dim-aliases`. - **Three transient migration shims in place** (see §5). Each is @@ -290,7 +290,7 @@ Features that exist only in capgen-ng (some exist in prebuild): auto-clone path behind `--legacy-auto-clone-constituents`. This is the no-decision-needed bridge for CAM-SIMA — the ~16 schemes that declare `advected = True` in `_run` arg-tables and rely on the - framework to register the constituent will now work under capgen-ng + framework to register the constituent will now work under capgen without metadata edits. - **Multi-instance + constituents fix landed 2026-05-18**. The new combined end-to-end test surfaced a latent shared-buffer mutation @@ -304,10 +304,10 @@ Features that exist only in capgen-ng (some exist in prebuild): - **UFS Weather Model**: not yet attempted; SCM is the proving ground first. Expecting updates due to the "fast physics" called directly from the FV3 dynamical core as separate group. -- **CAM-SIMA**: **re-connected (2026-06-03 → 06-05).** capgen-ng now +- **CAM-SIMA**: **re-connected (2026-06-03 → 06-05).** capgen now drives the production CAM-SIMA build on the Derecho supercomputer through a small compatibility layer that lets CAM-SIMA's existing - build scripts call capgen-ng without being rewritten. Three + build scripts call capgen without being rewritten. Three configurations build **and run to completion under both the Intel and GNU compilers**, with bit-comparable results: `kessler`, `rrtmgp`, and `se_cslam`/CSLAM — the last being the full CAM7 physics suite @@ -344,7 +344,7 @@ the table: the CAM-SIMA atmospheric_physics tree, and CAM-SIMA itself. These are open questions for the framework-team meeting, not -capgen-ng decisions. capgen-ng is structured so all three +capgen decisions. capgen is structured so all three proposals are implementable on top of it. --- @@ -353,12 +353,12 @@ proposals are implementable on top of it. | Risk | Status | Mitigation | |---|---|---| -| capgen-ng diverges from capgen feature set | LOW | Cross-checked by `doc/redesign_analysis.md`; the feature comparison table in §4 / §5 is exhaustive | +| capgen diverges from capgen feature set | LOW | Cross-checked by `doc/redesign_analysis.md`; the feature comparison table in §4 / §5 is exhaustive | | Host metadata break for UFS / NEPTUNE / CAM-SIMA | LOW | Three transient shims (`--legacy-mode`, `--gfs-dim-aliases`, `--legacy-auto-clone-constituents`) together cover the known-incompatible standard-name pair, the GFS radiation/composition vertical-dim spellings, and original capgen's auto-clone registration path. Remaining required changes (e.g., `_finalize` → `_final`) are mechanical and listed in `doc/migration.md` §3 | -| Constituent overhaul stalls | LOW | Proposal A unblocks the immediate bug; capgen-ng works with the current framework today; `--legacy-auto-clone-constituents` lets CAM-SIMA's atmospheric_physics build without an overhaul decision; the overhaul is a separate decision track | -| Bus-factor on capgen-ng itself | MEDIUM | Procedural code style + flat data classes + 1426-test safety net; significantly lower than capgen's bus factor | -| Two host call-shape conventions (prebuild-style vs capgen-style) coexist forever | LOW | capgen-ng emits one shape; downstream host conversions are tracked in `doc/migration.md` | -| Regression discovered during NEPTUNE / UFS testing | EXPECTED | SCM proving ground catches most; remaining issues become capgen-ng tickets, not host-side patches | +| Constituent overhaul stalls | LOW | Proposal A unblocks the immediate bug; capgen works with the current framework today; `--legacy-auto-clone-constituents` lets CAM-SIMA's atmospheric_physics build without an overhaul decision; the overhaul is a separate decision track | +| Bus-factor on capgen itself | MEDIUM | Procedural code style + flat data classes + 1426-test safety net; significantly lower than capgen's bus factor | +| Two host call-shape conventions (prebuild-style vs capgen-style) coexist forever | LOW | capgen emits one shape; downstream host conversions are tracked in `doc/migration.md` | +| Regression discovered during NEPTUNE / UFS testing | EXPECTED | SCM proving ground catches most; remaining issues become capgen tickets, not host-side patches | | ccpp-prebuild end-of-life requires a sunset plan | OPEN | Not yet scoped; both generators currently coexist in the framework repo | --- @@ -372,16 +372,16 @@ Three points worth raising explicitly: The flat-field convention is load-bearing throughout capgen's variable-matching, resolution, and emission code. Once that change is made, the resulting generator looks substantially - like capgen-ng anyway. + like capgen anyway. 2. **The features capgen pioneered (constituents, suite-owned variables, introspection) are kept and improved — not - discarded.** capgen-ng is genuinely the successor, not a + discarded.** capgen is genuinely the successor, not a parallel project. The contributions made on the capgen side are - what made the capgen-ng feature set possible. A significant + what made the capgen feature set possible. A significant portion of capgen's code, in particular metadata parsing, Fortran-metadata validation, and constituents, were imported - into capgen-ng. -3. **The team owning capgen-ng can be larger than the team owning + into capgen. +3. **The team owning capgen can be larger than the team owning capgen.** This is the most important practical point for long-term program health. A framework that three organizations can maintain is more resilient than a framework that one @@ -398,7 +398,7 @@ Three points worth raising explicitly: - `doc/migration.md` — host-author migration guide. - `doc/constituents_overhaul.md` — the constituent-reform discussion document. -- `doc/capgen_compat_layer.md` — short brief on the CAM-SIMA ↔ capgen-ng +- `doc/capgen_compat_layer.md` — short brief on the CAM-SIMA ↔ capgen compatibility layer (for the original ccpp-capgen author). - `end-to-end-tests/` — the working examples (`instances_advection` is the newest, exercises everything end-to-end). diff --git a/doc/cam4_fwaut_constituent_order.md b/doc/cam4_fwaut_constituent_order.md index 2cd72f1b..0e4f042e 100644 --- a/doc/cam4_fwaut_constituent_order.md +++ b/doc/cam4_fwaut_constituent_order.md @@ -18,18 +18,18 @@ The physics source, `suite_cam4.xml`, and `src/data/registry.xml` are **byte-identical** between the two builds. The difference is purely in the generated CCPP caps. We have traced it to a single cause and **proven** it: -> **capgen-ng registers the advected constituents in a different order than the +> **capgen registers the advected constituents in a different order than the > original capgen.** Specifically, `cloud_liquid` and `cloud_ice` > are swapped. This changes the floating-point summation order in the energy/water > thermodynamic diagnostics, which the energy fixer then spreads across all columns > as a tiny, pervasive heating — the source of the b4b difference. -A one-off patch that forces capgen-ng's advected water species into the +A one-off patch that forces capgen's advected water species into the original-capgen order makes **QPC4 bit-for-bit identical** to the baseline. ## The difference (runtime constituent list, `debug_output = 2`) -| index | original capgen (baseline) | capgen-ng | +| index | original capgen (baseline) | capgen | |------:|----------------------------|-----------| | 1 | **cloud_liquid** (advected) | **cloud_ice** (advected) | | 2 | **cloud_ice** (advected) | **cloud_liquid** (advected) | @@ -45,7 +45,7 @@ both — the only advected difference is the **cloud_liquid ↔ cloud_ice swap** 1. `air_composition` builds `thermodynamic_active_species_idx` by walking the advected constituents in **constituent-index order**. 2. `get_hydrostatic_energy` (`cam_thermo`) sums the water species in that order. - Baseline sums `cloud_liquid + cloud_ice + water_vapor`; capgen-ng sums + Baseline sums `cloud_liquid + cloud_ice + water_vapor`; capgen sums `cloud_ice + cloud_liquid + water_vapor`. Same values, **different FP order**. 3. The resulting machine-eps difference in total energy/water is picked up by the global energy fixer (`check_energy_fix`), which redistributes it as a uniform @@ -61,7 +61,7 @@ physical ordering. ## Proof -Forcing capgen-ng's advected water species into the baseline order +Forcing capgen's advected water species into the baseline order `[cloud_liquid = 1, cloud_ice = 2, water_vapor = 3]` (a flag-guarded one-off patch in the framework's `ccp_model_const_table_lock`) makes QPC4 reproduce the ccpp-prebuild baseline **bit-for-bit** (cprnc: all fields identical). This @@ -73,14 +73,14 @@ below for the full patch. Both builds register the same constituents with identical properties; the ordering is not physically meaningful, and the resulting solutions are roundoff-equivalent and both physically correct. The b4b failure reflects only -that capgen-ng's (arbitrary) order differs from the (equally arbitrary) order +that capgen's (arbitrary) order differs from the (equally arbitrary) order the capgen baseline happened to produce. ## Decision requested To resolve QPC4 (and any other case sensitive to constituent order), we propose: -1. Give capgen-ng a **deterministic, documented** constituent-registration order +1. Give capgen a **deterministic, documented** constituent-registration order (e.g. water vapor first, with a clear rule for how constituents land in the array) — replacing today's hash-bucket order. 2. Adopt the new documented order and **re-baseline** the affected CAM-SIMA cases once. @@ -90,10 +90,10 @@ The temporary proof patch will be removed once the path is agreed. ## Artifacts - **Patch:** Stored as `ccpp_constituent_prop_mod.F90.patch` in the top-level -directory of the `feature/capgen-ng` ccpp-framework branch): +directory of the `feature/capgen` ccpp-framework branch): ``` ---- capgen-ng/src/ccpp_constituent_prop_mod.F90 -+++ capgen-ng/src/ccpp_constituent_prop_mod.F90 +--- capgen/src/ccpp_constituent_prop_mod.F90 ++++ capgen/src/ccpp_constituent_prop_mod.F90 @@ -1392,6 +1392,17 @@ type(ccpp_constituent_properties_t), pointer :: cprop character(len=dimname_len) :: dimname @@ -142,39 +142,39 @@ directory of the `feature/capgen-ng` ccpp-framework branch): ``` - **Run directories (Derecho) Intel:** Because the SIMA baselines change continuously, - - Baseline (original capgen, https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng-reference): + - Baseline (original capgen, https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-reference): - `/glade/derecho/scratch/heinzell/aux_sima_intel_20260614203021/` - - capgen-ng differences to be evaluated against this baseline, because the official baseline changes frequently - - Both the capgen baseline and the capgen-ng test fail for this test: + - capgen differences to be evaluated against this baseline, because the official baseline changes frequently + - Both the capgen baseline and the capgen test fail for this test: ``` SMS_Ln9.ne3pg3_ne3pg3_mg37.FKESSLER.derecho_intel.cam-outfrq_se_cslam_multitape (Overall: NLFAIL) details: FAIL SMS_Ln9.ne3pg3_ne3pg3_mg37.FKESSLER.derecho_intel.cam-outfrq_se_cslam_multitape NLCOMP ``` - - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng), unpatched (shows the FWAUT diff): + - capgen (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen), unpatched (shows the FWAUT diff): - `/glade/derecho/scratch/heinzell/aux_sima_intel_20260614202951/` with the following `mpasa120_mpasa120.QPC4` test dirs: - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951.ORIGINAL_NO_PATCH` - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951.ORIGINAL_NO_PATCH` - - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng) + reorder patch (**b4b**): + - capgen (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen) + reorder patch (**b4b**): - `/glade/derecho/scratch/heinzell/aux_sima_intel_20260614202951/` with the following `mpasa120_mpasa120.QPC4` test dirs: - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951` - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_intel.cam-outfrq_analy_ic_cam4.GC.aux_sima_intel_20260614202951` - **Run directories (Derecho) GNU:** Because the SIMA baselines change continuously, - - Baseline (original capgen, https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng-reference): + - Baseline (original capgen, https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-reference): - `/glade/derecho/scratch/heinzell/aux_sima_gnu_20260611123848/` - - capgen-ng differences to be evaluated against this baseline, because the official baseline changes frequently - - Both the capgen baseline and the capgen-ng test fail for this test: + - capgen differences to be evaluated against this baseline, because the official baseline changes frequently + - Both the capgen baseline and the capgen test fail for this test: ``` SMS_Ln2.ne3pg3_ne3pg3_mg37.FPHYStest.derecho_gnu.cam-outfrq_hb_vdiff_derecho (Overall: FAIL) details: FAIL SMS_Ln2.ne3pg3_ne3pg3_mg37.FPHYStest.derecho_gnu.cam-outfrq_hb_vdiff_derecho RUN time=13 SMS_Ln9.ne3pg3_ne3pg3_mg37.FADIAB.derecho_gnu.cam-outfrq_se_cslam (Overall: FAIL) details: FAIL SMS_Ln9.ne3pg3_ne3pg3_mg37.FADIAB.derecho_gnu.cam-outfrq_se_cslam RUN time=13 ``` - - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng), unpatched (shows the FWAUT diff): + - capgen (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen), unpatched (shows the FWAUT diff): - `/glade/derecho/scratch/heinzell/aux_sima_gnu_20260611123837/` with the following `mpasa120_mpasa120.QPC4` test dirs: - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837.ORIGINAL_NO_PATCH` - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837.ORIGINAL_NO_PATCH` - - capgen-ng (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen-ng) + reorder patch (**b4b**): + - capgen (https://github.com/climbfuji/CAM-SIMA/tree/feature/capgen) + reorder patch (**b4b**): - `/glade/derecho/scratch/heinzell/aux_sima_gnu_20260611123837/` with the following `mpasa120_mpasa120.QPC4` test dirs: - `SMS_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837` - `SMS_D_Ln9.mpasa120_mpasa120.QPC4.derecho_gnu.cam-outfrq_analy_ic_cam4.GC.aux_sima_gnu_20260611123837` diff --git a/doc/capgen-ng_review_plan.xlsx b/doc/capgen-ng_review_plan.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..6e19ab99fc138602df74fe3b5a2d1cfb589f7e72 GIT binary patch literal 11801 zcmeHt1$$h()%7@znVFfH*%>o4Gseu!F*C=^%*@PA%n&nEOffUl>wDXNw@v%~f>!g) z(K9nzd(M%xw6(XEycFn1Q~)>t5&!@Y0Sr$w&2&KkfY^@!015!|gO-r3jgzsBlb*7> zow1`1y_>ZaVcy3NlsSM8@9qCr{)>B{B4JdvpAk{yPW)MDhfZ>}jvtck#D5T(S^?Pe zEq+f(4hO^c!DF&+ z46HZ@O^vf0R2+ULEIqv_lXO6eeRIDetMHb9WCgYv8amK-J9BwZ6=`Fj6}C*XgllM} zcT-k%9f@ULM_)(Gl13k8SK|W_om5S^R!P z{?m&9dCQ0f^h#!%$uHmd(DJ26&-)11p2)UPaKCNf7`0`^#2U$B5?j-577k_!%W-i~ z`+FA;4YzRi$enz~^|C3Ng4p>ea{q>Ui*(R-)gy>F+(ShACFEpt=zz!b_{XkR!0AQO zfe-U5H~{eW1`3e>7g{!|Fp^xq&o$|H*1^2fQqRHI%8`NokL&;F_+QMye|q$aI2pNq zM%dsp@#m1CyScSkL_sN60r7SsWp7`JRfNW2af8u6rTS?O$Y`t^-dBOl2`Zgb2G=xmp$B zPp6^{IU1TSRe23r5#2mst0ls<~TnVRlqr7<0$z}{G60;)TGf#>Q zlI7g^Xg083jApuj)TPLlilMBj^df^@%A2;seF+^iT}Z5=EP zZEY?8@LL5c3UVuqh~C)^Z?PJQw+ZC3=9mgf9}DKFYvH2cbg5OUw%O66Gne4$1fTCR zQ8X#nKm#ai9lL&Axw>v}1#CN0b!ZjAsWFSR;1I`1lsV8RQ$4;|9{UJn)e7$kQw}RS z4Z?Ecdp|R(Q+;7kI~)K7m@6u)Ymo`)oApW>wv`zo^THHL8TQVO_9&-)T}6#tQv9fZ zIUpd>B1|Wkrx%cm4kN2kWTwMxu7x=Mv6g=WwSyU_A+8I&iuAFl%BySyns86s&U3++ zxVm82h$@lX5yYU{enZQ$<+Ga|q)@{Wl(wf&Zf)~Q8{ap5*cU3jzl1qvqoMpUqXW|S(j_&Uziqm~U4Haxm^O}T0terefRfm0&# z5X!ejg$cS*B`3%t&Du;b9kpUMjLnlE1Jr!|mc*+da8L^$V%nKFb(f+7u+mS8KGhZ2 z$Mu4Kp`F>sa^q|P>mX19oQ%k6v8}CyP3IPLimaJck-wlt!YC7JX9&{ddnZ~e7V7h{ zve3(nX0#KP7NV3S%SZI?6O^UMzqEct$OOhyEuDqEAyVhk%Yp0^?QV2+kD&zS#z$c? zRNQWUiOUocpdY47fDi^$q@8B+zuMc9m&%?W1Mvw=+yYtnpgqN21(NU{i(hm%_{Il1 z&79C%(Mqi^)ZHO6^FS%DvW{>xY_pd3(#i+i#~rl_C=W%x{3>)gq1hh&fVusvhfa`z(}Zf- zAfYjG!0y3j(EybS6tzcTgHl2cF9EB#%x1sP-p%s7S2#Mfe!=EAID<&F*hCn(BIFHK1}@<$I8t1})j+wrS$P)Ng*N_>AYFcjqD!=v&O z82B_tlvZHe5N#GJ=iWUf8CFPs5wY-f>vS-y#RQ`>vxVaiypi`6b+lo}_c2iExYEC& zxtTDH%4GMgRcPfFJlo%$VCo7BN`e>89WeE4mWbxQSL7D*vg6rA#(l!FH~$fK7q?sr zzl`mZSFjUd$dDL;4^^DwZeD7;%Gs8a;L`|I_sS*h==GV&&hX&e#`{SB`+^#kE{wvSw(m*yM{Ne$g|31GxY;;hfWjv!tcu=$NZv zpB(R|>SyHCMqvzDt391*t*OMGmVMlgWbV4jEtzpX5bEFE-+!6#94l|5M4-`&BJWBj zsB=)lJym@r=%&asVCaIHNSb;x~HPK6Jxa3Iu76aj(OAh#pU8^Ew^HN1@GKj zRUyrN673o6oX<+`vp0t6GKPlA{B9GL-!9W5?E%B!sC}Aq66u?Oi85=|<)9$M`JD1O zQ}S#>Jz57g44;|RJl4#NiViPodg9wz;OiHE*Frt=YKD_VpVztFB40>LRoz5ewO0)1 zuYtLIU2UvT*QQ_j=H^YSC2YHQ+j%V=IKwVKWo9=UKj?t^1ZR`$U#8$Yob9e4&r+cLXkJC0rinA|&p6Wi~L=mTmW`zJEz^9rwQZoPz^M<}3{nlZ!p z_FC(5K*?yvph4ExemFBS=1Lo%s0YD~f^(z5iJ0;NBIMfvSCRO!NzT`~k)!(_ATRPdzEK?gU{_VSdaUSIcDoC) zSutwyqZcFvBBG&|Ve<5bbVz;?p=88PNF2yWt)ariMRvx zJyHT5@q6rvO{|C!LEPtv_TFox)Qb6foNWZelhatR?I_4l#SnGo^~%p}Hr8pILz|SA z6;iSnj3nPoirq4hkW*6E;E023m3^_}e&>HB*`1di%X^)X5i!B?n%O2H=817yiB6!=RM$16Lv!%$ZfvG~Y zU0*g|oIuVFcaY{Rhjo2dy9Bm*W1RCu2qJi`u4}@NOW-IbgacFSlqBB_Mi0r&KkZtf zY1hDE;9`6ES2ML-;`LR4F9haN>(*_4XI+&_Wn|xvU0^c!K$3p4tCm?;`2+q$6U9 z{VKvQ(dbHwk@l%xGMMcpT;*&AX1USA1rWcOm<<-3Na+bSl#C8>B}{rw56L8?6wqto z>&bcM3yEK*H`)tPx33bN@WVH~r6UjUsG4-eDx9@Z2B7LG&9q|Z-0#AMg?)~a!fBDg zUb0A)q~qac^Y$1yHkE3Hh;NpZD^{EMVJIYjq2ZqGGq7QIdq{BmxZejj3a5dv%zO2c zN8(*m>D539!9}@lqy&39W-+>0%RP9kwbF>Qd}ViH5FcqQ&56Mb0qV7aYE_8{=1<%4 zePP}Du}{OX^woh*-=We6R63khLi5y5#yvMZzRD}|LD@b^vj3%Ew`x(@Dx$NQ0X-vs z3ie((Pb!Li`yG~BejG&ji9i1<6wfqEmxJqRxG>2^y7gh&pTQdW(Pmbi0)=N@{Bze z&#=;nhM*=!l-+9Hv1{$Ug}RS)CzxWV#A|46K5+js=$P=0tD6OJ#?I$Qp~p7H`TLmW zyfNLNH)yHO>)ufZj^zo;dD#`t1PqD?BZEN$MvQ^evPrt3{p%S z*Klkjj~sXF1%oyy@dIOFCt-@ZQ)M(b=45CQJa|TXPNl_ZQSaYIf4)?@O#Wcq>oPoV z6mriX*cBnV{qlUl3-4q9R(I399fP2?3HNxNOBHXpu#ERvC2TM4On+R%n;`Tg?Dr=z#lR2tmy8xpC|2t+dNJLMt%?>#a&jQDKn^j5Uz_AV~x=2&GbH-_V6v}(+I?x z6sJi+#L`~zdh0BA6CxuegS<6$0B!acF7^6sED~H3xoeyi54#T6Dqia^hEy&5fdFJJ zp|V1iIw}!$hL#^o*6;XaD>nG!4coB3qh!bECX{J)D^Oc)u_`ZXa+k5cq5jjxxV45C9)`_a|TYhjLnl8M?eKSL!Jkx*o6Z z&hR}qwLTz>oC9UTgqw$umiMWCnZRyK%7Xv&;(3HBnt;y$SR))%N#br+LYV-y(_04G zX>#1|eHRAr@H7yRnN(-{A4KE0dvqo=(qXDWY|M1YVD1Jrj!o$Yj8}mJA;5LK46dq^ zXvdPYnH(pp4T(^aJ^cREKBnPN!ACp-q@Po#_=V9ruI7|2J3>LH3bO;VmT^ElPCwR- z1|g;M(Qm~pEz7mzr#iEHW5qY@TuxtO5mVd)&rtSKF1Yb&G9JZm?MFtyyTep|qFm|| z_c2$|pt*6AN9NlI7hIpT((!M^&DiRfsOdJJ)kz!@2j!6vjHjnuF*~Hka6`#DR*$ht zjVvPUKnkMff2#62*Eux_^#oO$SYZveUUN0Z8IFB zRNi*|?&iU5-8;1-6_$xY=7x2ur|#B#!MQ?$coYFK+1~b4bqfCW(AG|yJgI5D4~Cw$ zEcP61E9nyu!uZD0yL|ek(FE74kd{tn|(qC&T`ecF{pZxn|@rHtwSJcVbS;xwWKdPGq+mXCk;Ej1GKrf2>fKtuR z6JfrxQ}skwSq|HY3#z!dn15eYz~onud{9TSj};a(>lrX42zYWX@dqL$+VL2?r)_=9 zS*@+FM(8F)rV|-zU?-|RCsDLeHgk&=&ieV|wF%Z1*Bg7f@9-zt6LYvy;P?gl3Sb~n7&O&Idn zJSgVJh-uA6fwtS`MUc=HGn&vognlwMwaNMhNbhx-#+kc4EQD3Ze zZk@SF!H&B%3O~Fx;~v5sOwnr`;p4sN{bF~`u6+@c8h2~)2BLk@n_%EW>buiC!KTIW z_MVOS_rMb`n_sX26aYwr2LRyy=9V0t+^viq|A;on)U<3fI8nXzO5Z>pGVP#Z4L&NT z!YCG=S`-=;c6s!Ssk5;&XC;`=y>$%+V}-^w#9KAVh(h1??q9jm10g8R+S4>#g`1Sm zQQa}<>mV6~B_7yC-d6W+VKplII=)HU#pQvTPdy#45lEE7N3I@2DxmY9vy1Q)rnPQ2 ze2<=O=|qIvIxj-58-OlyIBJL@ZLw-p9LL>oCMh0B$j3L(#n|_qRrg-!a3MMbr)kTL zEDPurRMi>dvucfX>-V+T2WzR}W{B&W8{Q@tebmoRLsx9bQ^fE+3N6b6a~J1uF@(S% zK_={m9frYM=r;AF&OD$K= zN}Pqg7GyTpQ}aNH>Hyvw>|`=H|O!Th$rm zt9`0VUbRt7K=9E#nu3F%yfOocV|HOamt6VaPwOWwFMNqhKF`?oiB%seq~HWA)tOJD zGx2`&Ve0|9QomuAwn=KcyO*2+VP1^|l*ORv#Q{Eof?uunwUhZ5vu1n5W}?8A7vA^k zan@`)jD>QFRO|&of2%;++_WguP0A5uQED)A@&lLRY)RI0ACd%rlx#Ax?^?Y~GY2aH zs|$hDkYEfV77$=ZZo0v4a)S9BSht)?{pUn{y+BLnIrD&Gb$5O{eK0UP zBe2W-rl9m-JSTzKLYox72~MpPTI~IWNE>$TG)K8h`&PQD^~gdm;&U+R&0gA?{>eD}QVjc}e$v{a zgSJk3!fPPM_^$>Sr>r<-x5?R)1LN?4i|m4eCvITnWBNLjF`*)BG~eOrnTQ9%&~^+R ztEHQ!r-ahAb!SVs+R8YU!L^5f^wq0#qzkaKF<>lhd+V0)3$WK0PlCe$*O@E9r zE%0PGE=-;jHUP-4O zwzw&Q;i8UeFV0MG7*mhtZ+*G1@o0pXlVgU$rgLq>d6wfv~z! z@DQZsvqohje6Zf>x}s;`I~^s=xiVyYu;|lp-oTfuY}t&fz|O=cM#_%rjlo(e7!7C+ z=EjP16B8>cUhidIm&Qb5$Do9+*^8;It(6wkGHiK^w2}T(gldRc<_QMWy^L9?MfWW$r*b80Cq5Gvjz+kL zFh;NqpG@tSsvQ)%0s`w5kH(jr1YM)Ph!*tu;cqzkYnMKz*y)nmKAw|}?#tdzgY920 zMn)&`1-@o0X?Si%Kj$Uc2)jll78Is9YkTy9T5~?GG?53@Fq3qQ6&4QARCf4B*|uTF z9#xhgx1<@Qmj<3PUBC>^O&PbRoHc4jrq9h!$)_uul?hX=;cv+!m#R5prQ2C{P|K?g zslQs=P{pGvE|kpS+u9gGi2BrzjEaxvp}IH-e-2j6okO~AZ?I^BL@bjwvd~>pE^lN? z)J&YtP{nYEs&Xu99 zoi~8xrORv9diO>M>XZPy6+6lwiI5e!y4+!4#n9}{*@E?t>xf`cY2(Zk$rx^7)m;3@ zyDyqQwH`V!mNaeu$)awP6eC4nLytCqNp!w)WwTOcQAlaptsW*$B+z}KvlkEybOg7E!`y_Z1q$z}K_1fHuJ;!M)OMhZ9* zT@Y^YN)7CFS`CL0{-}qAvp)QIhm6r!sGO_oWzC&t+YTcoLq*ieZ&c=$M zGkX|?mQ1^LLbVd+oZ2;i(JU}T5+ZE_2jM$w9Jh)wl!f%m^JGb?O)5>XqKpzts^il- zp;u(oNV`zC;-cua#657(f3q`w<1;i-re`ue@3*XgUj^LYP;@nPvlm0!_I zs+@v5&k;^z(D%FeOGg!^z5HZQi#OZ{2f4-ww1 zFHb>M{^&9%k$PAfCL2{9EKTACL`PdXEZbcJuSWd(BEPK8tw==%WBcGfLq&nF_r#>J zSx@Q)h9kurH$MvcXoJLJD{Cor90AgSgHB!)!px+IRk{PB0wybh;M=J6389^%g-WLt zJ;`I?utQi{Kvw$$Ybx@cE`MpT6;S{uvgtuT64&lSAieQx%M%cnE;-QuY*Z{3D%w8> zPD(7953h@Vm`=Kt?2fc~ch?Rbl(xcnA-)D5Vy5Pn-J@=dMt}ds8Nt8HSKD848R0~a zyfU|^Hg+cTJ(-`tX1r!Rcp_;{W}N)hg?rJ)?cs2e+h}2Yn{lt=W+b$@oXC`|g!j$LxG`C`HtV&1sYMQgIc2w#0dbXFv*pD-L~)*>fBdt#k`g1^mIVl%@n3h)xB z;jIEYL=cy3P#4Fcm`oJver>VQ1lhCJWbd!D9;?KKTT>+GMdx<0OIAB+V+0BkIb!hu zbfsc&FIy(Zv_nN_s%_h$JeyP@)o9s_Ai;qUUKi0gds4dRuO*4MRi&0=>6p^E!e{-4 zEHhM!qQ!%K4KtT%>RN!>qtvpHo*Xf zZaxi4rx%-|w*iI3aQvC%u{EW3(Cn31$m;Q)?|A(?rF}=Ic+%R<3g;GPw9>2RoYupg z2=~fHNW0{WE547@vaZg##(!>6s{}@dLgzD)D2?^ZJww6HOr^se58eY@9{JvV_P=l1 z&A>IM<=?YR*6&+-r1t|xBiqmN4z_lV44-WsjQ`lUzbjS#cdq!}b&+v;QvHmm-#5Uz zg@!%TUwf%@PxuGVd;KH@YT zcr}@z;PF37rPz#Lup-{4A>4cXi1^Rb(7?{_f3xs?`u)9R#Esj$%V7e~z#id0oU!jV zKtcx?B9c-$m??m|OkX8wN(h_9ZQ3Ze%jro9;&L67HosuO^1+u@!F#G*rwoDXh0WEW zf_b1=9IZ(59(U1qh5BQ9!O^=_d_$7ZK8PWhB@YS>qf`7!$E%X}IogtH*r^3VbFXnS zoxI?aosANW(2ROOb1(1NRDB2Ks8kq)sNUz!PzoU?t}4%0y&3KlzZiVi!4$v7w&*Dh z&=76 z`_3`JLexXA!?il0KQR#DITL>oX`E-&f-lWRHb#7tc!NC%;}#|-tJlYl-qq%*J~8dY z25aTSfConr*Uca~rR6Cg7zamw3vgqjz{etAhZ^L!nqu(MB0Z*irOyL>gLjoCBzWE{ zv7P6e5?T$?()S9x^eSn>Bdu_JEH04)b`I}Uht{-D>?%CK=&RA{bv#GKL+d{^?#7$%1iyLfq#_?{sZ_&o&3& **Status: temporary draft for the developer walkthrough.** All `file → routine → line` > anchors were verified against the current tree; line numbers drift, so treat them as > “go here,” not gospel. Three running examples: a **simple** one > (`end-to-end-tests/instances/`) used to teach the whole pipeline, an **advanced** -> one (`end-to-end-tests/capgen_ng/`) for the resolver’s harder features, and a +> one (`end-to-end-tests/capgen/`) for the resolver’s harder features, and a > **constituents** one (`end-to-end-tests/advection/`) for the constituent subsystem. > Once reviewed, this folds into `doc/DevelopersGuide/`. @@ -13,7 +13,7 @@ ## 0. Orientation for prebuild/capgen developers If you come from **ccpp-prebuild**: there is no Python-templated giant cap and no -`ccpp_prebuild_config.py`. capgen-ng parses metadata and the SDF, **resolves every scheme +`ccpp_prebuild_config.py`. capgen parses metadata and the SDF, **resolves every scheme argument into an explicit Python object** that records *exactly* where the host data lives and what (if any) unit/kind/flip transform it needs, then emits Fortran from those objects. @@ -32,7 +32,7 @@ The single sentence to keep in mind: ## 1. The pipeline at a glance -Everything is orchestrated by `capgen()` in **`ccpp_capgen_ng.py:863`**. +Everything is orchestrated by `capgen()` in **`ccpp_capgen.py:863`**. ```mermaid flowchart TD @@ -147,10 +147,10 @@ flowchart TD - **Found in host** → `host_dict.get(std)` (`_resolve_single_bound`, `suite_resolver.py:429`). - **Not found, first use is `intent(out)`** → it’s an interstitial; **promote** it to a suite-owned variable: a `SuiteVar` (`:964`) is created and added to the running `suite_vars` - dict, so later schemes that read it bind via `source='suite'`. This is capgen-ng’s answer + dict, so later schemes that read it bind via `source='suite'`. This is capgen’s answer to prebuild’s “where do interstitials live” — they’re emitted into `ccpp__data.F90`. - **Not found, first use is `in`/`inout`** → hard error (nobody ever writes it). See the - “undefined intent(out)” discipline — capgen-ng refuses to silently read an unproduced var. + “undefined intent(out)” discipline — capgen refuses to silently read an unproduced var. The two dictionaries in play during resolution: @@ -272,7 +272,7 @@ That is the whole chain: **`.meta` → `host_dict`/scheme args → `ResolvedArg. `transform_case` → these emitted lines.** > Note the **scheme appears twice** in `instances/` (`unit_conv_scheme_1`, `_2`, `_1`). -> Each appearance is its own `ResolvedCall`; capgen-ng dedups *init/finalize* phases by +> Each appearance is its own `ResolvedCall`; capgen dedups *init/finalize* phases by > scheme name within a group, but **run** phases emit every appearance. --- @@ -301,7 +301,7 @@ scalar-index dimensions, control vars, a unit transform (case 3), and an optiona --- -## 7. Running example 2 (advanced) — `capgen_ng/` : what the resolver adds +## 7. Running example 2 (advanced) — `capgen/` : what the resolver adds Same pipeline; this case exercises the features `instances/` doesn’t. Read it for: @@ -356,7 +356,7 @@ Three questions prebuild developers always ask: - the arg’s **type** is `ccpp_constituent_properties_t` — the register-phase descriptor array (a separate flag, `is_constituent_arg`); **and** - constituent-ness is ultimately the **host’s** decision. A scheme that only *reads* a name - need not re-flag it — capgen-ng infers it from the set of names *some* scheme flags (“rule + need not re-flag it — capgen infers it from the set of names *some* scheme flags (“rule b”). If the host declares the name as an ordinary variable, that wins (`design_constituent_host_wins`). @@ -496,7 +496,7 @@ to §8.1–8.4. Use it only when the audience needs the multi-instance constitue ## 9. How to follow along live -- **Run it:** point `capgen()` / `ccpp_capgen_ng.py` at the example’s `.meta` + SDF and inspect +- **Run it:** point `capgen()` / `ccpp_capgen.py` at the example’s `.meta` + SDF and inspect the generated `ccpp_*_cap.F90`, `ccpp_*_data.F90`, and `datatable.xml`. - **Read the resolution:** `write_suite_meta` (`suite_data.py:481`) emits the resolved suite as a `.meta` — the cleanest dump of `SuiteResolution`. @@ -512,8 +512,8 @@ to §8.1–8.4. Use it only when the audience needs the multi-instance constitue | Concept | Routine | File:line | |---|---|---| -| Orchestrator | `capgen` | `ccpp_capgen_ng.py:863` | -| Load metadata | `_load_metadata_files` | `ccpp_capgen_ng.py:637` | +| Orchestrator | `capgen` | `ccpp_capgen.py:863` | +| Load metadata | `_load_metadata_files` | `ccpp_capgen.py:637` | | Parse `.meta` | `parse_metadata_file` | `metadata/metadata_table.py:1166` | | Parsed table / var | `MetadataTable` / `MetaVar` | `metadata/metadata_table.py:940` / `:414` | | Host dict entry | `HostVarEntry` | `metadata/variable_resolver.py:244` | diff --git a/doc/constituents.md b/doc/constituents.md index 6a8a6507..775d653b 100644 --- a/doc/constituents.md +++ b/doc/constituents.md @@ -1,9 +1,9 @@ -# CCPP capgen-ng — Constituents Reference +# CCPP capgen — Constituents Reference *Last revised: 2026-05-13.* This document is the authoritative reference for **constituent variables** in -capgen-ng — what they are, how scheme authors declare them in metadata, what +capgen — what they are, how scheme authors declare them in metadata, what the host model has to do to plumb them through, what the generator emits, and how the per-instance lifecycle works. @@ -36,7 +36,7 @@ typically a tracer / mass mixing ratio (water vapor, cloud liquid, ozone, chemistry species) — together with its **tendency**, the rate of change that physics writes back so the dycore can advect/integrate it forward. -In capgen-ng, the constituent layer has three concerns: +In capgen, the constituent layer has three concerns: 1. **Registration** — declaring at model startup which constituents exist (their standard name, units, vertical layout, advection flag, …). @@ -127,7 +127,7 @@ name is a constituent or an ordinary variable is the **host's** decision (CAM-SIMA exposes water vapor as a constituent; CCPP-SCM may expose the same name as an ordinary host variable), so a scheme that only **reads** a constituent — the base species, or a `tendency_of_` — does **not** -repeat the `advected` / `constituent` flag. capgen-ng infers +repeat the `advected` / `constituent` flag. capgen infers constituent-ness for an unflagged `intent=in/inout` consumer from the scheme-metadata-wide set of names *some* scheme flags (`VariableResolver.constituent_stdnames()`): an unflagged read of the @@ -540,7 +540,7 @@ The host should call this for every instance that successfully called ## 6. Generated code structure -When any suite touches constituent state, capgen-ng emits one extra +When any suite touches constituent state, capgen emits one extra module per generator run: **`ccpp_host_constituents.F90`**. ### Module declarations @@ -670,7 +670,7 @@ get the absolute paths to these files at the right output location. ## 7. Multi-instance design -In capgen-ng, **per-instance state** means: each "instance" (typically +In capgen, **per-instance state** means: each "instance" (typically an OpenMP team / chunk-domain partition) has its own copy of the state arrays, indexed by `instance_number ∈ [1, number_of_instances]`. @@ -776,7 +776,7 @@ The framework's `ccpp_constituent_properties_t` now carries a private `ccpt_deallocate` only deallocates the underlying prop when the flag is `.true.`; otherwise it just nullifies its pointer. -Under capgen-ng's explicit-registration model, all +Under capgen's explicit-registration model, all `ccpp_constituent_properties_t` objects are **target-owned by the caller** (the host's `host_constituents(:)` array, or the per-suite `_dynamic_constituents(:)` buffer). We never set the flag, so @@ -858,7 +858,7 @@ message naming the offending token. build, even when no scheme or host actually uses the constituent system (no `ccpp_constituent_properties_t(:)` register-phase arg, no `is_constituent`-flagged scheme arg, no framework-named - `index_of_` / `ccpp_constituents` / etc. claimed by capgen-ng). + `index_of_` / `ccpp_constituents` / etc. claimed by capgen). When the host owns its own indices (SCM/GFS) and no scheme exercises the constituent path, the generated file is dead code that should be suppressed. Tracked as a deferred item; the `host_dict` precedence @@ -868,7 +868,7 @@ message naming the offending token. ## 9. Differences from original capgen -| Aspect | Original capgen | capgen-ng | +| Aspect | Original capgen | capgen | |---|---|---| | Constituent object location | Generated `_ccpp_cap.F90` module | `ccpp_host_constituents.F90` (one per generator run) | | Per-instance | No (single instance) | Yes (`obj(:)` allocatable, sized to `number_of_instances`) | @@ -888,13 +888,13 @@ message naming the offending token. those work unchanged. For the ~16 schemes that rely on original capgen's auto-clone path (`advected = True` on a `_run` arg with no matching register-phase source), pass - `--legacy-auto-clone-constituents` to `ccpp_capgen_ng.py` and - `ccpp_validator.py` — capgen-ng then auto-registers those + `--legacy-auto-clone-constituents` to `ccpp_capgen.py` and + `ccpp_validator.py` — capgen then auto-registers those constituents into the per-suite dynamic-constituents buffer the same way original capgen did. See `doc/auto_clone_constituents.md`. - **Host metadata**: drop any explicit declaration of `ccpp_model_constituents_object` if you carried one over from a - previous capgen-ng experiment — the generator owns it now. + previous capgen experiment — the generator owns it now. - **Host Fortran**: change all `_ccpp_*_constituents` calls to the unprefixed names (`ccpp_register_constituents` etc.) and add `instance_number` to every call site. diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index a599b89e..6343c6b7 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -6,12 +6,12 @@ **Intended audience:** CCPP framework team, CAM-SIMA team **Status:** Discussion document — no decisions are final. Proposals A/B/C below remain pending the upcoming meeting; the bug fix from -Proposal A (the `ccpt_deallocate` ownership flag) and the capgen-ng +Proposal A (the `ccpt_deallocate` ownership flag) and the capgen internal cleanup from Proposal B (§4.8) have landed; the missing setters from Proposal A and the `is_match` relaxation from Proposal B have not. Independent of A/B/C, the per-suite dynamic_constituents buffer was made per-instance on 2026-05-18 to fix a multi-instance -mutation conflict — see §4.13. Since 2026-06-03 capgen-ng drives the +mutation conflict — see §4.13. Since 2026-06-03 capgen drives the real CAM-SIMA build (via the `cime_config/capgen_compat/` facade): the `kessler`, `rrtmgp`, and `se_cslam`/CSLAM (FCAM7 `cam7`) cases all build and run on Derecho. That integration added the **rule-b** consumer path @@ -30,7 +30,7 @@ but it carries: - **A latent framework bug** in `ccpp_constituent_prop_mod` that crashes on teardown of explicitly-registered (target-passed) constituent property - arrays. Fixed in capgen-ng's framework copy 2026-05-12; needs to land + arrays. Fixed in capgen's framework copy 2026-05-12; needs to land upstream. - **Architectural confusion** about which properties are *physics-portable* (the scheme owns them) versus *host-configuration* (the host owns them). @@ -41,8 +41,8 @@ but it carries: have no setters; `is_match` is overly strict about properties hosts should be free to change. - **Two registration models** coexist — original capgen's auto-clone of - is_constituent scheme args, and capgen's/capgen-ng's explicit register-phase + - host-side declaration. Capgen-ng deliberately dropped auto-clone. + is_constituent scheme args, and capgen's/capgen's explicit register-phase + + host-side declaration. Capgen deliberately dropped auto-clone. This document is a structured brief for a discussion this week. It does NOT pre-commit to any decision; it lays out what exists, what's broken, @@ -53,7 +53,7 @@ what we audited, and what proposals are on the table. ## Table of contents 1. [How original capgen handles constituents](#1-how-original-capgen-handles-constituents) -2. [How capgen-ng handles constituents](#2-how-capgen-ng-handles-constituents) +2. [How capgen handles constituents](#2-how-capgen-handles-constituents) 3. [What CAM-SIMA actually needs (audit)](#3-what-cam-sima-actually-needs-audit) 4. [Bugs and design flaws](#4-bugs-and-design-flaws) 5. [Property classification (Class A vs Class B)](#5-property-classification-class-a-vs-class-b) @@ -160,7 +160,7 @@ All three flow into one `%new_field` table. --- -## 2. How capgen-ng handles constituents +## 2. How capgen handles constituents ### 2.1 Mental model @@ -197,7 +197,7 @@ The resolver classifies each scheme arg into exactly one source. A flag. Whether a standard name is a constituent or an ordinary variable is the **host's** decision (CAM-SIMA exposes water vapor as a constituent; CCPP-SCM may expose the same name as an ordinary host - variable), so capgen-ng infers it from the scheme-metadata-wide set of + variable), so capgen infers it from the scheme-metadata-wide set of flagged names (`VariableResolver.constituent_stdnames()`) rather than from the consumer's own metadata. An unflagged `intent=in` read of the base name resolves to `%vars_layer(...)`; an unflagged `intent=in` read @@ -223,9 +223,9 @@ The resolver classifies each scheme arg into exactly one source. A drained into `ccpp_model_constituents_obj(inst)` by `ccpp_register_constituents`. -The auto-clone-from-metadata path is **gone from capgen-ng's default +The auto-clone-from-metadata path is **gone from capgen's default behaviour**. If a scheme declares `advected=true` on an arg but no -source registers that standard name, capgen-ng emits a runtime check +source registers that standard name, capgen emits a runtime check during `ccpp_initialize_constituents` that errors with the missing name. @@ -236,9 +236,9 @@ registration (production CAM-SIMA's atmospheric_physics tree is the immediate consumer). This is a transient migration shim — see `doc/auto_clone_constituents.md` for the full reference and removal procedure. It is single-instance only and explicitly -flagged so future capgen-ng work is *not* expected to keep it +flagged so future capgen work is *not* expected to keep it indefinitely. The reform proposals in §6–§8 below are unchanged by -the shim's existence: capgen-ng's chosen architecture is still +the shim's existence: capgen's chosen architecture is still explicit registration. ### 2.4 Per-instance state @@ -380,18 +380,18 @@ single-global limitation immediately. ## 4. Bugs and design flaws This section lists known issues across the three layers (framework, -original capgen, capgen-ng). Items marked **(FIXED)** were resolved +original capgen, capgen). Items marked **(FIXED)** were resolved 2026-05-12 and either are or will be PRs; items marked **(OPEN)** are intentionally left for this discussion. -### 4.1 Framework: `ccpt_deallocate` ownership bug (FIXED in capgen-ng tree, needs upstream PR) +### 4.1 Framework: `ccpt_deallocate` ownership bug (FIXED in capgen tree, needs upstream PR) - **Location**: `src/ccpp_constituent_prop_mod.F90`, `ccpt_deallocate` + `ccpt_set`. - **Symptom**: `free(): invalid size` crash when `ccp_model_const_reset` is called on a properly-locked table whose entries came from pointer-assigned targets (the common pattern - under capgen-ng's explicit registration; also potentially under + under capgen's explicit registration; also potentially under original capgen's `host_constituents` path). - **Root cause**: `ccpt_set` does pointer assignment (`this%prop => const_ptr`); `ccpt_deallocate` does an unconditional @@ -400,7 +400,7 @@ intentionally left for this discussion. - **Why it didn't surface earlier**: original capgen's advection test only calls `deallocate` once between a *failing* register and a *successful* one — at that point `lock_table` has not populated - `const_metadata`, so the broken inner loop is skipped. Capgen-ng + `const_metadata`, so the broken inner loop is skipped. Capgen triggers it because its teardown calls `reset` after a successful lock. - **Fix landed 2026-05-12**: added `framework_owns_me` private flag on @@ -409,9 +409,9 @@ intentionally left for this discussion. setter; `ccpt_deallocate` now only deallocates when the flag is set. Original capgen's auto-clone path in `scripts/constituents.py` updated to call `set_framework_owned(.true.)` after `allocate`. - Diffs in `src/ccpp_constituent_prop_mod.F90` (and capgen-ng's + Diffs in `src/ccpp_constituent_prop_mod.F90` (and capgen's parallel copy) + `scripts/constituents.py`. -- **Status**: framework tests pass; capgen-ng unit-test suite (1127 passing +- **Status**: framework tests pass; capgen unit-test suite (1127 passing as of 2026-05-13) is green. Still needs upstream PR to ccpp-framework + original ccpp-capgen. @@ -495,16 +495,16 @@ either generate one cap per instance or restructure. The synthetic scope between suite and host serves correctness but adds a code path that most contributors don't read. If we drop it -(capgen-ng has), the variable-matching algorithm shrinks. +(capgen has), the variable-matching algorithm shrinks. -### 4.8 Capgen-ng: `_FRAMEWORK_CONST_DIM_INPUTS` cleanup (LANDED 2026-05-13) +### 4.8 Capgen: `_FRAMEWORK_CONST_DIM_INPUTS` cleanup (LANDED 2026-05-13) `generator/host_cap.py` no longer carries the hand-curated frozenset of standard names; framework-constituent dimension references now ride on a dedicated `used_const_dim_std_names` field on `ResolvedArg`. Closes the "hand-curated → structured field" REVISIT note that was in the code. -### 4.9 Capgen-ng: no codegen-time cross-check of scheme registration (OPEN) +### 4.9 Capgen: no codegen-time cross-check of scheme registration (OPEN) The resolver knows every `is_constituent` arg's standard name (in `SuiteResolution.constituent_index_names`) but doesn't know what each @@ -518,15 +518,15 @@ added 2026-05-12). Stronger options: calls and cross-check. - (c) Keep runtime check as authoritative, document the gap. -### 4.10 Capgen-ng: scheme-metadata `diagnostic_name` for is_constituent args is host-specific (OPEN) +### 4.10 Capgen: scheme-metadata `diagnostic_name` for is_constituent args is host-specific (OPEN) -Same issue as §4.4 but in capgen-ng's metadata layer. Today's +Same issue as §4.4 but in capgen's metadata layer. Today's `diagnostic_name` attribute on a scheme metadata arg flows into `datatable.xml` and is then trusted as "the" diagnostic name. If we adopt setter-based class-B overrides, this attribute should either be dropped for constituent args or marked as a default-only hint. -### 4.11 Capgen-ng: `ccpp_scheme_utils` singleton (OPEN — documented limit) +### 4.11 Capgen: `ccpp_scheme_utils` singleton (OPEN — documented limit) `ccpp_initialize_constituent_ptr(const_obj)` stores a single module-level pointer. Schemes that use `ccpp_constituent_index(stdname)` get that @@ -538,7 +538,7 @@ scheme-registering schemes don't rely on this; documented in `instance_number` through `ccpp_constituent_index` (interface change) or maintaining a per-instance pointer table. -### 4.12 Capgen-ng: drop `diagnostic_name_fixed`, keep only `diagnostic_name` (OPEN — proposed simplification) +### 4.12 Capgen: drop `diagnostic_name_fixed`, keep only `diagnostic_name` (OPEN — proposed simplification) Today the metadata layer carries two mutually-exclusive scheme-arg attributes: @@ -592,11 +592,11 @@ added on the framework side. Hosts that want runtime override get spirit to the existing `horizontal_loop_extent → horizontal_dimension` shim. Remove the rewrite once known consumers are migrated. -### 4.13 Capgen-ng: per-suite `dynamic_constituents` buffer was shared across instances (FIXED 2026-05-18) +### 4.13 Capgen: per-suite `dynamic_constituents` buffer was shared across instances (FIXED 2026-05-18) -- **Location**: `capgen-ng/generator/host_constituents.py` (buffer +- **Location**: `capgen/generator/host_constituents.py` (buffer declaration + `ccpp_register_constituents` iteration); - `capgen-ng/generator/suite_cap.py::_register_lines` (the two-pass + `capgen/generator/suite_cap.py::_register_lines` (the two-pass count→allocate→pack inside `_register`). - **Symptom**: with two or more instances and any register-phase scheme that produces constituents, the second per-instance @@ -647,10 +647,10 @@ shim. Remove the rewrite once known consumers are migrated. - **Position relative to Proposals A/B/C**: orthogonal — none of the three proposed touching the buffer. Independently adopted. -### 4.14 Capgen-ng: error-output keyword inconsistency across emitted public API (OPEN — observation) +### 4.14 Capgen: error-output keyword inconsistency across emitted public API (OPEN — observation) -- **Location**: `capgen-ng/generator/host_cap.py:370,446,*` (lifecycle - subs) vs `capgen-ng/generator/host_constituents.py` (the entire +- **Location**: `capgen/generator/host_cap.py:370,446,*` (lifecycle + subs) vs `capgen/generator/host_constituents.py` (the entire constituent wrapper family). - **Symptom**: the public Fortran argument carrying the CCPP error flag does not have a consistent name across the cap's surface area. @@ -688,7 +688,7 @@ shim. Remove the rewrite once known consumers are migrated. thin shims around framework methods (`ccpp_model_constituents_t%new_field`, `%lock_table`, `%num_constituents`, etc.) that all take `errcode=` per - `capgen-ng/src/ccpp_constituent_prop_mod.F90`. Hardcoding `errcode` + `capgen/src/ccpp_constituent_prop_mod.F90`. Hardcoding `errcode` on the wrapper means the wrapper body just forwards `errcode=errcode` instead of `errcode=` -- one less host-dict lookup, but at the cost of breaking the @@ -721,7 +721,7 @@ shim. Remove the rewrite once known consumers are migrated. ### 4.15 CAM-SIMA compat layer: `write_init_files` mis-flagged unflagged constituent-tendency consumers (FIXED 2026-06-05) - **Location**: `cime_config/capgen_compat/_var_wrapper.py` in CAM-SIMA - — the facade that lets capgen-ng drive CAM-SIMA's *unchanged* + — the facade that lets capgen drive CAM-SIMA's *unchanged* `write_init_files.py` / `cam_autogen.py` — method `_VarWrapper.from_resolved_arg`. - **Symptom**: the `se_cslam` (FCAM7 `cam7`) build failed AFTER cap @@ -731,7 +731,7 @@ shim. Remove the rewrite once known consumers are migrated. - **Mechanism**: in `cam7` the convection/stratiform schemes (`dadadj`, `zm_conv_evap`, `rk_stratiform`, `zm_convr`, `cloud_particle_sedimentation`) write that tendency as a FLAGGED - constituent tendency (`constituent=true intent=out`) → capgen-ng routes + constituent tendency (`constituent=true intent=out`) → capgen routes them to `%vars_layer_tend` (`source='constituent'`, NOT recorded in `suite_vars`) and the name enters `const_stds`. The four `sima_diagnostics` schemes read it back `intent=in` UNFLAGGED → rule b @@ -815,15 +815,15 @@ constituent property is conceptually owned by either the scheme - **The `diag_name` requirement at `%instantiate`** — demote to optional with `''` default. - **(Not adopting)** Original capgen's auto-clone path. Already gone - in capgen-ng; this discussion does not propose bringing it back. + in capgen; this discussion does not propose bringing it back. Listed for completeness because the option is in memory. ### Replace -- **`ConstituentVarDict`** as a concept — capgen-ng already runs +- **`ConstituentVarDict`** as a concept — capgen already runs without it. If the framework or future generator code references it, dropping is fine. -- **Single-global `ccpp_model_constituents_obj`** — capgen-ng's +- **Single-global `ccpp_model_constituents_obj`** — capgen's per-instance array is the replacement. Original capgen could be retrofitted, but the priority depends on whether multi-instance enters the original capgen's roadmap. @@ -843,7 +843,7 @@ constituent property is conceptually owned by either the scheme - **Document the lifecycle** clearly. `doc/constituents.md` is ~960 lines; targeted additions for "register-then-override" workflow once the new setters land. -- **Capgen-ng-internal cleanup** (LANDED 2026-05-13): replaced +- **Capgen-internal cleanup** (LANDED 2026-05-13): replaced `_FRAMEWORK_CONST_DIM_INPUTS` with a `used_const_dim_std_names` field on `ResolvedArg`. @@ -925,7 +925,7 @@ These are the calls we need to make in the meeting. - (a) Maintain a per-instance pointer table; threading `instance_number` through `ccpp_constituent_index`. - (b) Document the limitation, route around it (no scheme uses - `ccpp_constituent_index` under multi-instance — capgen-ng + `ccpp_constituent_index` under multi-instance — capgen already enforces `index_of_` everywhere). - **Recommendation**: (b). It's a one-line doc note and zero code change. @@ -974,7 +974,7 @@ These are the calls we need to make in the meeting. - *Against:* many constituents have no physics tendency (the column is already there regardless, so forcing a declaration buys little); the index is implicit by design and exposing it as a required member - re-introduces the index bookkeeping capgen-ng deliberately hid; the + re-introduces the index bookkeeping capgen deliberately hid; the base is the only thing that *must* be registered. - *Open sub-question:* if not forced, should the resolver at least **warn** when a `tendency_of_` is produced for an `` that no register scheme @@ -1016,12 +1016,12 @@ scheme on `advected` still hit the "incompatible constituent" error. override. - Update `doc/constituents.md` with the register-then-override workflow. -- (capgen-ng) Reject `diagnostic_name` on `is_constituent=True` +- (capgen) Reject `diagnostic_name` on `is_constituent=True` scheme args at parse time, or downgrade it to a default-only hint. -- (capgen-ng) **DONE 2026-05-13**: replaced `_FRAMEWORK_CONST_DIM_INPUTS` +- (capgen) **DONE 2026-05-13**: replaced `_FRAMEWORK_CONST_DIM_INPUTS` with a `ResolvedArg.used_const_dim_std_names` field. -**Cost**: ~150 lines framework + ~50 lines capgen-ng + tests. +**Cost**: ~150 lines framework + ~50 lines capgen + tests. CAM-SIMA host code can stay as-is (the 4 scheme-side registrations continue to work with their existing class-B values; they're just not enforced anymore). Optional: tidy the 4 schemes to pass class-A @@ -1032,7 +1032,7 @@ hosts. The class-B override pattern that CAM-SIMA already uses for `thermo_active` and `water_species` generalizes. **Limit**: does not change the registration model (still -explicit-only in capgen-ng, still auto-clone in original capgen). +explicit-only in capgen, still auto-clone in original capgen). ### Proposal C — host-only registration @@ -1040,7 +1040,7 @@ explicit-only in capgen-ng, still auto-clone in original capgen). - Move the 4 cam-sima scheme-side register calls into a CAM-SIMA helper module called from `cam_comp.F90`'s initialization. - Drop register-phase `ccpp_constituent_properties_t(:)` support - from capgen-ng (and possibly original capgen). Schemes only + from capgen (and possibly original capgen). Schemes only consume constituents; only the host registers. - Codegen-time enforcement: any `advected=true` scheme arg whose std_name is not in the host's enumeration → codegen error. @@ -1048,7 +1048,7 @@ explicit-only in capgen-ng, still auto-clone in original capgen). entirely. **Cost**: ~300 lines code total; requires coordinated PRs across -ccpp-framework, ccpp-capgen, ccpp-capgen-ng, atmospheric_physics, and +ccpp-framework, ccpp-capgen, ccpp-capgen, atmospheric_physics, and CAM-SIMA. The 4 schemes need their `_register` routines deleted (or made no-ops); the host needs a new helper. @@ -1065,7 +1065,7 @@ registration model. | Aspect | A | B | C | |---|---|---|---| | Lines changed | ~50 | ~200 | ~500+ | -| Coordination needed | framework only | framework + capgen-ng | framework + both generators + cam-sima | +| Coordination needed | framework only | framework + capgen | framework + both generators + cam-sima | | Fixes the crash | yes | yes | yes | | Fixes `diag_name` portability | yes (host overrides) | yes | yes | | Relaxes `is_match` | no | yes | yes | @@ -1139,13 +1139,13 @@ setters that delegate to the underlying `ccpp_constituent_properties_t`. ## Cross-references -- `doc/constituents.md` — capgen-ng's user-facing constituents reference. -- `design_constituent_api.md` (memory) — capgen-ng's per-instance option-A design. +- `doc/constituents.md` — capgen's user-facing constituents reference. +- `design_constituent_api.md` (memory) — capgen's per-instance option-A design. - `design_constituents_mutability.md` (memory) — extended design notes incl. class A/B classification. - `project_implementation_status.md` (memory) — current implementation state and deferred items. - `scripts/constituents.py` — original capgen's host-cap generator. - `src/ccpp_constituent_prop_mod.F90` — framework. -- `capgen-ng/generator/host_constituents.py` — capgen-ng's host-side module emitter. -- `capgen-ng/generator/suite_resolver.py` (`_resolve_constituent_arg`) — capgen-ng's resolver routing. +- `capgen/generator/host_constituents.py` — capgen's host-side module emitter. +- `capgen/generator/suite_resolver.py` (`_resolve_constituent_arg`) — capgen's resolver routing. - `EXT/cam-sima/CAM-SIMA/src/physics/utils/cam_constituents.F90` — CAM-SIMA's host-side wrappers around framework setters. diff --git a/doc/file_catalogue_DRAFT.md b/doc/file_catalogue_DRAFT.md index aba0b3e0..778ea16a 100644 --- a/doc/file_catalogue_DRAFT.md +++ b/doc/file_catalogue_DRAFT.md @@ -1,9 +1,9 @@ -# capgen-ng repository — file catalogue (DRAFT) +# capgen repository — file catalogue (DRAFT) > **Status: temporary draft for the code-walkthrough prep.** One row per file, except > the many test/example *input* fixtures, which are collapsed. Once reviewed, the > relevant sections will be folded into `README.md` / `doc/DevelopersGuide/`. -> External checkouts under `EXT/` (UFS reference + capgen-ng integration trees) are +> External checkouts under `EXT/` (UFS reference + capgen integration trees) are > intentionally excluded — they are not part of this repository. ## Top level @@ -17,16 +17,16 @@ | `CODEOWNERS`, `.codecov.yml`, `.codee-format`, `.gitignore` | Repo/CI configuration (code owners, coverage, formatter, ignore rules). | | `.github/` | GitHub Actions CI workflows (unit tests, end-to-end tests, doxygen). | -## `capgen-ng/` — command-line entry points +## `capgen/` — command-line entry points | File | Description | |------|-------------| | `__init__.py` | Package marker (“next-generation CCPP code generator”). | -| `ccpp_capgen_ng.py` | **Main generator CLI.** Parses metadata + the SDF, resolves variables, and writes the caps, `ccpp_kinds.F90`, and `datatable.xml`. Hosts flags like `--kind-type`, `--trace`, `--no-host-introspection`, and the compat shims. | +| `ccpp_capgen.py` | **Main generator CLI.** Parses metadata + the SDF, resolves variables, and writes the caps, `ccpp_kinds.F90`, and `datatable.xml`. Hosts flags like `--kind-type`, `--trace`, `--no-host-introspection`, and the compat shims. | | `ccpp_datafile.py` | CLI to query the generated `datatable.xml` (generated files, scheme files, dependencies) for build systems / CMake. | | `ccpp_validator.py` | **Standalone validator** — checks scheme Fortran source against its `.meta` (intent/type/kind/rank/dimensions). Separate tool from the generator; owns the one Fortran parser. | -## `capgen-ng/generator/` — cap code generation +## `capgen/generator/` — cap code generation | File | Description | |------|-------------| @@ -43,7 +43,7 @@ | `kinds_writer.py` | Writes `ccpp_kinds.F90` (kind definitions the caps `use`). | | `trace.py` | Shared helpers emitting the gated `if (trace) write(...) 'CCPP TRACE …'` lines in every cap (toggled by `--trace`). | -## `capgen-ng/metadata/` — metadata parsing & variable resolution +## `capgen/metadata/` — metadata parsing & variable resolution | File | Description | |------|-------------| @@ -56,7 +56,7 @@ | `dim_aliases.py` | **Transient shim** — collapses equivalent GFS-physics dimension names. | | `auto_clone_constituents.py` | **Transient shim** — reinstates original-capgen auto-cloning of static constituents. | -## `capgen-ng/metadata/parse_tools/` — shared parse utilities +## `capgen/metadata/parse_tools/` — shared parse utilities | File | Description | |------|-------------| @@ -68,7 +68,7 @@ | `fortran_conditional.py` | Builds Fortran conditional expressions (in local names) for active/optional-argument handling. | | `xml_tools.py` | XML helpers — entity expansion and pretty-printed writing (SDF / datatable). | -## `capgen-ng/schema/` & `capgen-ng/src/` — schema + shipped runtime Fortran +## `capgen/schema/` & `capgen/src/` — schema + shipped runtime Fortran | File | Description | |------|-------------| @@ -114,7 +114,7 @@ comparison driver, and CMake glue. The fixtures are collapsed; the row describes | Case | What it exercises | |------|-------------------| -| `capgen_ng/` | **Overall generator capabilities** — multiple suites & groups, DDT usage (incl. an undocumented DDT member), `ccpp_constant_one:N` and bare-`N` dimensions, non-standard/integer dimensions, variables promoted to suite level, dimensions set in the register phase and used to allocate module-level interstitials, and threading. | +| `capgen/` | **Overall generator capabilities** — multiple suites & groups, DDT usage (incl. an undocumented DDT member), `ccpp_constant_one:N` and bare-`N` dimensions, non-standard/integer dimensions, variables promoted to suite level, dimensions set in the register phase and used to allocate module-level interstitials, and threading. | | `advection/` | Constituent advection — cloud liquid/ice constituents with tendency application (`apply_constituent_tendencies` invoked twice); includes a deliberate error suite (`cld_suite_error.xml`) to exercise diagnostics. | | `advection_auto_clone/` | Same fixtures as `advection/`, run through the `--legacy-auto-clone-constituents` shim path. | | `ddthost/` | A host whose CCPP data is carried in a derived type (`host_ccpp_ddt`); runs the temp + DDT suites against it. | @@ -134,9 +134,9 @@ comparison driver, and CMake glue. The fixtures are collapsed; the row describes | File | Description | |------|-------------| | `README.md` | Documentation index. | -| `redesign_prompt.md`, `redesign_analysis.md`, `redesign_analysis_original_*.md` | Original redesign brief and analysis that motivated capgen-ng. | +| `redesign_prompt.md`, `redesign_analysis.md`, `redesign_analysis_original_*.md` | Original redesign brief and analysis that motivated capgen. | | `briefing.md`, `briefing_pm.md` | Design briefings. | -| `migration.md` | Guide for migrating a host from ccpp-prebuild/original-capgen to capgen-ng. | +| `migration.md` | Guide for migrating a host from ccpp-prebuild/original-capgen to capgen. | | `capgen_compat_layer.md` | Documents the transient compatibility shims (legacy names, dim aliases, auto-clone). | | `constituents.md` | Constituent-handling design. | | `constituents_overhaul.md` | Proposed constituent-model overhaul (proposals A/B/C). | diff --git a/doc/migration.md b/doc/migration.md index 084eb80d..7673b472 100644 --- a/doc/migration.md +++ b/doc/migration.md @@ -1,15 +1,15 @@ -# Migrating from ccpp-prebuild / ccpp-capgen to capgen-ng +# Migrating from ccpp-prebuild / ccpp-capgen to capgen This document captures the **user-facing differences** a host model author or scheme author needs to know when moving metadata, suite XML, and host Fortran from the legacy ccpp-prebuild + ccpp-capgen toolchain to -**capgen-ng**. It complements `doc/redesign_prompt.md` (design spec) and +**capgen**. It complements `doc/redesign_prompt.md` (design spec) and `doc/redesign_analysis.md` (analysis of the old systems). *Last revised: 2026-06-05.* Current unit-test suite: 1516 passing. **Repository layout** (post-2026-05-13 cleanup): tooling lives under -`capgen-ng/` (top-level of this repo). Unit tests live at the top +`capgen/` (top-level of this repo). Unit tests live at the top level in `unit-tests/`; end-to-end tests in `end-to-end-tests/`. Run the unit suite from the repo root with `python -m pytest unit-tests/`. @@ -78,7 +78,7 @@ Example with multi-line dependencies (real CCPP physics pattern): > `ty_optical_props_1scl_ccpp`) cannot inherit its module from a sibling. > If the defining Fortran module name differs from the DDT table (type) > name — which it almost always does for these wrappers — you **must** -> declare `module_name` explicitly. capgen-ng does *not* guess (e.g. +> declare `module_name` explicitly. capgen does *not* guess (e.g. > from the file name); a DDT it can't resolve raises a clear error at > generation time naming the type and the `module_name` remedy. @@ -99,7 +99,7 @@ Inside a `[ var_name ]` section. All optional. When a host variable carries `active = ()`, the host's contract with the cap is "this variable's storage is only valid when -the condition holds". capgen-ng honors that contract differently +the condition holds". capgen honors that contract differently depending on the matching scheme arg's optionality: **Scheme arg is `optional = True`** — the cap uses pointer association @@ -207,7 +207,7 @@ Error messages name the source as `host`, `control`, or `suite` so you know whose contract you're violating. **Suite-owned storage is never default-initialized** — by design. -capgen-ng emits the `ccpp__data` components with no default value. +capgen emits the `ccpp__data` components with no default value. An `intent(out)` argument is the scheme's contract to define that variable on *every* return path; the framework will not paper over an unset output the way original capgen's zero-initialized interstitials did. A ported @@ -221,7 +221,7 @@ early-return paths for unset `intent(out)` args. A **suite-owned variable** (an interstitial: first written by a scheme with `intent=out`, then consumed by another) is stored as a component -of the generated `ccpp__data` DDT. capgen-ng allocates it for +of the generated `ccpp__data` DDT. capgen allocates it for you — **once**, in `suite_data_init_fields`, which runs at the very start of `_init`. That works only when every dimension is known that early, i.e. a **host variable** or a value set in the **`register`** @@ -239,7 +239,7 @@ Such a variable must be declared **`allocatable`** and allocated by its auto-deallocated on entry, so element assignment needs it allocated first). -For an `allocatable` arg capgen-ng then: (1) does **not** pre-allocate it +For an `allocatable` arg capgen then: (1) does **not** pre-allocate it in `init_fields`; (2) passes the **whole** component at call sites (`...%var`, no array section — an allocatable/assumed-shape mismatch is otherwise a compile error); and (3) still frees it in @@ -255,7 +255,7 @@ declared `number_of_vertical_interfaces_in_RRTMGP` must be sized with that value, **not** the host's `vertical_interface_dimension` nor `nlay+1`, which differ when the scheme runs on a reduced vertical grid.) -**Generation-time guard.** capgen-ng rejects a *non*-`allocatable` +**Generation-time guard.** capgen rejects a *non*-`allocatable` suite-owned array whose dimension is written by a scheme in any phase after `register` — it would otherwise be allocated from uninitialized memory. The error names the variable, the offending dimension, and the @@ -328,10 +328,10 @@ both pairs entirely. ### 1.8 Deprecated standard names rewritten by `--legacy-mode` `--legacy-mode` is a transient migration shim that rewrites a small -set of deprecated standard names to their canonical capgen-ng +set of deprecated standard names to their canonical capgen equivalents at parse time. The full table currently covers: -| Deprecated (legacy) | Canonical (capgen-ng) | +| Deprecated (legacy) | Canonical (capgen) | |--------------------------------|--------------------------| | `horizontal_loop_extent` | `horizontal_dimension` | | `number_of_openmp_threads` | `number_of_threads` | @@ -339,14 +339,14 @@ equivalents at parse time. The full table currently covers: Why each entry: * `horizontal_loop_extent` — ccpp-prebuild / original ccpp-capgen used - this for the horizontal-axis std name in scheme metadata. capgen-ng + this for the horizontal-axis std name in scheme metadata. capgen uses `horizontal_dimension` uniformly; the run-vs-non-run distinction isn't expressed in scheme metadata anymore (host passes `horizontal_loop_begin` / `horizontal_loop_end` as control vars and the generated cap slices accordingly). * `number_of_openmp_threads` — legacy CCPP-physics hosts (CCPP-SCM 17p8 in particular) size per-thread DDT containers by - `number_of_openmp_threads` (e.g. `physics%Interstitial`). capgen-ng + `number_of_openmp_threads` (e.g. `physics%Interstitial`). capgen uses `number_of_threads`, which matches the `thread_number` control variable, so the registered scalar-index dim table can substitute `physics%Interstitial(thread_number)%…` automatically (see §3.4). @@ -360,7 +360,7 @@ Migration paths: 1. **Edit the metadata** (recommended) — search-and-replace the legacy names in every host / scheme `.meta` you maintain. 2. **Use `--legacy-mode`** (transient) — pass `--legacy-mode` to both - `ccpp_capgen_ng.py` and `ccpp_validator.py` and the renames happen + `ccpp_capgen.py` and `ccpp_validator.py` and the renames happen at parse time. A loud warning banner prints at startup, listing every pair the shim is rewriting, so the substitution is never invisible. This shim *will be removed*; treat it as a runway, @@ -430,7 +430,7 @@ favour of `vertical_layer_dimension` and the flag becomes unnecessary. ### 2.1 Schema v2.0 with nested-suite expansion -Capgen-ng parses v2.0 SDFs and expands `` references +Capgen parses v2.0 SDFs and expands `` references recursively at parse time. See `doc/redesign_prompt.md` §3 and the `suite_v2_0.xsd` schema. @@ -484,7 +484,7 @@ counter and the total iteration count via two CCPP standard names: | `ccpp_loop_extent` | integer | Total iterations — the `loop=` value on the `` | These are **loop-context control variables**: the host model does **not** -declare them. capgen-ng emits them automatically as locals in the +declare them. capgen emits them automatically as locals in the generated group cap (the `do` loop's induction variable for the counter, the loop bound for the extent), and resolves any scheme arg requesting them against those locals. @@ -627,7 +627,7 @@ elsewhere for API symmetry. ### 3.3 Module-name convention (host, scheme, and DDT tables) -capgen-ng trusts metadata and does **not** parse Fortran, so it derives +capgen trusts metadata and does **not** parse Fortran, so it derives the Fortran module name from the metadata: by default `module name = table name`. When the Fortran `module` statement does not match the `[ccpp-table-properties] name`, declare the real module name with the @@ -661,10 +661,10 @@ their module names, is largely a batch of `module_name` injections. ### 3.4 Registered scalar-index dimensions A small set of CCPP standard-name dimensions are *registered*: each -one is a count that capgen-ng auto-collapses to a paired scalar index +one is a count that capgen auto-collapses to a paired scalar index variable at every access site. -| Count dim (in `dimensions = (...)`) | Index var (capgen-ng substitutes) | +| Count dim (in `dimensions = (...)`) | Index var (capgen substitutes) | |---|---| | `number_of_instances` | `instance_number` | | `number_of_threads` | `thread_number` | @@ -699,12 +699,12 @@ call site — no metadata work required on the scheme side. type = real | kind = kind_phys dimensions = (number_of_threads, horizontal_dimension) # ILLEGAL - capgen-ng will reject it at parse time with a message pointing + capgen will reject it at parse time with a message pointing at the wrap-in-DDT remediation pattern. Wrap the leaf in a container DDT instead. The registered table lives in -[`capgen-ng/metadata/registered_dimensions.py`](../capgen-ng/metadata/registered_dimensions.py). +[`capgen/metadata/registered_dimensions.py`](../capgen/metadata/registered_dimensions.py). It carries a four-step recipe at the top of the file for adding new pairings. @@ -712,10 +712,10 @@ pairings. ## 4. Generator CLI and build integration -### 4.1 `ccpp_capgen_ng.py` invocation +### 4.1 `ccpp_capgen.py` invocation ``` -python ccpp_capgen_ng.py \ +python ccpp_capgen.py \ --host-files [,,...] \ --scheme-files [,,...] \ --suites [,,...] \ @@ -741,13 +741,13 @@ self-contained and grep-tagged for clean removal: **`--legacy-mode`** (transient migration shim, will be removed): silently rewrites a small set of deprecated CCPP standard names to -their capgen-ng equivalents at parse time — see §1.8 for the full +their capgen equivalents at parse time — see §1.8 for the full table (`horizontal_loop_extent` → `horizontal_dimension`, `number_of_openmp_threads` → `number_of_threads`). The rewrite fires for both standard-name attributes AND dimension tokens. Prints a loud warning banner at startup, enumerating every pair the shim is rewriting, so the substitution is never invisible. Available on both -`ccpp_capgen_ng.py` and `ccpp_validator.py` (keep the flag consistent +`ccpp_capgen.py` and `ccpp_validator.py` (keep the flag consistent between the two when both are invoked from CMake). All translation logic is isolated in `metadata/legacy_compat.py` and tagged with `# legacy-compat:` comments at every touchpoint. @@ -774,7 +774,7 @@ the standard name when missing, `diag_name` falls back to local_name, `vertical_dim` lifted from the arg's dim list). Adds four legacy `%instantiate` kwargs to the parser (`default_value`, `min_value`, `water_species`, `mixing_ratio_type`). Available on both -`ccpp_capgen_ng.py` and `ccpp_validator.py` (the validator must +`ccpp_capgen.py` and `ccpp_validator.py` (the validator must accept the four extra attrs). **Single-instance only** — declaring the `instance_number` + `number_of_instances` pair while the flag is on is a hard error before any suite is parsed. Module @@ -834,23 +834,23 @@ file lives in the target's parent directory (always under ### 4.5 Driving an existing capgen-based build: the CAM-SIMA compatibility layer A host whose build system was written against **original ccpp-capgen's -Python API** can adopt capgen-ng without rewriting that build system, by +Python API** can adopt capgen without rewriting that build system, by inserting a thin facade. CAM-SIMA does exactly this with -`cime_config/capgen_compat/` (in the CAM-SIMA tree, not in capgen-ng). +`cime_config/capgen_compat/` (in the CAM-SIMA tree, not in capgen). CAM-SIMA's `cam_autogen.py`, `generate_registry_data.py`, and `write_init_files.py` are unmodified; they import the facade instead of original capgen and keep calling the same object surface (`cap_database.host_model_dict()`, `cap_database.call_list(phase)`, `Var.get_prop_value(...)`, `Var.source.ptype`, …). -The facade re-implements that surface on top of capgen-ng's outputs: +The facade re-implements that surface on top of capgen's outputs: -- `_runner.py` invokes `ccpp_capgen_ng.py` and returns the resolver +- `_runner.py` invokes `ccpp_capgen.py` and returns the resolver results plus the `datatable.xml`. - `_cap_database.py` (`CapDatabase`) exposes `host_model_dict()` over the flat `host_dict` and `call_list(phase)` over the per-(scheme, phase) `ResolvedArg` lists, mapping original-capgen phase spellings - (`initialize`/`finalize`) onto capgen-ng's (`init`/`final`). + (`initialize`/`finalize`) onto capgen's (`init`/`final`). - `_var_wrapper.py` (`_VarWrapper`) reconstructs original capgen's per-variable accessors over a `HostVarEntry` (host path) or a `ResolvedArg` (call-list path). @@ -876,7 +876,7 @@ learned from the CAM-SIMA bring-up: "Missing required host variables: tendency_of_water_vapor_…" failure (`doc/constituents_overhaul.md` §4.15). -This facade is how capgen-ng currently drives the `kessler`, `rrtmgp`, +This facade is how capgen currently drives the `kessler`, `rrtmgp`, and `se_cslam`/CSLAM (FCAM7 `cam7`) CAM-SIMA cases end-to-end on Derecho — building and running to completion under both **gnu and intel**, with bit-comparable results. A short shareable brief (for the original @@ -1007,12 +1007,12 @@ state array is unallocated — there, "not allocated" really does mean Backward-compatible. Original capgen's auto-clone path in `scripts/constituents.py` has been updated to call the setter. -capgen-ng's `--legacy-auto-clone-constituents` shim (§6.4) +capgen's `--legacy-auto-clone-constituents` shim (§6.4) synthesises `%instantiate(...)` directly on slots of the per-suite dynamic-constituents buffer, so the properties objects are owned by the buffer from creation — no ownership transfer call needed. -### 6.2 capgen-ng constituent API +### 6.2 capgen constituent API (See `doc/constituents.md` for the full reference.) Highlights: @@ -1036,7 +1036,7 @@ the buffer from creation — no ownership transfer call needed. is a codegen error. A scheme that only READS a constituent or a `tendency_of_` need not re-flag it — see §6.5. - **`_register` is called exactly once per scheme** (2026-06-08). - capgen-ng packs each constituent scheme's returned + capgen packs each constituent scheme's returned `ccpp_constituent_properties_t(:)` array into the per-suite buffer in a single append pass, so a register routine may safely allocate persistent module state. (An earlier two-pass count+copy called register twice and @@ -1049,7 +1049,7 @@ If the host declares a framework-named standard name (`ccpp_constituents` / `ccpp_constituent_tendencies` / `ccpp_constituent_properties` / `number_of_ccpp_constituents` / `index_of_`) as a regular host variable, the resolver uses the -host's declaration and skips capgen-ng auto-provisioning. Matters +host's declaration and skips capgen auto-provisioning. Matters most for legacy hosts (GFS / SCM) that own their own tracer indices — e.g. `[ntcw]` with `standard_name = index_of_cloud_liquid_water_mixing_ratio_in_tracer_concentration_array` @@ -1069,7 +1069,7 @@ notably CAM-SIMA's atmospheric_physics tree, where ~16 of the ~20 constituent-touching schemes declare `advected = True` (or `constituent = True`, or `molar_mass = …`) in `_run` arg tables and rely on the framework to register the constituent — pass -`--legacy-auto-clone-constituents` to both `ccpp_capgen_ng.py` and +`--legacy-auto-clone-constituents` to both `ccpp_capgen.py` and `ccpp_validator.py`. What changes: @@ -1094,7 +1094,7 @@ What changes: from auto-clone — those resolve through the framework whole-buffer path, not as individual registrations. -What capgen-ng's other rules still require (the shim does **not** +What capgen's other rules still require (the shim does **not** relax them): - `intent = inout` on base constituents (`advected = True` on a @@ -1123,7 +1123,7 @@ variable. A scheme that merely **reads** such a name therefore must **not** repeat the `advected` / `constituent` flag — only the declaring/producing scheme (or the host) does. -capgen-ng infers constituent-ness for an unflagged consumer from the +capgen infers constituent-ness for an unflagged consumer from the scheme-metadata-wide set of names that *some* scheme flags (`VariableResolver.constituent_stdnames()`): @@ -1149,7 +1149,7 @@ must key constituent handling on `ResolvedArg.source == 'constituent'`, ### 6.6 `number_of_ccpp_constituents` as a dimension A scheme (or a suite-owned interstitial) may be dimensioned by the -framework constituent count `number_of_ccpp_constituents`. capgen-ng +framework constituent count `number_of_ccpp_constituents`. capgen resolves that count for *any* variable: call-site subscripts emit `:` for the constituent axis, and `_data` allocations size the axis from the per-instance constituent object's `%num_layer_vars`. This is @@ -1162,7 +1162,7 @@ number_of_ccpp_constituents)`. E2e fixture: ## 7. Validator -`capgen-ng/ccpp_validator.py` — standalone Fortran-vs-metadata checker. +`capgen/ccpp_validator.py` — standalone Fortran-vs-metadata checker. Validates **scheme** metadata against scheme Fortran files, and (since 2026-06-01) **host** and **DDT** metadata against host module-level declarations and derived-type definitions. @@ -1272,7 +1272,7 @@ dummy arguments (scheme args and control/lifecycle variables). | Codegen-time scheme-registration cross-check | Deferred; would require new `registers_std_names` metadata attr. | | `_FRAMEWORK_CONST_DIM_INPUTS` cleanup | **Done 2026-05-13**: hand-curated frozenset gone; framework-constituent dim refs ride on a dedicated `used_const_dim_std_names` field on `ResolvedArg`. | | Suppress `ccpp_host_constituents.F90` when unused | Deferred; currently emitted for every build even when no scheme/host actually exercises the constituent system. Now *correct* (empty) for SCM-style hosts thanks to the host-wins rule, but still dead code. See `design_constituent_host_wins.md`. | -| Python linter / formatter pass | Deferred; pick `ruff` and apply across `capgen-ng/`. | +| Python linter / formatter pass | Deferred; pick `ruff` and apply across `capgen/`. | | Generated Fortran ↔ Codee formatter idempotency | Deferred; emitted `.F90` must round-trip cleanly through the project's Codee Fortran formatter. | | `fortran_to_metadata` developer utility | Deferred; bootstraps a `.meta` skeleton from an existing `.F90` subroutine. | | `--legacy-mode` shim removal | Transient; remove `metadata/legacy_compat.py`, `unit-tests/test_legacy_compat.py`, and every `# legacy-compat:` touchpoint when scheme metadata has migrated. | @@ -1288,10 +1288,10 @@ dummy arguments (scheme args and control/lifecycle variables). marked "historic" where the implementation has evolved). - `doc/redesign_analysis.md` — analysis of the legacy ccpp-prebuild + ccpp-capgen toolchains. -- `doc/constituents.md` — full constituents reference for capgen-ng. +- `doc/constituents.md` — full constituents reference for capgen. - `doc/constituents_overhaul.md` — architecture review and reform proposals for the next iteration. -- `doc/capgen_compat_layer.md` — short brief on the CAM-SIMA ↔ capgen-ng +- `doc/capgen_compat_layer.md` — short brief on the CAM-SIMA ↔ capgen compatibility layer (§4.5); full reference is `cime_config/capgen_compat/README.md` in the CAM-SIMA tree. diff --git a/doc/redesign_analysis.md b/doc/redesign_analysis.md index e7dc360b..cb8e4d7a 100644 --- a/doc/redesign_analysis.md +++ b/doc/redesign_analysis.md @@ -1102,7 +1102,7 @@ physics-internal variables actually need to be managed. ### 8.6 Implementation decisions made during redesign -The following decisions were made during implementation of `capgen-ng` and are recorded +The following decisions were made during implementation of `capgen` and are recorded here as amendments to the analysis above. **State machine parameters are local to each generated group cap module.** diff --git a/doc/redesign_prompt.md b/doc/redesign_prompt.md index ad9ed41c..5ed79126 100644 --- a/doc/redesign_prompt.md +++ b/doc/redesign_prompt.md @@ -5,7 +5,7 @@ ## Purpose This document is a complete implementation specification for a new CCPP Framework code -generator (`ccpp-capgen-ng`). It supersedes both `ccpp-prebuild` and `ccpp-capgen`. An +generator (`ccpp-capgen`). It supersedes both `ccpp-prebuild` and `ccpp-capgen`. An implementer should be able to build the new generator from scratch using this document alone, supplemented by the real-world examples in `redesign_analysis.md`. @@ -71,7 +71,7 @@ omitted, the validator auto-discovers the Fortran source for each scheme table u `source_path` table-level property (Section 3.5): it looks for a `.F90` file with the same base name as the `.meta` file, in the directory given by `source_path`. -### 2.2 Code Generator (`ccpp_capgen_ng.py`) +### 2.2 Code Generator (`ccpp_capgen.py`) Parses metadata only. Assumes metadata correctly describes the Fortran source — performs no Fortran parsing. Generates all cap files and supporting modules. @@ -298,7 +298,7 @@ The generator has built-in semantic knowledge of these dimension standard names: | Standard name | Indexing semantic | |---|---| -| Any key of `SCALAR_INDEX_DIMS` (currently `number_of_instances`, `number_of_threads`) | Scalar extraction: substitute the paired index variable's local Fortran name (currently `instance_number`, `thread_number`). See `capgen-ng/metadata/registered_dimensions.py` for the full table and the contract. | +| Any key of `SCALAR_INDEX_DIMS` (currently `number_of_instances`, `number_of_threads`) | Scalar extraction: substitute the paired index variable's local Fortran name (currently `instance_number`, `thread_number`). See `capgen/metadata/registered_dimensions.py` for the full table and the contract. | | `horizontal_dimension` | **At scheme call sites**: always `horizontal_loop_begin:horizontal_loop_end` (using control variable local names), for all phases. **For suite-owned array allocation sizing**: local name of `horizontal_dimension` from the host `type=host` table (accessed via module USE, not the control variable). | | `vertical_*` | Slice: `1:` | @@ -434,7 +434,7 @@ entrypoints. All inputs derive from generator-time data already held in `SuiteResolution` plus the host/scheme metadata; no new metadata is required. The `_variables` vs `_host_data` split distinguishes the flat-leaf view (every DDT field that is actually consumed) from the DDT-collapsed view -(parent DDT instances), and excludes capgen-ng-generated control +(parent DDT instances), and excludes capgen-generated control variables from `_host_data` since the host owns those. --- @@ -719,7 +719,7 @@ dimension rules to each dimension in order: `number_of_threads` → `thread_number`) → scalar extraction using the paired index variable's local Fortran name. Only permitted on container DDT-instance variables, never on leaves (Rule 2; see - `capgen-ng/metadata/registered_dimensions.py`). + `capgen/metadata/registered_dimensions.py`). 2. **`horizontal_dimension`** → always substitute `horizontal_loop_begin:horizontal_loop_end` (using control variable local names) at scheme call sites. For suite-owned array allocation sizing, `horizontal_dimension` from the host `type=host` table is used directly. @@ -963,7 +963,7 @@ subsequent runs. ## 14. Constituent API -> **Status (2026-05-12).** The constituent API in capgen-ng has evolved past +> **Status (2026-05-12).** The constituent API in capgen has evolved past > the sketch below. The current implementation is: > > - One `ccpp_model_constituents_obj(:)` array (sized to @@ -982,7 +982,7 @@ subsequent runs. > API + examples). > - **Architecture review and proposed reforms**: `doc/constituents_overhaul.md` > (2026-05-12, meeting-quality discussion of original capgen vs -> capgen-ng vs cam-sima needs, bugs/flaws, class-A/B property +> capgen vs cam-sima needs, bugs/flaws, class-A/B property > classification, three proposals A/B/C). > > The historic text below is retained for context but does not describe @@ -1042,7 +1042,7 @@ All files are written to `--output-root`. ## 16. CLI Invocation ``` -ccpp_capgen_ng.py +ccpp_capgen.py --host-name --host-files --scheme-files @@ -1197,7 +1197,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` - **Group-state alloc idempotency** (matches suite-state alloc). - **Framework PR**: `ccpt_deallocate` ownership tracking via `framework_owns_me` flag. Backward-compatible. Landed in - capgen-ng's vendored framework copy; still needs upstream merge + capgen's vendored framework copy; still needs upstream merge to ccpp-framework + original ccpp-capgen. - **Identity unit conversions** no longer emit misleading "unit conversion: kind_phys to kind_phys" comment. @@ -1209,7 +1209,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` (``, ``, ``) rejected. - **Constituent resolver — host metadata wins**: hosts that declare framework-named std_names (`ccpp_constituents`, `index_of_`, ...) - short-circuit capgen-ng's auto-provisioning so legacy hosts (GFS, + short-circuit capgen's auto-provisioning so legacy hosts (GFS, SCM) keep using their own short local names (e.g. `ntcw`) without blowing Fortran's 63-char identifier limit. @@ -1217,7 +1217,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` - **`--legacy-mode` shim** — transient parse-time rewrite of legacy CCPP standard names (`horizontal_loop_extent` → - `horizontal_dimension`). Available on `ccpp_capgen_ng.py` and + `horizontal_dimension`). Available on `ccpp_capgen.py` and `ccpp_validator.py`; loud banner at startup. Isolated in `metadata/legacy_compat.py` and tagged `# legacy-compat:` for clean removal once scheme metadata has been migrated. @@ -1346,7 +1346,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` `metadata/dim_aliases.py`; touchpoints tagged `# dim-aliases:` for clean removal. Generator-only (the validator never reaches the canonicaliser). Required for CCPP-SCM 17p8 to build under - capgen-ng. + capgen. - **`--legacy-auto-clone-constituents` shim** — transient CLI flag that reinstates original ccpp-capgen's auto-clone-static-constituent registration path. Every `is_constituent` consumer scheme arg @@ -1357,7 +1357,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` parser (`default_value`, `min_value`, `water_species`, `mixing_ratio_type`). Synthesises `long_name` from std_name when missing; falls back `diag_name` to local_name; lifts `vertical_dim` - from the arg's dim list. Available on both `ccpp_capgen_ng.py` and + from the arg's dim list. Available on both `ccpp_capgen.py` and `ccpp_validator.py`. **Single-instance only** — aborts before parsing if the host declares `instance_number` + `number_of_instances`. Module @@ -1371,7 +1371,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` - **Unit tests**: 1426 passing (1438 with doctests; as of 2026-06-01). Run via `python unit-tests/run_tests.py [--doctest]`. - **End-to-end tests** (10 passing): `advection`, - `advection_auto_clone`, `capgen_ng`, `chunked_data`, `ddthost`, + `advection_auto_clone`, `capgen`, `chunked_data`, `ddthost`, `instances`, `instances_advection`, `nested_suite`, `opt_arg`, `var_compat`. SCM running against ccpp-physics continues to be the active driver — most of the landings since 2026-05-13 were @@ -1425,7 +1425,7 @@ See `MEMORY.md` (auto-memory index) and `project_implementation_status.md` OUTERMOST counter, not the innermost. None of the cam-sima schemes use this — revisit if a real scheme needs the innermost. - **Python linter / formatter pass** — pick `ruff` and apply across - `capgen-ng/`. + `capgen/`. - **Generated Fortran ↔ Codee formatter idempotency** — emitted `.F90` must round-trip cleanly through the project's Codee formatter. - **`fortran_to_metadata` developer utility** — bootstrap a `.meta` diff --git a/end-to-end-tests/CMakeLists.txt b/end-to-end-tests/CMakeLists.txt index 3e7232b9..3ea171a8 100644 --- a/end-to-end-tests/CMakeLists.txt +++ b/end-to-end-tests/CMakeLists.txt @@ -77,7 +77,7 @@ add_subdirectory(instances) #------------------------------------------------------------------------------ # Run most complex tests last -add_subdirectory(capgen_ng) +add_subdirectory(capgen) add_subdirectory(var_compat) add_subdirectory(suite_allocate) add_subdirectory(constituents_dim) diff --git a/end-to-end-tests/advection_auto_clone/cld_liq.meta b/end-to-end-tests/advection_auto_clone/cld_liq.meta index 7013aea8..a26e5600 100644 --- a/end-to-end-tests/advection_auto_clone/cld_liq.meta +++ b/end-to-end-tests/advection_auto_clone/cld_liq.meta @@ -112,7 +112,7 @@ dimensions = (horizontal_dimension, vertical_layer_dimension) type = real | kind = kind_phys # Advected species that needs to be promoted from suite. - # Note that in capgen-ng, intent 'out' is no longer permitted + # Note that in capgen, intent 'out' is no longer permitted intent = inout [ tcld] standard_name = minimum_temperature_for_cloud_liquid diff --git a/end-to-end-tests/capgen_ng/CMakeLists.txt b/end-to-end-tests/capgen/CMakeLists.txt similarity index 86% rename from end-to-end-tests/capgen_ng/CMakeLists.txt rename to end-to-end-tests/capgen/CMakeLists.txt index 04411444..4df45d1e 100644 --- a/end-to-end-tests/capgen_ng/CMakeLists.txt +++ b/end-to-end-tests/capgen/CMakeLists.txt @@ -39,7 +39,7 @@ ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} METADATA_FILES ${HOST_METADATA_FILES} TYPE "HOST") -# Run ccpp_capgen_ng +# Run ccpp_capgen ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} SCHEMEFILES ${SCHEME_METADATA_FILES} @@ -67,7 +67,7 @@ set(EXTRA_FILES ${CMAKE_CURRENT_SOURCE_DIR}/../utils/test_utils.F90 ) -add_executable(test_capgen_ng.x +add_executable(test_capgen.x ${EXTRA_FILES} ${CAPGEN_DEPENDENCIES} ${SCHEME_FORTRAN_FILES_FILTERED} @@ -75,25 +75,25 @@ add_executable(test_capgen_ng.x ${CAPGEN_FILES} test_capgen_host_integration.F90 ) -target_link_libraries(test_capgen_ng.x PRIVATE MPI::MPI_Fortran) +target_link_libraries(test_capgen.x PRIVATE MPI::MPI_Fortran) if(OPENMP) - target_link_libraries(test_capgen_ng.x PRIVATE OpenMP::OpenMP_Fortran) + target_link_libraries(test_capgen.x PRIVATE OpenMP::OpenMP_Fortran) endif() -set_target_properties(test_capgen_ng.x PROPERTIES LINKER_LANGUAGE Fortran) +set_target_properties(test_capgen.x PROPERTIES LINKER_LANGUAGE Fortran) # Add executable to be called with ctest -add_test(NAME test_capgen_ng_omp1 - COMMAND test_capgen_ng.x) +add_test(NAME test_capgen_omp1 + COMMAND test_capgen.x) -add_test(NAME test_capgen_ng_omp2 - COMMAND test_capgen_ng.x) +add_test(NAME test_capgen_omp2 + COMMAND test_capgen.x) -set_tests_properties(test_capgen_ng_omp1 +set_tests_properties(test_capgen_omp1 PROPERTIES ENVIRONMENT "OMP_NUM_THREADS=1" ) -set_tests_properties(test_capgen_ng_omp2 +set_tests_properties(test_capgen_omp2 PROPERTIES ENVIRONMENT "OMP_NUM_THREADS=2" ) diff --git a/end-to-end-tests/capgen_ng/README.md b/end-to-end-tests/capgen/README.md similarity index 100% rename from end-to-end-tests/capgen_ng/README.md rename to end-to-end-tests/capgen/README.md diff --git a/end-to-end-tests/capgen_ng/adjust/temp_kinds.F90 b/end-to-end-tests/capgen/adjust/temp_kinds.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/adjust/temp_kinds.F90 rename to end-to-end-tests/capgen/adjust/temp_kinds.F90 diff --git a/end-to-end-tests/capgen_ng/capgen_test_reports.py b/end-to-end-tests/capgen/capgen_test_reports.py similarity index 100% rename from end-to-end-tests/capgen_ng/capgen_test_reports.py rename to end-to-end-tests/capgen/capgen_test_reports.py diff --git a/end-to-end-tests/capgen_ng/ddt2.F90 b/end-to-end-tests/capgen/ddt2.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/ddt2.F90 rename to end-to-end-tests/capgen/ddt2.F90 diff --git a/end-to-end-tests/capgen_ng/ddt_suite.xml b/end-to-end-tests/capgen/ddt_suite.xml similarity index 100% rename from end-to-end-tests/capgen_ng/ddt_suite.xml rename to end-to-end-tests/capgen/ddt_suite.xml diff --git a/end-to-end-tests/capgen_ng/environ_conditions.meta b/end-to-end-tests/capgen/environ_conditions.meta similarity index 100% rename from end-to-end-tests/capgen_ng/environ_conditions.meta rename to end-to-end-tests/capgen/environ_conditions.meta diff --git a/end-to-end-tests/capgen_ng/make_ddt.F90 b/end-to-end-tests/capgen/make_ddt.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/make_ddt.F90 rename to end-to-end-tests/capgen/make_ddt.F90 diff --git a/end-to-end-tests/capgen_ng/make_ddt.meta b/end-to-end-tests/capgen/make_ddt.meta similarity index 100% rename from end-to-end-tests/capgen_ng/make_ddt.meta rename to end-to-end-tests/capgen/make_ddt.meta diff --git a/end-to-end-tests/capgen_ng/setup_coeffs.F90 b/end-to-end-tests/capgen/setup_coeffs.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/setup_coeffs.F90 rename to end-to-end-tests/capgen/setup_coeffs.F90 diff --git a/end-to-end-tests/capgen_ng/setup_coeffs.meta b/end-to-end-tests/capgen/setup_coeffs.meta similarity index 100% rename from end-to-end-tests/capgen_ng/setup_coeffs.meta rename to end-to-end-tests/capgen/setup_coeffs.meta diff --git a/end-to-end-tests/capgen_ng/source_dir1/environ_conditions.F90 b/end-to-end-tests/capgen/source_dir1/environ_conditions.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/source_dir1/environ_conditions.F90 rename to end-to-end-tests/capgen/source_dir1/environ_conditions.F90 diff --git a/end-to-end-tests/capgen_ng/source_dir2/temp_set.F90 b/end-to-end-tests/capgen/source_dir2/temp_set.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/source_dir2/temp_set.F90 rename to end-to-end-tests/capgen/source_dir2/temp_set.F90 diff --git a/end-to-end-tests/capgen_ng/temp_adjust.F90 b/end-to-end-tests/capgen/temp_adjust.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/temp_adjust.F90 rename to end-to-end-tests/capgen/temp_adjust.F90 diff --git a/end-to-end-tests/capgen_ng/temp_adjust.meta b/end-to-end-tests/capgen/temp_adjust.meta similarity index 100% rename from end-to-end-tests/capgen_ng/temp_adjust.meta rename to end-to-end-tests/capgen/temp_adjust.meta diff --git a/end-to-end-tests/capgen_ng/temp_calc_adjust.F90 b/end-to-end-tests/capgen/temp_calc_adjust.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/temp_calc_adjust.F90 rename to end-to-end-tests/capgen/temp_calc_adjust.F90 diff --git a/end-to-end-tests/capgen_ng/temp_calc_adjust.meta b/end-to-end-tests/capgen/temp_calc_adjust.meta similarity index 100% rename from end-to-end-tests/capgen_ng/temp_calc_adjust.meta rename to end-to-end-tests/capgen/temp_calc_adjust.meta diff --git a/end-to-end-tests/capgen_ng/temp_set.meta b/end-to-end-tests/capgen/temp_set.meta similarity index 100% rename from end-to-end-tests/capgen_ng/temp_set.meta rename to end-to-end-tests/capgen/temp_set.meta diff --git a/end-to-end-tests/capgen_ng/temp_suite.xml b/end-to-end-tests/capgen/temp_suite.xml similarity index 100% rename from end-to-end-tests/capgen_ng/temp_suite.xml rename to end-to-end-tests/capgen/temp_suite.xml diff --git a/end-to-end-tests/capgen_ng/test_capgen_host_integration.F90 b/end-to-end-tests/capgen/test_capgen_host_integration.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/test_capgen_host_integration.F90 rename to end-to-end-tests/capgen/test_capgen_host_integration.F90 diff --git a/end-to-end-tests/capgen_ng/test_host.F90 b/end-to-end-tests/capgen/test_host.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/test_host.F90 rename to end-to-end-tests/capgen/test_host.F90 diff --git a/end-to-end-tests/capgen_ng/test_host.meta b/end-to-end-tests/capgen/test_host.meta similarity index 100% rename from end-to-end-tests/capgen_ng/test_host.meta rename to end-to-end-tests/capgen/test_host.meta diff --git a/end-to-end-tests/capgen_ng/test_host_data.F90 b/end-to-end-tests/capgen/test_host_data.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/test_host_data.F90 rename to end-to-end-tests/capgen/test_host_data.F90 diff --git a/end-to-end-tests/capgen_ng/test_host_data.meta b/end-to-end-tests/capgen/test_host_data.meta similarity index 100% rename from end-to-end-tests/capgen_ng/test_host_data.meta rename to end-to-end-tests/capgen/test_host_data.meta diff --git a/end-to-end-tests/capgen_ng/test_host_mod.F90 b/end-to-end-tests/capgen/test_host_mod.F90 similarity index 100% rename from end-to-end-tests/capgen_ng/test_host_mod.F90 rename to end-to-end-tests/capgen/test_host_mod.F90 diff --git a/end-to-end-tests/capgen_ng/test_host_mod.meta b/end-to-end-tests/capgen/test_host_mod.meta similarity index 100% rename from end-to-end-tests/capgen_ng/test_host_mod.meta rename to end-to-end-tests/capgen/test_host_mod.meta diff --git a/end-to-end-tests/chunked_data/CMakeLists.txt b/end-to-end-tests/chunked_data/CMakeLists.txt index 85124125..2bfef4f4 100644 --- a/end-to-end-tests/chunked_data/CMakeLists.txt +++ b/end-to-end-tests/chunked_data/CMakeLists.txt @@ -28,7 +28,7 @@ ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} METADATA_FILES ${HOST_METADATA_FILES} TYPE "HOST") -# Run ccpp_capgen_ng +# Run ccpp_capgen ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} SCHEMEFILES ${SCHEME_METADATA_FILES} diff --git a/end-to-end-tests/cmake/ccpp_capgen.cmake b/end-to-end-tests/cmake/ccpp_capgen.cmake index 8647a851..a6b73899 100644 --- a/end-to-end-tests/cmake/ccpp_capgen.cmake +++ b/end-to-end-tests/cmake/ccpp_capgen.cmake @@ -21,7 +21,7 @@ function(ccpp_validator) cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN}) # Error if script file not found. - set(CCPP_VALIDATOR_CMD_LIST "${CMAKE_SOURCE_DIR}/../capgen-ng/ccpp_validator.py") + set(CCPP_VALIDATOR_CMD_LIST "${CMAKE_SOURCE_DIR}/../capgen/ccpp_validator.py") if(NOT EXISTS ${CCPP_VALIDATOR_CMD_LIST}) message(FATAL_ERROR "function(ccpp_validator): Could not find ccpp_validator.py. Looked for ${CCPP_VALIDATOR_CMD_LIST}.") endif() @@ -83,7 +83,7 @@ function(ccpp_validator) endfunction() -# CMake wrapper for ccpp_capgen_ng.py +# CMake wrapper for ccpp_capgen.py # # TRACE - ON/OFF (Default: OFF) - Add --trace flag to capgen call # HOST_NAME - String name of host (drives _ccpp_cap.F90 filename @@ -95,8 +95,8 @@ endfunction() # SUITES - CMake list of suite xml files # KIND_SPECS - Comma-separated kind mappings, e.g. "kind_phys=REAL32" or # "kind_phys=my_mod:kind_r4,kind_dyn=REAL64". Each pair is -# forwarded as `--kind-type ` to capgen-ng (see the -# capgen-ng docstring for the `=[:]` +# forwarded as `--kind-type ` to capgen (see the +# capgen docstring for the `=[:]` # grammar; bare ISO specs default to iso_fortran_env). function(ccpp_capgen) set(optionalArgs TRACE) @@ -106,9 +106,9 @@ function(ccpp_capgen) cmake_parse_arguments(arg "${optionalArgs}" "${oneValueArgs}" "${multi_value_keywords}" ${ARGN}) # Error if script file not found. - set(CCPP_CAPGEN_CMD_LIST "${CMAKE_SOURCE_DIR}/../capgen-ng/ccpp_capgen_ng.py") + set(CCPP_CAPGEN_CMD_LIST "${CMAKE_SOURCE_DIR}/../capgen/ccpp_capgen.py") if(NOT EXISTS ${CCPP_CAPGEN_CMD_LIST}) - message(FATAL_ERROR "function(ccpp_capgen): Could not find ccpp_capgen_ng.py. Looked for ${CCPP_CAPGEN_CMD_LIST}.") + message(FATAL_ERROR "function(ccpp_capgen): Could not find ccpp_capgen.py. Looked for ${CCPP_CAPGEN_CMD_LIST}.") endif() # Interpret parsed arguments @@ -150,7 +150,7 @@ function(ccpp_capgen) if(DEFINED arg_KIND_SPECS) # Accept either a comma-separated string ("kind_phys=REAL64,kind_dyn=REAL32") # or a CMake list of pairs. Each pair becomes a separate - # `--kind-type ` argv pair so capgen-ng's argparse sees one + # `--kind-type ` argv pair so capgen's argparse sees one # `--kind-type` per pair (the flag is `action='append'`). string(REPLACE "," ";" KIND_SPEC_LIST "${arg_KIND_SPECS}") foreach(pair IN LISTS KIND_SPEC_LIST) @@ -194,7 +194,7 @@ function(ccpp_datafile) set(mandatoryArgs DATATABLE REPORT_NAME) cmake_parse_arguments(arg "" "${mandatoryArgs}" "" ${ARGN}) - set(CCPP_DATAFILE_CMD "${CMAKE_SOURCE_DIR}/../capgen-ng/ccpp_datafile.py") + set(CCPP_DATAFILE_CMD "${CMAKE_SOURCE_DIR}/../capgen/ccpp_datafile.py") if(NOT EXISTS ${CCPP_DATAFILE_CMD}) message(FATAL_ERROR "function(ccpp_datafile): Could not find ccpp_datafile.py. Looked for ${CCPP_DATAFILE_CMD}.") diff --git a/end-to-end-tests/constituents_dim/CMakeLists.txt b/end-to-end-tests/constituents_dim/CMakeLists.txt index b05b5b98..f60c9b2f 100644 --- a/end-to-end-tests/constituents_dim/CMakeLists.txt +++ b/end-to-end-tests/constituents_dim/CMakeLists.txt @@ -28,7 +28,7 @@ ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} METADATA_FILES ${HOST_METADATA_FILES} TYPE "HOST") -# Run ccpp_capgen_ng +# Run ccpp_capgen ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} SCHEMEFILES ${SCHEME_METADATA_FILES} diff --git a/end-to-end-tests/constituents_dim/register_consts.F90 b/end-to-end-tests/constituents_dim/register_consts.F90 index 84a9a4d9..cfcba24a 100644 --- a/end-to-end-tests/constituents_dim/register_consts.F90 +++ b/end-to-end-tests/constituents_dim/register_consts.F90 @@ -1,6 +1,6 @@ !>\file register_consts.F90 !! Register-phase scheme that registers three dynamic constituents. Declaring a -!! ccpp_constituent_properties_t(:) argument is what activates capgen-ng's +!! ccpp_constituent_properties_t(:) argument is what activates capgen's !! constituent machinery, giving the suite a meaningful !! number_of_ccpp_constituents (= 3 here) for the rest of this test. diff --git a/end-to-end-tests/ddthost/CMakeLists.txt b/end-to-end-tests/ddthost/CMakeLists.txt index aaa10fdc..d8f612b0 100644 --- a/end-to-end-tests/ddthost/CMakeLists.txt +++ b/end-to-end-tests/ddthost/CMakeLists.txt @@ -28,7 +28,7 @@ ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} METADATA_FILES ${HOST_METADATA_FILES} TYPE "HOST") -# Run ccpp_capgen_ng +# Run ccpp_capgen ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} SCHEMEFILES ${SCHEME_METADATA_FILES} diff --git a/end-to-end-tests/instances/CMakeLists.txt b/end-to-end-tests/instances/CMakeLists.txt index f64ebf24..c4e6fbbd 100644 --- a/end-to-end-tests/instances/CMakeLists.txt +++ b/end-to-end-tests/instances/CMakeLists.txt @@ -28,7 +28,7 @@ ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} METADATA_FILES ${HOST_METADATA_FILES} TYPE "HOST") -# Run ccpp_capgen_ng +# Run ccpp_capgen ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} SCHEMEFILES ${SCHEME_METADATA_FILES} diff --git a/end-to-end-tests/nested_suite/CMakeLists.txt b/end-to-end-tests/nested_suite/CMakeLists.txt index bd449ba7..5903fcb9 100644 --- a/end-to-end-tests/nested_suite/CMakeLists.txt +++ b/end-to-end-tests/nested_suite/CMakeLists.txt @@ -28,7 +28,7 @@ ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} METADATA_FILES ${HOST_METADATA_FILES} TYPE "HOST") -# Run ccpp_capgen_ng +# Run ccpp_capgen ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} SCHEMEFILES ${SCHEME_METADATA_FILES} diff --git a/end-to-end-tests/opt_arg/CMakeLists.txt b/end-to-end-tests/opt_arg/CMakeLists.txt index 041f9bbb..d791fa28 100644 --- a/end-to-end-tests/opt_arg/CMakeLists.txt +++ b/end-to-end-tests/opt_arg/CMakeLists.txt @@ -28,7 +28,7 @@ ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} METADATA_FILES ${HOST_METADATA_FILES} TYPE "HOST") -# Run ccpp_capgen_ng. Override kind_phys to REAL32 so the whole test +# Run ccpp_capgen. Override kind_phys to REAL32 so the whole test # runs in single precision; exercises the --kind-type plumbing. ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} diff --git a/end-to-end-tests/opt_arg/main.F90 b/end-to-end-tests/opt_arg/main.F90 index 25701be4..e0643513 100644 --- a/end-to-end-tests/opt_arg/main.F90 +++ b/end-to-end-tests/opt_arg/main.F90 @@ -28,6 +28,10 @@ program test_opt_arg flag_for_opt_arg = .true. allocate(opt_arg(nx)) allocate(opt_arg_2(nx)) + ! capgen does not default-initialize host data; the host must. Zero these + ! so the post-ccpp_init checks below test real wiring, not stale memory. + opt_arg = 0 + opt_arg_2 = 0 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! CCPP register step ! @@ -55,7 +59,7 @@ program test_opt_arg write(output_unit, '(a)') "After ccpp_init: check std_arg(:)==1, opt_arg(:)==0, opt_arg_2(:)==0" if (.not. all(std_arg == 1)) write(error_unit, '(a,3i3)') "Error after ccpp_init: std_arg=", std_arg if (.not. all(opt_arg == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_init: opt_arg=", opt_arg - if (.not. all(opt_arg_2 == 0)) write(error_unit, '(a,3i3)') "Error after ccpp_init: opt_arg_2=", opt_arg_2 + if (.not. all(opt_arg_2 == 0)) write(error_unit, '(a,3es13.5)') "Error after ccpp_init: opt_arg_2=", opt_arg_2 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! CCPP physics init step ! diff --git a/end-to-end-tests/suite_allocate/CMakeLists.txt b/end-to-end-tests/suite_allocate/CMakeLists.txt index 76a11d4b..3c887fb7 100644 --- a/end-to-end-tests/suite_allocate/CMakeLists.txt +++ b/end-to-end-tests/suite_allocate/CMakeLists.txt @@ -28,7 +28,7 @@ ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} METADATA_FILES ${HOST_METADATA_FILES} TYPE "HOST") -# Run ccpp_capgen_ng +# Run ccpp_capgen ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} SCHEMEFILES ${SCHEME_METADATA_FILES} diff --git a/end-to-end-tests/suite_allocate/README.md b/end-to-end-tests/suite_allocate/README.md index 7add35d1..b1601d11 100644 --- a/end-to-end-tests/suite_allocate/README.md +++ b/end-to-end-tests/suite_allocate/README.md @@ -5,7 +5,7 @@ suite-owned, scheme-allocated variable (allocatable = True). scratch_workspace_field is produced by make_workspace (intent=out, allocatable = True) and consumed by use_workspace. No host table declares -it, so capgen-ng promotes it to a suite-owned variable stored in +it, so capgen promotes it to a suite-owned variable stored in ccpp__data. Crucially, its dimension workspace_dimension is also suite-owned: it is set @@ -28,6 +28,6 @@ use-after-free in the scheme-allocates / suite-frees ownership split fails the test. This is distinct from: -- capgen_ng — suite-owned vars allocated at init from a register-set dim +- capgen — suite-owned vars allocated at init from a register-set dim (non-allocatable path), and a host-owned allocatable var (model_times). - nested_suite / var_compat — standalone DDTs with module_name. diff --git a/end-to-end-tests/suite_allocate/make_workspace.F90 b/end-to-end-tests/suite_allocate/make_workspace.F90 index ab371995..bf1d3c68 100644 --- a/end-to-end-tests/suite_allocate/make_workspace.F90 +++ b/end-to-end-tests/suite_allocate/make_workspace.F90 @@ -1,6 +1,6 @@ !>\file make_workspace.F90 !! Producer scheme: allocates and fills a suite-owned scratch workspace. -!! The workspace standard name is not provided by the host, so capgen-ng +!! The workspace standard name is not provided by the host, so capgen !! promotes it to a suite-owned, scheme-allocated (allocatable=True) variable !! stored in ccpp__data. suite_data_init_fields skips its allocation; !! this scheme owns it; final_fields frees it. diff --git a/end-to-end-tests/suite_allocate/use_workspace.F90 b/end-to-end-tests/suite_allocate/use_workspace.F90 index 25e2fe81..c2d66e13 100644 --- a/end-to-end-tests/suite_allocate/use_workspace.F90 +++ b/end-to-end-tests/suite_allocate/use_workspace.F90 @@ -2,7 +2,7 @@ !! Consumer scheme: reads the suite-owned scratch workspace allocated by !! make_workspace and reduces it into a host-owned scalar. Receiving the !! suite-owned allocatable component through a plain (non-allocatable) dummy -!! exercises capgen-ng passing the whole allocated component to a consumer. +!! exercises capgen passing the whole allocated component to a consumer. module use_workspace diff --git a/end-to-end-tests/var_compat/CMakeLists.txt b/end-to-end-tests/var_compat/CMakeLists.txt index 5e006e13..18ef84c6 100644 --- a/end-to-end-tests/var_compat/CMakeLists.txt +++ b/end-to-end-tests/var_compat/CMakeLists.txt @@ -28,7 +28,7 @@ ccpp_validator(VERBOSITY ${CCPP_VERBOSITY} METADATA_FILES ${HOST_METADATA_FILES} TYPE "HOST") -# Run ccpp_capgen_ng +# Run ccpp_capgen ccpp_capgen(VERBOSITY ${CCPP_VERBOSITY} HOSTFILES ${HOST_METADATA_FILES} SCHEMEFILES ${SCHEME_METADATA_FILES} diff --git a/unit-tests/__init__.py b/unit-tests/__init__.py index 35c2ae0f..4f9a36ed 100644 --- a/unit-tests/__init__.py +++ b/unit-tests/__init__.py @@ -1 +1 @@ -"""Unit and integration tests for ccpp-capgen-ng.""" +"""Unit and integration tests for ccpp-capgen.""" diff --git a/unit-tests/conftest.py b/unit-tests/conftest.py index 638eb4eb..4413cf36 100644 --- a/unit-tests/conftest.py +++ b/unit-tests/conftest.py @@ -1,12 +1,12 @@ -"""pytest configuration for capgen-ng unit tests. +"""pytest configuration for capgen unit tests. -Adds the capgen-ng package root to sys.path so that ``import metadata`` and +Adds the capgen package root to sys.path so that ``import metadata`` and ``import generator`` work regardless of where pytest is invoked from. Layout assumed:: / - capgen-ng/ <-- the package being tested + capgen/ <-- the package being tested unit-tests/ <-- this file's parent directory conftest.py test_*.py @@ -22,8 +22,8 @@ _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) # repository root (parent of unit-tests/) _REPO_ROOT = os.path.dirname(_TESTS_DIR) -# capgen-ng/ package directory (sibling of unit-tests/) -_CAPGEN_DIR = os.path.join(_REPO_ROOT, 'capgen-ng') +# capgen/ package directory (sibling of unit-tests/) +_CAPGEN_DIR = os.path.join(_REPO_ROOT, 'capgen') for _path in (_TESTS_DIR, _CAPGEN_DIR, _REPO_ROOT): if _path not in sys.path: diff --git a/unit-tests/run_tests.py b/unit-tests/run_tests.py index a4769948..12ad7e02 100644 --- a/unit-tests/run_tests.py +++ b/unit-tests/run_tests.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Convenience script to run all capgen-ng unit tests. +"""Convenience script to run all capgen unit tests. Usage:: @@ -18,7 +18,7 @@ # ---- path setup ------------------------------------------------------------ _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) _REPO_ROOT = os.path.dirname(_TESTS_DIR) -_CAPGEN_DIR = os.path.join(_REPO_ROOT, 'capgen-ng') +_CAPGEN_DIR = os.path.join(_REPO_ROOT, 'capgen') for _p in (_CAPGEN_DIR, _REPO_ROOT): if _p not in sys.path: diff --git a/unit-tests/sample_files/scheme_consume_constituent.meta b/unit-tests/sample_files/scheme_consume_constituent.meta index ab6b216b..278e66ad 100644 --- a/unit-tests/sample_files/scheme_consume_constituent.meta +++ b/unit-tests/sample_files/scheme_consume_constituent.meta @@ -1,6 +1,6 @@ # cam-sima-style scheme that consumes a base constituent and produces # a tendency. No host or earlier-scheme provider for either — both -# are auto-resolved by capgen-ng via the ccpp_constituents / +# are auto-resolved by capgen via the ccpp_constituents / # ccpp_constituent_tendencies arrays owned by the suite cap. [ccpp-table-properties] diff --git a/unit-tests/sample_files/scheme_module_name_override.meta b/unit-tests/sample_files/scheme_module_name_override.meta index a2ba5935..6aa16208 100644 --- a/unit-tests/sample_files/scheme_module_name_override.meta +++ b/unit-tests/sample_files/scheme_module_name_override.meta @@ -1,5 +1,5 @@ # Scheme metadata that declares ``module_name`` in [ccpp-table-properties] -# distinct from the table ``name``. Used to verify capgen-ng emits +# distinct from the table ``name``. Used to verify capgen emits # ``use mod_alt_name, only: ...`` (the explicit module name) rather than # ``use scheme_alt_name``. diff --git a/unit-tests/test_auto_clone_constituents.py b/unit-tests/test_auto_clone_constituents.py index b9c586f4..5c378d2a 100644 --- a/unit-tests/test_auto_clone_constituents.py +++ b/unit-tests/test_auto_clone_constituents.py @@ -16,7 +16,7 @@ import unittest _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) -_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen-ng') +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen') if _CAPGEN_DIR not in sys.path: sys.path.insert(0, _CAPGEN_DIR) @@ -512,7 +512,7 @@ def test_explicit_long_name_wins(self): def test_long_name_synthesised_when_missing(self): # The _scheme_var helper sets long_name='water vapor'; override - # to a no-op so the helper sees an empty long_name. capgen-ng + # to a no-op so the helper sees an empty long_name. capgen # then synthesises from std_name: 'water_vapor_specific_humidity' # → 'Water vapor specific humidity'. from metadata.metadata_table import MetaVar diff --git a/unit-tests/test_ccpp_datafile.py b/unit-tests/test_ccpp_datafile.py index 70363bdf..fe03ae98 100644 --- a/unit-tests/test_ccpp_datafile.py +++ b/unit-tests/test_ccpp_datafile.py @@ -14,7 +14,7 @@ import unittest _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) -_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen-ng') +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen') for _p in (_CAPGEN_DIR, _TESTS_DIR): if _p not in sys.path: sys.path.insert(0, _p) @@ -154,7 +154,7 @@ def test_inspection_files_empty_when_none_given(self): class TestDatatableReportSchemeActions(_DTBase): def test_process_list_empty(self): - # capgen-ng does not emit attrs. + # capgen does not emit attrs. out = datatable_report(self._datatable, DatatableReport('process_list'), ',') self.assertEqual(out, '') diff --git a/unit-tests/test_control_validation.py b/unit-tests/test_control_validation.py index 3d381089..ba2641b1 100644 --- a/unit-tests/test_control_validation.py +++ b/unit-tests/test_control_validation.py @@ -19,11 +19,11 @@ _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) _REPO_ROOT = os.path.dirname(_TESTS_DIR) -_CAPGEN_DIR = os.path.join(_REPO_ROOT, 'capgen-ng') +_CAPGEN_DIR = os.path.join(_REPO_ROOT, 'capgen') if _CAPGEN_DIR not in sys.path: sys.path.insert(0, _CAPGEN_DIR) -from ccpp_capgen_ng import _validate_required_control_vars, _check_no_loop_dimensions +from ccpp_capgen import _validate_required_control_vars, _check_no_loop_dimensions from metadata.parse_tools import CCPPError from metadata.variable_resolver import build_flat_host_dict, HostVarEntry from metadata.metadata_table import parse_metadata_file @@ -380,7 +380,7 @@ def test_loop_begin_error_names_variable(self): def test_all_three_forbidden_names_detected(self): """Verify all three names are in the forbidden set.""" - from ccpp_capgen_ng import _FORBIDDEN_DIMENSION_NAMES + from ccpp_capgen import _FORBIDDEN_DIMENSION_NAMES self.assertIn('horizontal_loop_extent', _FORBIDDEN_DIMENSION_NAMES) self.assertIn('horizontal_loop_begin', _FORBIDDEN_DIMENSION_NAMES) self.assertIn('horizontal_loop_end', _FORBIDDEN_DIMENSION_NAMES) diff --git a/unit-tests/test_dim_aliases.py b/unit-tests/test_dim_aliases.py index e2fb1159..5c0f50ae 100644 --- a/unit-tests/test_dim_aliases.py +++ b/unit-tests/test_dim_aliases.py @@ -16,7 +16,7 @@ import unittest _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) -_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen-ng') +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen') if _CAPGEN_DIR not in sys.path: sys.path.insert(0, _CAPGEN_DIR) diff --git a/unit-tests/test_host_cap.py b/unit-tests/test_host_cap.py index e0ca78e9..b82848df 100644 --- a/unit-tests/test_host_cap.py +++ b/unit-tests/test_host_cap.py @@ -621,7 +621,7 @@ def setUp(self): # Use the host-only dict (no ccpp_model_constituents_t DDT # instance) so the host-wins rule doesn't fire and the scheme's - # ccpp_constituents arg routes through capgen-ng's + # ccpp_constituents arg routes through capgen's # auto-provisioning path — that's the code path that surfaces # number_of_ccpp_constituents as an input via used_const_dim_std_names. self.hd = _load_full_host_dict() @@ -1044,7 +1044,7 @@ def test_signature_includes_optional_filters(self): ) def test_no_struct_elements_arg(self): - # struct_elements is intentionally dropped (was a no-op in capgen-ng). + # struct_elements is intentionally dropped (was a no-op in capgen). self.assertNotIn('struct_elements', self.text) def test_errmsg_is_assumed_length(self): diff --git a/unit-tests/test_host_constituents.py b/unit-tests/test_host_constituents.py index 25d2513b..0511c692 100644 --- a/unit-tests/test_host_constituents.py +++ b/unit-tests/test_host_constituents.py @@ -480,7 +480,7 @@ def _render_long_name_constituent(): host_tbls = parse_metadata_file(os.path.join(samples, 'host_with_constituents.meta')) ctrl_tbls = parse_metadata_file(os.path.join(samples, 'control_full.meta')) fw_meta = os.path.join( - os.path.dirname(here), 'capgen-ng', 'src', 'ccpp_constituent_prop_mod.meta', + os.path.dirname(here), 'capgen', 'src', 'ccpp_constituent_prop_mod.meta', ) ddt_tbls = parse_metadata_file(fw_meta) if os.path.isfile(fw_meta) else [] hd = build_flat_host_dict(host_tbls, ctrl_tbls, ddt_tbls) diff --git a/unit-tests/test_integration.py b/unit-tests/test_integration.py index d5451739..eecd7463 100644 --- a/unit-tests/test_integration.py +++ b/unit-tests/test_integration.py @@ -1,4 +1,4 @@ -"""End-to-end integration tests for capgen_ng.capgen(). +"""End-to-end integration tests for capgen.capgen(). These tests invoke the full pipeline — metadata loading, variable resolution, and file generation — and verify that all expected output files are produced @@ -12,7 +12,7 @@ import unittest import xml.etree.ElementTree as ET -from ccpp_capgen_ng import capgen +from ccpp_capgen import capgen _TESTS_DIR = os.path.dirname(__file__) _SAMPLES_DIR = os.path.join(_TESTS_DIR, 'sample_files') @@ -295,13 +295,13 @@ def test_framework_paths_absolute_and_existing(self): self.assertTrue(os.path.isfile(path), 'framework file missing: ' + path) - def test_framework_paths_resolve_under_capgen_ng_src(self): - """Every framework F90 listed must resolve under capgen-ng/src/. - Capgen-ng ships self-contained — no parent-dir fallback. If any - framework file lands outside capgen-ng/src/ this test fails so - downstream consumers (vendoring just capgen-ng/) don't silently + def test_framework_paths_resolve_under_capgen_src(self): + """Every framework F90 listed must resolve under capgen/src/. + Capgen ships self-contained — no parent-dir fallback. If any + framework file lands outside capgen/src/ this test fails so + downstream consumers (vendoring just capgen/) don't silently miss a required dependency.""" - from ccpp_capgen_ng import _FRAMEWORK_SRC_DIR + from ccpp_capgen import _FRAMEWORK_SRC_DIR framework_names = { 'ccpp_constituent_prop_mod.F90', 'ccpp_hashable.F90', @@ -314,34 +314,34 @@ def test_framework_paths_resolve_under_capgen_ng_src(self): if os.path.basename(path) in framework_names: self.assertEqual( os.path.abspath(os.path.dirname(path)), canonical, - 'framework F90 outside capgen-ng/src/: ' + path, + 'framework F90 outside capgen/src/: ' + path, ) class TestResolveFrameworkF90FilesMissingRaises(unittest.TestCase): """``_resolve_framework_f90_files`` raises CCPPError listing the missing file(s) when a required framework F90 is not present under - capgen-ng/src/. Catches deployment errors immediately instead of + capgen/src/. Catches deployment errors immediately instead of leaving the host build to fail with an opaque "Cannot open module file" message at compile time.""" def test_missing_file_raises_with_actionable_message(self): - import ccpp_capgen_ng + import ccpp_capgen from metadata.parse_tools import CCPPError # Append a never-vendored sentinel to the framework-F90 list, # then restore on teardown via a try/finally so we don't leak # state into other tests. - original = list(ccpp_capgen_ng._FRAMEWORK_F90_FILES) - ccpp_capgen_ng._FRAMEWORK_F90_FILES.append('definitely_missing.F90') + original = list(ccpp_capgen._FRAMEWORK_F90_FILES) + ccpp_capgen._FRAMEWORK_F90_FILES.append('definitely_missing.F90') try: with self.assertRaises(CCPPError) as cm: - ccpp_capgen_ng._resolve_framework_f90_files() + ccpp_capgen._resolve_framework_f90_files() finally: - ccpp_capgen_ng._FRAMEWORK_F90_FILES[:] = original + ccpp_capgen._FRAMEWORK_F90_FILES[:] = original msg = str(cm.exception) # Names the missing file, the search dir, and what to do. self.assertIn('definitely_missing.F90', msg) - self.assertIn(ccpp_capgen_ng._FRAMEWORK_SRC_DIR, msg) + self.assertIn(ccpp_capgen._FRAMEWORK_SRC_DIR, msg) self.assertIn('Vendor', msg) @@ -1208,7 +1208,7 @@ class TestUnusedSchemeDependenciesFiltered(unittest.TestCase): """Scheme metadata files supplied on the CLI but not referenced by any loaded suite must not contribute to datatable.xml's . Host build systems often pass the full physics - metadata catalog and rely on capgen-ng to narrow the compile set. + metadata catalog and rely on capgen to narrow the compile set. """ _USED_DEP = '/tmp/used_phys/used_dep.F90' @@ -1408,7 +1408,7 @@ class TestDdtDependenciesInSchemeMetaPreserved(unittest.TestCase): The DDT block's ``dependencies = …`` must reach datatable.xml even though the table name ('vmr_type', not the scheme name) won't match the used-schemes set. Regression for the bug that broke the - end-to-end-tests/capgen_ng test where ddt2.F90 went missing because + end-to-end-tests/capgen test where ddt2.F90 went missing because the DDT's deps were filtered out alongside actual scheme deps.""" _DDT_DEP = '/tmp/ddt_phys/inner_ddt.F90' @@ -1969,7 +1969,7 @@ def test_datatable_has_chunked_data_suite(self): # --------------------------------------------------------------------------- def _run_interstitial(tmpdir): - from ccpp_capgen_ng import capgen + from ccpp_capgen import capgen capgen( host_name='test_host', host_files=[_sf('host_full.meta'), _sf('control_full.meta')], diff --git a/unit-tests/test_legacy_compat.py b/unit-tests/test_legacy_compat.py index 6253d74b..1d154099 100644 --- a/unit-tests/test_legacy_compat.py +++ b/unit-tests/test_legacy_compat.py @@ -15,7 +15,7 @@ import unittest _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) -_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen-ng') +_CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen') if _CAPGEN_DIR not in sys.path: sys.path.insert(0, _CAPGEN_DIR) @@ -130,7 +130,7 @@ def test_horizontal_loop_extent_rewritten(self): def test_number_of_openmp_threads_rewritten(self): """Legacy CCPP-physics hosts (and SCM 17p8) size per-thread DDT - containers by ``number_of_openmp_threads``. The capgen-ng + containers by ``number_of_openmp_threads``. The capgen convention is ``number_of_threads`` (matching the ``thread_number`` control variable name). Legacy mode rewrites both as a standard_name attribute AND as a dimension token.""" diff --git a/unit-tests/test_metadata_table.py b/unit-tests/test_metadata_table.py index fe5e282b..ea69c3d1 100644 --- a/unit-tests/test_metadata_table.py +++ b/unit-tests/test_metadata_table.py @@ -4,11 +4,11 @@ Run with:: - python -m pytest capgen-ng/tests/test_metadata_table.py -v + python -m pytest capgen/tests/test_metadata_table.py -v or:: - python -m unittest capgen-ng.tests.test_metadata_table + python -m unittest capgen.tests.test_metadata_table All test methods follow the ``test_`` naming convention and are documented inline to explain both what is being tested and *why* it matters @@ -23,7 +23,7 @@ # ---- locate the package root ----------------------------------------------- _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) -_PKG_ROOT = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen-ng') +_PKG_ROOT = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen') if _PKG_ROOT not in sys.path: sys.path.insert(0, _PKG_ROOT) @@ -1610,13 +1610,13 @@ def test_dedup_across_phases(self): ######################################################################## class TestCLIHelpers(unittest.TestCase): - """Tests for CLI utility functions in ccpp_capgen_ng.""" + """Tests for CLI utility functions in ccpp_capgen.""" def setUp(self): import importlib import importlib.util - script = os.path.join(_PKG_ROOT, 'ccpp_capgen_ng.py') - spec = importlib.util.spec_from_file_location('ccpp_capgen_ng', script) + script = os.path.join(_PKG_ROOT, 'ccpp_capgen.py') + spec = importlib.util.spec_from_file_location('ccpp_capgen', script) self.mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(self.mod) @@ -1764,7 +1764,7 @@ def test_merge_cli_and_metadata_conflict_raises(self): def test_merge_then_default_kind_phys_injected_when_neither_provides(self): """Default kind_phys is injected after the merge when neither side declares it.""" import logging - log = logging.getLogger('ccpp_capgen_ng_test') + log = logging.getLogger('ccpp_capgen_test') merged = self.mod._merge_cli_and_metadata_kinds({}, {}) merged = self.mod._ensure_kind_phys_default(merged, log) self.assertEqual( @@ -1774,7 +1774,7 @@ def test_merge_then_default_kind_phys_injected_when_neither_provides(self): def test_metadata_kind_phys_suppresses_default(self): """Metadata declaring kind_phys keeps the default from being injected.""" import logging - log = logging.getLogger('ccpp_capgen_ng_test') + log = logging.getLogger('ccpp_capgen_test') meta = {'kind_phys': ('host_kinds', 'kind_r8')} merged = self.mod._merge_cli_and_metadata_kinds({}, meta) merged = self.mod._ensure_kind_phys_default(merged, log) diff --git a/unit-tests/test_registered_dimensions.py b/unit-tests/test_registered_dimensions.py index bff8923f..73a77c5e 100644 --- a/unit-tests/test_registered_dimensions.py +++ b/unit-tests/test_registered_dimensions.py @@ -1,6 +1,6 @@ """Tests for :mod:`metadata.registered_dimensions`. -This module is the single source of truth for capgen-ng's registered +This module is the single source of truth for capgen's registered scalar-index dimension table. Tests here cover: * Every entry in :data:`SCALAR_INDEX_DIMS` is present (regression @@ -29,7 +29,7 @@ class TestSCalarIndexDimsContents(unittest.TestCase): """The registered table must contain at least the two pairings that - capgen-ng has committed to. Removing or renaming either is a + capgen has committed to. Removing or renaming either is a breaking change for hosts in production.""" def test_number_of_instances_pair(self): diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index 34b6a7bd..c939fc3e 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -2226,7 +2226,7 @@ class TestResolveSuiteMissingSchemeFailsLoudly(unittest.TestCase): """An SDF that references a scheme whose ``.meta`` was not passed via ``--scheme-files`` MUST raise ``CCPPError`` at resolve time, listing every missing scheme. Regression for the silent-empty-cap - bug: capgen-ng would otherwise emit a syntactically valid but + bug: capgen would otherwise emit a syntactically valid but semantically empty group cap and the build would succeed with the wrong runtime behaviour.""" @@ -3441,7 +3441,7 @@ def _load_constituent_host_dict(): ddt_tbls = [] fw_meta = os.path.join( os.path.dirname(os.path.dirname(__file__)), - 'capgen-ng', 'src', 'ccpp_constituent_prop_mod.meta', + 'capgen', 'src', 'ccpp_constituent_prop_mod.meta', ) if os.path.isfile(fw_meta): ddt_tbls = parse_metadata_file(fw_meta) @@ -4233,7 +4233,7 @@ def _scheme_var(self, local, std_name, dims, intent='in'): def test_ccpp_constituents_dim_lands_on_dedicated_field(self): # Uses _load_full_host_dict (no ccpp_model_constituents_t DDT # instance), so the host-wins gate does NOT fire and the - # resolver routes through capgen-ng's auto-provisioning path. + # resolver routes through capgen's auto-provisioning path. from generator.suite_resolver import _resolve_constituent_arg hd = _load_full_host_dict() suite_var = self._scheme_var( @@ -4283,7 +4283,7 @@ def test_minimum_values_routes_to_vars_minvalue(self): # ``vars_minvalue`` member, not Path 2 (constituent auto- # provisioning). Drives cam-sima's ``qneg`` scheme: under the # original capgen contract this was a host-USE'd module array; - # capgen-ng exposes it through the per-instance object. + # capgen exposes it through the per-instance object. from generator.suite_resolver import _resolve_constituent_arg hd = _load_full_host_dict() suite_var = self._scheme_var( @@ -4766,7 +4766,7 @@ def test_host_index_of_resolves_to_host_local_name(self): def test_unclaimed_index_of_still_routes_to_constituents(self): """The framework auto-provisioning path is preserved for ``index_of_`` names the host does NOT declare — required for - capgen-ng-owned constituent flows (cf. the advection e2e test).""" + capgen-owned constituent flows (cf. the advection e2e test).""" hd = build_flat_host_dict(_parse(self._HOST_SRC), [], []) suite_var = self._scheme_var( 'idx_other', 'index_of_some_other_constituent_not_in_host', @@ -4792,7 +4792,7 @@ class TestDimDDTComponentResolution(unittest.TestCase): (``access_path == local_name``). Historical: this was a long-standing pain point in the original - capgen → SCM migration. The pre-2026-05-13 capgen-ng emitted + capgen → SCM migration. The pre-2026-05-13 capgen emitted ``use scm_type_defs, only: levs`` and similar bogus imports for every DDT-component dim, producing many ``Symbol referenced ... not found in module`` errors at compile time. diff --git a/unit-tests/test_suite_types.py b/unit-tests/test_suite_types.py index 5ffa2c80..5a2fedfd 100644 --- a/unit-tests/test_suite_types.py +++ b/unit-tests/test_suite_types.py @@ -116,7 +116,7 @@ def test_character_len_parameter_symbol(self): def test_character_len_assumed_rejected(self): """``character(len=*)`` cannot appear as a DDT component, so - capgen-ng cannot synthesise a wrapper. Error must explain that + capgen cannot synthesise a wrapper. Error must explain that and suggest using a concrete length or deferred-length.""" with self.assertRaisesRegex(CCPPError, 'cannot appear as a DDT component'): _ptr_type_name('character', 'len=*', 1) diff --git a/unit-tests/test_suite_xml.py b/unit-tests/test_suite_xml.py index 054ed7cf..11d94bac 100644 --- a/unit-tests/test_suite_xml.py +++ b/unit-tests/test_suite_xml.py @@ -19,9 +19,9 @@ Run with:: - python -m pytest capgen-ng/tests/test_suite_xml.py -v + python -m pytest capgen/tests/test_suite_xml.py -v -or include ``--doctest-modules capgen-ng/generator/suite_xml.py``. +or include ``--doctest-modules capgen/generator/suite_xml.py``. """ import filecmp @@ -36,7 +36,7 @@ # ---- path setup ------------------------------------------------------------ _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) _REPO_ROOT = os.path.dirname(_TESTS_DIR) -_PKG_ROOT = os.path.join(_REPO_ROOT, 'capgen-ng') +_PKG_ROOT = os.path.join(_REPO_ROOT, 'capgen') for _p in (_PKG_ROOT, _REPO_ROOT): if _p not in sys.path: sys.path.insert(0, _p) diff --git a/unit-tests/test_variable_resolver.py b/unit-tests/test_variable_resolver.py index 28767911..072d79d8 100644 --- a/unit-tests/test_variable_resolver.py +++ b/unit-tests/test_variable_resolver.py @@ -915,7 +915,7 @@ def test_empty_inputs(self): class TestRule2LeafScalarDimRejection(unittest.TestCase): """Rule 2 of the registered-scalar-index-dimension contract (see - capgen-ng/metadata/registered_dimensions.py): leaf data variables — + capgen/metadata/registered_dimensions.py): leaf data variables — intrinsic-typed or external-typed, the kind a scheme binds to — MUST NOT declare a registered scalar-index dim like ``number_of_threads``. ``build_flat_host_dict`` is the validation @@ -977,7 +977,7 @@ def test_ddt_instance_with_non_registered_dim_skips_field_flatten(self): legitimate pattern: schemes take the whole sliced DDT array as a single arg, not individual flattened inner fields. - capgen-ng must NOT flatten the inner fields in this case — + capgen must NOT flatten the inner fields in this case — attempting to bake a scalar subscript would emit invalid Fortran like ``parent%var%field(...)``. Instead, only the DDT-instance's own entry is recorded; schemes that take it @@ -1023,7 +1023,7 @@ def test_ddt_instance_with_non_registered_dim_skips_field_flatten(self): d = build_flat_host_dict(host_tbls, [], ddt_tbls) self.assertIn('longwave_radiation_fluxes', d) # Inner field is NOT flattened (would have required a scalar - # subscript capgen-ng can't synthesize). + # subscript capgen can't synthesize). self.assertNotIn('surface_upwelling_longwave_radiation_flux', d) def test_ddt_instance_with_non_registered_dim_no_fields_accepted(self):